From 64e265141cc82951034a90ab0ad2e5ed67741b6a Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sat, 1 Aug 2026 07:46:38 -0700 Subject: [PATCH] fix(windows): count GPU crashes across launches so the fallback can engage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The safe-graphics fallback engages on the 3rd GPU child crash inside a 30s window, but the crash shape it exists for kills the app ~0.8s after launch. The in-memory tracker therefore resets to zero every launch and the same-session threshold is structurally unreachable: across 39 Windows bundles, 53 of the 55 launches that recorded a GPU death recorded exactly one, and none reached three. Add gpu-crash-history.json, a build-scoped record of GPU death timestamps read and written alongside the existing marker, so the count survives the launch that dies. Three deaths inside 5 minutes now engage the fallback whether they land in one session or across a dozen. Only `crashed` and `abnormal-exit` accumulate durably. `launch-failed` clusters across launches for recoverable reasons — a driver update in flight, an RDP transition, a monitor hotplug — and durable counting would latch build-sticky safe graphics on a condition that clears itself. The in-process tracker still counts it, since three launch failures inside one 30s session is a broken GPU either way. The marker is written before the user consents, so add a Help-menu "Turn Off Safe Graphics Mode" escape hatch that clears both the marker and the count; clearing only one leaves the user latched on the launch after. --- src/main/app-relaunch.ts | 2 + .../gpu-crash-fallback-decision.test.ts | 24 ++ .../gpu-crash-fallback-decision.ts | 20 + ...gpu-fallback-engagement-resolution.test.ts | 143 +++++++ .../gpu-fallback-engagement-resolution.ts | 41 ++ src/main/index.ts | 155 +++++-- src/main/menu/register-app-menu.test.ts | 31 ++ src/main/menu/register-app-menu.ts | 22 +- .../startup/desktop-startup-ordering.test.ts | 160 ++++++- src/main/startup/gpu-crash-history.test.ts | 404 ++++++++++++++++++ src/main/startup/gpu-crash-history.ts | 242 +++++++++++ src/main/startup/gpu-fallback-marker.test.ts | 140 +++++- src/main/startup/gpu-fallback-marker.ts | 15 +- src/renderer/src/i18n/locales/en.json | 1 + 14 files changed, 1337 insertions(+), 63 deletions(-) create mode 100644 src/main/crash-reporting/gpu-fallback-engagement-resolution.test.ts create mode 100644 src/main/crash-reporting/gpu-fallback-engagement-resolution.ts create mode 100644 src/main/startup/gpu-crash-history.test.ts create mode 100644 src/main/startup/gpu-crash-history.ts diff --git a/src/main/app-relaunch.ts b/src/main/app-relaunch.ts index ab7a725c847..9bc5d0e032e 100644 --- a/src/main/app-relaunch.ts +++ b/src/main/app-relaunch.ts @@ -5,6 +5,8 @@ import { recordDurableCrashBreadcrumb } from './crash-reporting/durable-crash-br export type AppRelaunchReason = | 'admin-restart' | 'gpu-fallback' + /** User turned safe graphics back off from the Help menu. */ + | 'gpu-fallback-opt-out' | 'profile-switch' | 'profile-transfer' | 'renderer-request' diff --git a/src/main/crash-reporting/gpu-crash-fallback-decision.test.ts b/src/main/crash-reporting/gpu-crash-fallback-decision.test.ts index b4c4e1210ed..a5786ba01dc 100644 --- a/src/main/crash-reporting/gpu-crash-fallback-decision.test.ts +++ b/src/main/crash-reporting/gpu-crash-fallback-decision.test.ts @@ -3,6 +3,7 @@ import { DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD, DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS, GpuCrashFallbackTracker, + countsTowardDurableGpuCrashHistory, isGpuChildProcessType, isGpuFallbackCrashCandidate } from './gpu-crash-fallback-decision' @@ -172,3 +173,26 @@ describe('isGpuFallbackCrashCandidate', () => { ).toBe(false) }) }) + +describe('countsTowardDurableGpuCrashHistory', () => { + it('counts the reasons the crash loop actually produces', () => { + // All 57 distinct GPU child deaths across the 39 cluster-E bundles are + // `crashed` (53 STATUS_BREAKPOINT) or `killed` — none are `launch-failed`. + expect(countsTowardDurableGpuCrashHistory('crashed')).toBe(true) + expect(countsTowardDurableGpuCrashHistory('abnormal-exit')).toBe(true) + }) + + it('refuses launch-failed, which clusters across launches for recoverable reasons', () => { + // A driver update, an RDP session transition, or a monitor hotplug can fail the + // GPU launch on several consecutive launches. Counting those durably would latch + // build-sticky safe graphics on a condition that clears itself; the in-process + // tracker still catches 3 inside one 30s session. + expect(countsTowardDurableGpuCrashHistory('launch-failed')).toBe(false) + }) + + it('ignores reasons that are not fallback candidates at all', () => { + for (const reason of ['killed', 'clean-exit', 'oom', 'integrity-failure', '']) { + expect([reason, countsTowardDurableGpuCrashHistory(reason)]).toEqual([reason, false]) + } + }) +}) diff --git a/src/main/crash-reporting/gpu-crash-fallback-decision.ts b/src/main/crash-reporting/gpu-crash-fallback-decision.ts index e962952c695..94b652e808e 100644 --- a/src/main/crash-reporting/gpu-crash-fallback-decision.ts +++ b/src/main/crash-reporting/gpu-crash-fallback-decision.ts @@ -7,6 +7,26 @@ export type GpuCrashFallbackOptions = { const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch-failed']) +/** + * Reasons that may accumulate in the *cross-launch* history. + * + * Why `launch-failed` is excluded here but kept in-process: a GPU that fails to + * launch clusters across launches for recoverable reasons — a driver update in + * flight, an RDP session transition, a monitor hotplug — and durable counting + * would latch build-sticky safe graphics on a condition that clears itself. Three + * launch failures inside one 30s session is still a broken GPU, so the in-process + * tracker keeps counting them. + * + * Costs nothing on the observed data: the 39 cluster-E bundles hold 57 distinct + * GPU child deaths (deduped by pid and timestamp) — 56 `crashed`, 53 of those + * STATUS_BREAKPOINT, one `killed`, and zero `launch-failed`. + */ +const DURABLE_GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed']) + +export function countsTowardDurableGpuCrashHistory(reason: string): boolean { + return DURABLE_GPU_FALLBACK_CRASH_REASONS.has(reason) +} + // Why: on old/flaky GPU drivers the GPU child process crashes (STATUS_BREAKPOINT // / ANGLE-D3D init failure) repeatedly - Windows clusters F0BDNADU79Q and // F0BDNRZ5MDG. GPU child deaths are intentionally suppressed as recoverable diff --git a/src/main/crash-reporting/gpu-fallback-engagement-resolution.test.ts b/src/main/crash-reporting/gpu-fallback-engagement-resolution.test.ts new file mode 100644 index 00000000000..e131e3f95eb --- /dev/null +++ b/src/main/crash-reporting/gpu-fallback-engagement-resolution.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest' +import type { GpuFallbackEngagementOutcome } from './gpu-fallback-engagement' +import { resolveGpuFallbackEngagement } from './gpu-fallback-engagement-resolution' + +const USER_DATA = '/tmp/orca-userdata' + +describe('resolveGpuFallbackEngagement', () => { + // Why exhaustive: every field here is a one-token change away from defeating the + // feature silently, and this table is the only thing that can see it. + const expected: Record< + GpuFallbackEngagementOutcome, + { + provisionalMarkerPath: string | null + rePersistConsentedMarker: boolean + clearDurableHistory: boolean + relaunch: boolean + } + > = { + restart: { + provisionalMarkerPath: null, + rePersistConsentedMarker: true, + clearDurableHistory: true, + relaunch: true + }, + 'confirmed-quitting': { + provisionalMarkerPath: null, + rePersistConsentedMarker: true, + clearDurableHistory: true, + relaunch: false + }, + deferred: { + provisionalMarkerPath: null, + rePersistConsentedMarker: false, + clearDurableHistory: true, + relaunch: false + }, + 'deferred-uncleared': { + provisionalMarkerPath: USER_DATA, + rePersistConsentedMarker: false, + clearDurableHistory: true, + relaunch: false + }, + latched: { + provisionalMarkerPath: USER_DATA, + rePersistConsentedMarker: false, + clearDurableHistory: true, + relaunch: false + }, + 'marker-failed': { + provisionalMarkerPath: null, + rePersistConsentedMarker: false, + clearDurableHistory: false, + relaunch: false + } + } + + for (const [outcome, resolution] of Object.entries(expected)) { + it(`resolves ${outcome}`, () => { + expect( + resolveGpuFallbackEngagement(outcome as GpuFallbackEngagementOutcome, USER_DATA) + ).toEqual(resolution) + }) + } + + it('stands the shutdown drop down for every settled outcome', () => { + // Why its own assertion: keeping the provisional path on a settled outcome + // deletes the consented marker on the way out, so the relaunch comes back on + // the same broken GPU — the exact mutation the source-text tests could not see. + for (const outcome of ['restart', 'confirmed-quitting', 'deferred', 'marker-failed'] as const) { + expect([ + outcome, + resolveGpuFallbackEngagement(outcome, USER_DATA).provisionalMarkerPath + ]).toEqual([outcome, null]) + } + }) + + it('keeps the shutdown drop armed only while the marker is unconfirmed', () => { + for (const outcome of ['latched', 'deferred-uncleared'] as const) { + expect([ + outcome, + resolveGpuFallbackEngagement(outcome, USER_DATA).provisionalMarkerPath + ]).toEqual([outcome, USER_DATA]) + } + }) + + it('records consent on both paths where the user actually answered "restart"', () => { + // 'restart' latched pre-prompt with consented:false, so without the re-persist + // the primary consent path would land on disk indistinguishable from a latch. + expect(resolveGpuFallbackEngagement('restart', USER_DATA).rePersistConsentedMarker).toBe(true) + expect( + resolveGpuFallbackEngagement('confirmed-quitting', USER_DATA).rePersistConsentedMarker + ).toBe(true) + expect(resolveGpuFallbackEngagement('latched', USER_DATA).rePersistConsentedMarker).toBe(false) + expect(resolveGpuFallbackEngagement('deferred', USER_DATA).rePersistConsentedMarker).toBe(false) + }) + + it('relaunches only on an answered restart that no quit already claimed', () => { + expect(resolveGpuFallbackEngagement('restart', USER_DATA).relaunch).toBe(true) + for (const outcome of [ + 'confirmed-quitting', + 'deferred', + 'deferred-uncleared', + 'latched', + 'marker-failed' + ] as const) { + expect([outcome, resolveGpuFallbackEngagement(outcome, USER_DATA).relaunch]).toEqual([ + outcome, + false + ]) + } + }) + + it('drops the cross-launch count once the prompt has been answered', () => { + // Why: after 'deferred' the app keeps running with gpuFallbackActiveThisLaunch + // false, so handleGpuChildCrash still runs — but the stored 2 entries would make + // the next GPU crash inside the 5-minute window the "third" and re-latch on a + // prompt the user just declined. + for (const outcome of [ + 'restart', + 'confirmed-quitting', + 'deferred', + 'deferred-uncleared', + 'latched' + ] as const) { + expect([ + outcome, + resolveGpuFallbackEngagement(outcome, USER_DATA).clearDurableHistory + ]).toEqual([outcome, true]) + } + }) + + it('keeps the count armed when the marker write is what failed', () => { + // 'marker-failed' never prompted, so the crashes are still unaddressed. Clearing + // would demand three fresh crashes before retrying a transient EPERM. + expect(resolveGpuFallbackEngagement('marker-failed', USER_DATA).clearDurableHistory).toBe(false) + }) + + it('echoes back the caller path so a split userData cannot go unnoticed', () => { + expect(resolveGpuFallbackEngagement('latched', '/other/path').provisionalMarkerPath).toBe( + '/other/path' + ) + }) +}) diff --git a/src/main/crash-reporting/gpu-fallback-engagement-resolution.ts b/src/main/crash-reporting/gpu-fallback-engagement-resolution.ts new file mode 100644 index 00000000000..745e7d6637e --- /dev/null +++ b/src/main/crash-reporting/gpu-fallback-engagement-resolution.ts @@ -0,0 +1,41 @@ +import { + shouldKeepProvisionalGpuFallbackMarker, + type GpuFallbackEngagementOutcome +} from './gpu-fallback-engagement' + +/** + * Bookkeeping that must follow an `engageGpuFallback` outcome. + * + * Why its own module: this lived inline in index.ts, where the only coverage was + * source-text matching. A mutation that kept the provisional path on *every* + * outcome — which deletes the consented marker on the way out and brings the app + * back on the same broken GPU — passed the entire suite. + */ +export type GpuFallbackEngagementResolution = { + /** userData path the shutdown drop must keep retrying, or null to stand down. */ + provisionalMarkerPath: string | null + /** Rewrite the marker with consent recorded, because the user answered "restart". */ + rePersistConsentedMarker: boolean + /** Drop the cross-launch crash count, because this engagement settled it. */ + clearDurableHistory: boolean + /** Relaunch into safe graphics now. */ + relaunch: boolean +} + +export function resolveGpuFallbackEngagement( + outcome: GpuFallbackEngagementOutcome, + userDataPath: string +): GpuFallbackEngagementResolution { + return { + provisionalMarkerPath: shouldKeepProvisionalGpuFallbackMarker(outcome) ? userDataPath : null, + // Why both outcomes: the pre-prompt persist recorded consented:false, and a + // session-end drop can take the marker while the prompt is still open. + rePersistConsentedMarker: outcome === 'restart' || outcome === 'confirmed-quitting', + // Why every outcome but 'marker-failed': a declined prompt must not re-fire on + // the very next GPU crash, and a consented one is about to relaunch. Only + // 'marker-failed' never prompted at all, so its count stays armed to retry a + // write that failed transiently rather than demanding three fresh crashes. + clearDurableHistory: outcome !== 'marker-failed', + relaunch: outcome === 'restart' + } +} diff --git a/src/main/index.ts b/src/main/index.ts index 629e812a1f8..de42f0f95b3 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -130,6 +130,7 @@ import { enableRendererHeapHeadroom } from './startup/renderer-heap-headroom' import { ensureVirtualDisplayForHeadlessServe } from './startup/ensure-virtual-display' import { clearGpuFallbackMarker, + gpuFallbackMarkerFileExists, readActiveGpuFallbackMarker, sweepStaleGpuFallbackMarkerTempFiles, writeGpuFallbackMarker, @@ -137,14 +138,22 @@ import { type WindowsGpuFallbackEnvironment } from './startup/gpu-fallback-marker' import { - engageGpuFallback, - shouldKeepProvisionalGpuFallbackMarker -} from './crash-reporting/gpu-fallback-engagement' + DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS, + clearGpuCrashHistory, + evaluateGpuCrashHistory, + gpuCrashHistoryFileExists, + inertGpuCrashHistoryDecision, + persistGpuCrashTimes, + sweepStaleGpuCrashHistoryTempFiles +} from './startup/gpu-crash-history' +import { engageGpuFallback } from './crash-reporting/gpu-fallback-engagement' +import { resolveGpuFallbackEngagement } from './crash-reporting/gpu-fallback-engagement-resolution' import { applyGpuFallbackCommandLineSwitches } from './startup/gpu-fallback-switches' import { DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD, DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS, GpuCrashFallbackTracker, + countsTowardDurableGpuCrashHistory, isGpuFallbackCrashCandidate } from './crash-reporting/gpu-crash-fallback-decision' import { promptForGpuFallbackRestart } from './crash-reporting/gpu-fallback-restart-prompt' @@ -369,6 +378,9 @@ const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({ threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD }) let gpuFallbackActiveThisLaunch = false +// Why a separate latch: the durable history and the in-process tracker can each +// fire, so without this a later same-session crash opens a second restart prompt. +let gpuFallbackEngagementStarted = false // Why: the safe-graphics marker is written before the user answers, so it must be // dropped again if the app reaches an orderly shutdown instead of a GPU kill. let provisionalGpuFallbackUserDataPath: string | null = null @@ -1565,51 +1577,100 @@ function maybeApplyGpuFallbackForThisLaunch(): void { // Why canonical: handleGpuChildCrash writes through the same helper, and a late // app.setName/setPath would silently split the read and write paths. const userDataPath = getCanonicalUserDataPath() - // A kill between the marker's write and rename orphans a temp file next to it. - void sweepStaleGpuFallbackMarkerTempFiles(userDataPath).catch(() => undefined) + // A kill between a write and its rename orphans a temp file next to the target. + // Why gated: the sweep readdirs all of userData — Cache, Code Cache, GPUCache, + // Local Storage — on a path that runs before whenReady on every Windows launch, + // and a temp can only be orphaned next to a file the fallback already wrote. + const sweepForOrphans = + gpuFallbackMarkerFileExists(userDataPath) || gpuCrashHistoryFileExists(userDataPath) const marker = readActiveGpuFallbackMarker(userDataPath, getGpuFallbackEnvironment()) + if (sweepForOrphans) { + void sweepStaleGpuFallbackMarkerTempFiles(userDataPath).catch(() => undefined) + void sweepStaleGpuCrashHistoryTempFiles(userDataPath).catch(() => undefined) + } if (!marker) { return } app.disableHardwareAcceleration() const appliedSwitches = applyGpuFallbackCommandLineSwitches(app.commandLine, process.platform) gpuFallbackActiveThisLaunch = true + // Why: the user is already in safe graphics, so pre-fallback crash times must not + // re-fire the prompt on the first unrelated GPU hiccup of this launch. + clearGpuCrashHistory(userDataPath) // Why: with no GPU child left, child-process-gone can't report a GPU fault, so // name the applied switches in the trail any later crash report carries. recordCrashBreadcrumb('gpu_fallback_applied', { crashesInWindow: marker.crashesInWindow, + consented: marker.consented, switches: appliedSwitches.join(',') }) } +// Why swallow: the marker is already on disk, so a tracer failure here must not +// surface as 'marker-failed' and cancel the restart the write just earned. +function recordGpuFallbackMarkerPersisted( + consented: boolean, + data: Record +): void { + try { + recordDurableCrashBreadcrumb('gpu_fallback_marker_persisted', { ...data, consented }) + } catch { + // Diagnostics only; the marker is what the next launch reads. + } +} + // Why: a burst of GPU child crashes means HW acceleration is unusable — persist a build-scoped marker and offer software rendering. async function handleGpuChildCrash(reason: string, exitCode: number | null): Promise { - // Software rendering already active or shutting down: nothing more to do. - if (gpuFallbackActiveThisLaunch || isQuitting || isServeMode) { + // Software rendering already active, engagement under way, or shutting down. + if (gpuFallbackActiveThisLaunch || gpuFallbackEngagementStarted || isQuitting || isServeMode) { return } - const result = gpuCrashFallbackTracker.recordGpuCrash(performance.now()) - if (!result.shouldEngageFallback) { - return - } - recordCrashBreadcrumb('gpu_fallback_engaged', { - reason, - exitCode, - crashesInWindow: result.crashesInWindow - }) - const engagedAt = Date.now() + // Why first: everything below this point must stay inert off Windows. const environment = getWindowsGpuFallbackEnvironment() if (!environment) { return } - const fallbackData = { - processReason: reason, - exitCode, - crashesInWindow: result.crashesInWindow - } // Why canonical: the reader above resolves the same way, and a late app.setName // would otherwise move the write out from under it. const userDataPath = getCanonicalUserDataPath() + const inProcess = gpuCrashFallbackTracker.recordGpuCrash(performance.now()) + // Why durable too: this failure kills the launch ~0.8s in, so the in-memory count + // resets at 1 and the same-session threshold is structurally unreachable. + // Why reason-gated: `launch-failed` clusters across launches for recoverable + // reasons (driver update, RDP transition, monitor hotplug) — see the reason set. + const countsDurably = countsTowardDurableGpuCrashHistory(reason) + const durable = countsDurably + ? evaluateGpuCrashHistory(userDataPath, environment, { + now: Date.now(), + windowMs: DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS, + threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD + }) + : inertGpuCrashHistoryDecision() + if (!inProcess.shouldEngageFallback && !durable.crossesThreshold) { + // Why only here: on the crash that fires, this fsync would land in front of the + // marker write, which is racing a kill that can arrive ~7ms after the failure. + // Why gated: an inert decision carries no times, so writing it would erase the + // real ones an excluded reason is only supposed to be ignored by. + if (countsDurably) { + persistGpuCrashTimes(userDataPath, environment, durable.crashTimes) + } + return + } + gpuFallbackEngagementStarted = true + const crashesInWindow = Math.max(inProcess.crashesInWindow, durable.crashesInWindow) + recordCrashBreadcrumb('gpu_fallback_engaged', { + reason, + exitCode, + crashesInWindow, + crashesAcrossLaunches: durable.crashesInWindow, + trigger: inProcess.shouldEngageFallback ? 'in-process' : 'durable' + }) + const engagedAt = Date.now() + const fallbackData = { + processReason: reason, + exitCode, + crashesInWindow + } const window = mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined // Why: Chromium fatally CHECKs the browser process on the 6th GPU failure, so // the marker must land before the prompt or an unattended crash never latches. @@ -1617,10 +1678,13 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro persistMarker: () => { writeGpuFallbackMarker( userDataPath, - { engagedAt, crashesInWindow: result.crashesInWindow }, + { engagedAt, crashesInWindow, consented: false }, environment ) provisionalGpuFallbackUserDataPath = userDataPath + // Why after the write: a kill lands here more often than anywhere else, and + // this is the only record separating a provisional latch from a consented one. + recordGpuFallbackMarkerPersisted(false, fallbackData) }, clearMarker: () => clearGpuFallbackMarker(userDataPath), prompt: () => promptForGpuFallbackRestart(window), @@ -1639,23 +1703,29 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro onRestartDeferred: () => recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData) }) - if (!shouldKeepProvisionalGpuFallbackMarker(outcome)) { - provisionalGpuFallbackUserDataPath = null + // Why delegated: this mapping is unit-tested; deriving it inline here left the + // whole feature defeatable by a one-token change no test could see. + const resolution = resolveGpuFallbackEngagement(outcome, userDataPath) + provisionalGpuFallbackUserDataPath = resolution.provisionalMarkerPath + // Why here and not at fire time: a kill during the prompt must leave the count + // armed to fire again, while a decline must not re-prompt on the very next crash. + if (resolution.clearDurableHistory) { + clearGpuCrashHistory(userDataPath) } - if (outcome === 'confirmed-quitting') { - // Why re-persist: a session-end drop can land while the prompt is still open. + if (resolution.rePersistConsentedMarker) { try { writeGpuFallbackMarker( userDataPath, - { engagedAt, crashesInWindow: result.crashesInWindow }, + { engagedAt, crashesInWindow, consented: true }, environment ) + recordGpuFallbackMarkerPersisted(true, fallbackData) } catch (error) { console.warn('[gpu-fallback] failed to re-persist marker after consent:', error) recordDurableCrashBreadcrumb('gpu_fallback_marker_failed', fallbackData) } } - if (outcome !== 'restart') { + if (!resolution.relaunch) { return } isQuitting = true @@ -1665,12 +1735,37 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro app.exit(0) } +// Why a user-visible escape hatch: the marker is build-sticky, so a false latch +// would otherwise hold a working GPU in software rendering until the next update. +function turnOffGpuFallback(): void { + const userDataPath = getCanonicalUserDataPath() + clearGpuCrashHistory(userDataPath) + if (!clearGpuFallbackMarker(userDataPath)) { + // Why not relaunch: the next launch would read the marker and come straight back. + recordDurableCrashBreadcrumb('gpu_fallback_marker_clear_failed', { phase: 'user-opt-out' }) + return + } + provisionalGpuFallbackUserDataPath = null + recordDurableCrashBreadcrumb('gpu_fallback_disabled_by_user') + isQuitting = true + relaunchApp('gpu-fallback-opt-out') + destroySystemTray() + app.exit(0) +} + // Why: an orderly shutdown proves Chromium's GPU CHECK never landed, so a marker // the user never confirmed must not survive into the next launch. function dropUnconfirmedGpuFallbackMarker(): void { if (provisionalGpuFallbackUserDataPath === null) { return } + // Why the count too: an orderly shutdown proves the burst was survivable, so + // leaving it armed would re-latch on the next single GPU crash inside the window. + // Why not hoisted above the null guard: only an engagement sets this path, and a + // quit that clears the count unconditionally would wipe the 1-2 crashes recorded + // by a loop the user quits out of by hand — the counting this file exists for. + // The residual is bounded: an uncleared count ages out of the 5-minute window. + clearGpuCrashHistory(provisionalGpuFallbackUserDataPath) if (!clearGpuFallbackMarker(provisionalGpuFallbackUserDataPath)) { // `quit` is the last JS event, so this is terminal: the next launch starts in // safe graphics and clears itself on the next Orca or Electron version. @@ -2693,6 +2788,8 @@ void app.whenReady().then(async () => { registerAppMenu({ appMenuLabel: devInstanceIdentity.name, + isGpuFallbackActive: () => gpuFallbackActiveThisLaunch, + onTurnOffGpuFallback: turnOffGpuFallback, onCheckForUpdates: (options) => runUserInitiatedUpdateCheck(options), onBeforeReload: ({ ignoreCache, webContentsId }) => { if (mainWindow?.webContents.id === webContentsId) { diff --git a/src/main/menu/register-app-menu.test.ts b/src/main/menu/register-app-menu.test.ts index 8433fd93e8a..fa8c90f2238 100644 --- a/src/main/menu/register-app-menu.test.ts +++ b/src/main/menu/register-app-menu.test.ts @@ -417,4 +417,35 @@ describe('registerAppMenu', () => { expect(appearanceSubmenu.find((item) => item.label === leftLabel)?.accelerator).toBeUndefined() expect(appearanceSubmenu.find((item) => item.label === rightLabel)?.accelerator).toBeUndefined() }) + + it('offers the safe-graphics escape hatch only while the fallback is latched', () => { + const label = 'Turn Off Safe Graphics Mode' + + registerAppMenu(buildMenuOptions()) + expect(getSubmenu(getTemplate(), 'Help').map((item) => item.label)).not.toContain(label) + + buildFromTemplateMock.mockReset() + buildFromTemplateMock.mockImplementation((template) => ({ template })) + const options = { + ...buildMenuOptions(), + isGpuFallbackActive: vi.fn(() => true), + onTurnOffGpuFallback: vi.fn() + } + registerAppMenu(options) + + // Why it must exist: the marker is build-sticky, so without this a false latch + // holds a working GPU in software rendering until the next app update. + const item = getSubmenu(getTemplate(), 'Help').find((entry) => entry.label === label) + expect(item).toBeDefined() + item?.click?.({} as never, {} as never, {} as never) + expect(options.onTurnOffGpuFallback).toHaveBeenCalledTimes(1) + }) + + it('hides the escape hatch when no handler is wired, rather than a dead item', () => { + registerAppMenu({ ...buildMenuOptions(), isGpuFallbackActive: vi.fn(() => true) }) + + expect(getSubmenu(getTemplate(), 'Help').map((item) => item.label)).not.toContain( + 'Turn Off Safe Graphics Mode' + ) + }) }) diff --git a/src/main/menu/register-app-menu.ts b/src/main/menu/register-app-menu.ts index d1e43d4fb82..d01e1c92a5b 100644 --- a/src/main/menu/register-app-menu.ts +++ b/src/main/menu/register-app-menu.ts @@ -37,6 +37,10 @@ type RegisterAppMenuOptions = { onToggleAppearance: (key: AppearanceMenuKey) => void getAppearanceState: () => AppearanceMenuState getKeybindings?: () => KeybindingOverrides | undefined + // Why an escape hatch: the safe-graphics marker is build-sticky, so without this + // a false latch leaves a user in software rendering until the next app update. + isGpuFallbackActive?: () => boolean + onTurnOffGpuFallback?: () => void // Why: the macOS app-menu title. Passed the per-branch dev label since // app.name is now pinned to a stable value for Keychain-key stability. appMenuLabel?: string @@ -57,7 +61,9 @@ function buildAndApplyMenu(options: RegisterAppMenuOptions): void { onToggleRightSidebar, onToggleAppearance, getAppearanceState, - getKeybindings + getKeybindings, + isGpuFallbackActive, + onTurnOffGpuFallback } = options const isMac = process.platform === 'darwin' @@ -292,10 +298,24 @@ function buildAndApplyMenu(options: RegisterAppMenuOptions): void { submenu: [{ role: 'minimize' }, { role: 'zoom' }] } + // Why only while latched: the item is meaningless otherwise, and the marker is + // read before whenReady so this state cannot change during the launch. + const gpuFallbackItems: Electron.MenuItemConstructorOptions[] = + isGpuFallbackActive?.() && onTurnOffGpuFallback + ? [ + { type: 'separator' }, + { + label: translateMain('menu.turnOffSafeGraphics', 'Turn Off Safe Graphics Mode'), + click: () => onTurnOffGpuFallback() + } + ] + : [] + const helpMenu: Electron.MenuItemConstructorOptions = { label: translateMain('menu.help', 'Help'), submenu: [ crashReportItem, + ...gpuFallbackItems, { type: 'separator' }, featureTourItem, setupGuideItem, diff --git a/src/main/startup/desktop-startup-ordering.test.ts b/src/main/startup/desktop-startup-ordering.test.ts index c4873449f25..c0abadab497 100644 --- a/src/main/startup/desktop-startup-ordering.test.ts +++ b/src/main/startup/desktop-startup-ordering.test.ts @@ -137,15 +137,109 @@ describe('startup ordering', () => { expect(persistBody).not.toMatch(/\bawait[\s(]/) // Why exactly two writers: the pre-prompt persist, plus the re-persist that - // recovers a consented restart whose marker a session-end drop already took. + // records consent (and recovers a marker a session-end drop already took). expect(handler.split('writeGpuFallbackMarker(')).toHaveLength(3) - const confirmedQuittingIndex = handler.indexOf("outcome === 'confirmed-quitting'") - expect(confirmedQuittingIndex).toBeGreaterThanOrEqual(0) - expect(handler.lastIndexOf('writeGpuFallbackMarker(')).toBeGreaterThan(confirmedQuittingIndex) + const resolutionIndex = handler.indexOf('resolveGpuFallbackEngagement(outcome') + expect(resolutionIndex).toBeGreaterThanOrEqual(0) + expect(handler.lastIndexOf('writeGpuFallbackMarker(')).toBeGreaterThan(resolutionIndex) - // Why pinned: the outcome -> retry mapping is unit-tested, so the handler must - // defer to it rather than re-deriving the outcome list inline. - expect(handler).toContain('if (!shouldKeepProvisionalGpuFallbackMarker(outcome)) {') + // Why pinned: the outcome -> bookkeeping mapping is unit-tested in + // gpu-fallback-engagement-resolution, so re-deriving it inline here would + // restore the zero-coverage gap this extraction closed. + expect(handler).toContain( + 'provisionalGpuFallbackUserDataPath = resolution.provisionalMarkerPath' + ) + expect(handler).toContain('if (resolution.rePersistConsentedMarker) {') + expect(handler).toContain('if (!resolution.relaunch) {') + expect(handler).not.toContain('shouldKeepProvisionalGpuFallbackMarker(') + + // Why: consent is what separates a latch the user chose from one the crash + // imposed, and without it on disk the fix's own efficacy is unmeasurable. + expect(handler).toContain('consented: false') + expect(handler).toContain('consented: true') + }) + + it('counts GPU crashes across launches, not just within one', () => { + const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8') + const start = source.indexOf('async function handleGpuChildCrash(') + const end = source.indexOf('function recordProcessGoneCrash(', start) + const handler = source.slice(start, end) + + expect(start).toBeGreaterThanOrEqual(0) + expect(end).toBeGreaterThan(start) + + // Why both: the in-process burst path is what catches a driver that fails + // mid-session; the durable path is the only one reachable when the crash + // kills the launch in 0.8s. + const inProcessIndex = handler.indexOf('gpuCrashFallbackTracker.recordGpuCrash(') + const durableIndex = handler.indexOf('evaluateGpuCrashHistory(') + const gateIndex = handler.indexOf( + 'if (!inProcess.shouldEngageFallback && !durable.crossesThreshold) {' + ) + const persistIndex = handler.indexOf('persistGpuCrashTimes(') + expect(inProcessIndex).toBeGreaterThanOrEqual(0) + expect(durableIndex).toBeGreaterThan(inProcessIndex) + expect(gateIndex).toBeGreaterThan(durableIndex) + // Why inside the non-firing branch: on the crash that fires, this write's fsync + // would land in front of the marker write that is racing Chromium's kill. + expect(persistIndex).toBeGreaterThan(gateIndex) + expect(persistIndex).toBeLessThan(handler.indexOf('gpuFallbackEngagementStarted = true')) + expect(handler.split('persistGpuCrashTimes(')).toHaveLength(2) + + // Why inert off Windows: the environment check must precede every write. + const environmentIndex = handler.indexOf('getWindowsGpuFallbackEnvironment()') + expect(environmentIndex).toBeGreaterThanOrEqual(0) + expect(environmentIndex).toBeLessThan(inProcessIndex) + expect(handler.indexOf('getCanonicalUserDataPath()')).toBeGreaterThan(environmentIndex) + + // Why cleared only after the prompt resolved: a kill while the modal is up must + // leave the count armed to fire again, while a decline must not re-prompt on the + // very next crash. + const clearIndex = handler.indexOf('clearGpuCrashHistory(userDataPath)') + expect(clearIndex).toBeGreaterThan(handler.indexOf('await engageGpuFallback({')) + // Why latched: either counter can fire, so a later same-session crash would + // otherwise stack a second modal on top of the first. + expect(handler).toContain('gpuFallbackEngagementStarted = true') + expect(handler).toContain('gpuFallbackEngagementStarted ||') + }) + + // Why one test over three call sites: they are the whole lifecycle of the durable + // count, each is a one-token deletion away from silently reverting the feature to + // the in-process tracker, and each survived the suite until this test existed. + it('writes and clears the cross-launch GPU crash count at exactly three places', () => { + const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8') + const slice = (from: string, to: string): string => { + const start = source.indexOf(from) + const end = source.indexOf(to, start) + expect(start).toBeGreaterThanOrEqual(0) + expect(end).toBeGreaterThan(start) + return source.slice(start, end) + } + + // Why guarded: an excluded reason (`launch-failed`) evaluates to the inert + // decision, whose crashTimes is []. Persisting that truncates the real count to + // zero, so one excluded death silently erases the whole accumulated history. + const handler = slice('async function handleGpuChildCrash(', 'function recordProcessGoneCrash(') + expect(handler).toContain('const countsDurably = countsTowardDurableGpuCrashHistory(reason)') + expect(handler).toContain( + 'if (countsDurably) {\n persistGpuCrashTimes(userDataPath, environment, durable.crashTimes)\n }' + ) + + // Why cleared on a safe-graphics launch: the marker is already applied, so the + // pre-fallback times must not re-fire the prompt on the first GPU hiccup here. + const reader = slice( + 'function maybeApplyGpuFallbackForThisLaunch()', + 'function recordGpuFallbackMarkerPersisted(' + ) + expect(reader).toContain('clearGpuCrashHistory(userDataPath)') + + // Why cleared on an orderly quit: a shutdown that reached 'quit' proves the + // burst was survivable, so an unconfirmed marker's count must not outlive it. + const drop = slice( + 'function dropUnconfirmedGpuFallbackMarker()', + 'function recordProcessGoneCrash(' + ) + expect(drop).toContain('clearGpuCrashHistory(provisionalGpuFallbackUserDataPath)') }) it('drops an unconfirmed GPU fallback marker only once the quit is committed', () => { @@ -166,9 +260,37 @@ describe('startup ordering', () => { 'dropUnconfirmedGpuFallbackMarker' ) - // Why source-wide: the marker may only be dropped by the shutdown helper and - // by the explicit "Keep Running" answer. - expect(source.split('clearGpuFallbackMarker(')).toHaveLength(3) + // Why source-wide: the marker may only be dropped by the shutdown helper, by + // the explicit "Keep Running" answer, and by the Help-menu opt-out. + expect(source.split('clearGpuFallbackMarker(')).toHaveLength(4) + expect(source).toContain('function turnOffGpuFallback()') + }) + + it('gives a latched user a way out that does not come straight back', () => { + const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8') + const start = source.indexOf('function turnOffGpuFallback()') + const end = source.indexOf('function dropUnconfirmedGpuFallbackMarker()', start) + const optOut = source.slice(start, end) + + expect(start).toBeGreaterThanOrEqual(0) + expect(end).toBeGreaterThan(start) + + // Why both: the marker latches the next launch and the count would re-latch the + // one after it, so clearing only one of them leaves the user still stuck. + expect(optOut).toContain('clearGpuCrashHistory(userDataPath)') + expect(optOut).toContain('clearGpuFallbackMarker(userDataPath)') + // Why gated on the clear: relaunching over a marker still on disk returns the + // user straight to safe graphics, having promised otherwise. + const clearIndex = optOut.indexOf('if (!clearGpuFallbackMarker(userDataPath)) {') + const relaunchIndex = optOut.indexOf("relaunchApp('gpu-fallback-opt-out'") + expect(clearIndex).toBeGreaterThanOrEqual(0) + expect(relaunchIndex).toBeGreaterThan(clearIndex) + // Why canonical: a split path would clear a marker the next launch never reads. + expect(optOut).toContain('getCanonicalUserDataPath()') + + // Why surfaced only while latched: the menu item is meaningless otherwise. + expect(source).toContain('isGpuFallbackActive: () => gpuFallbackActiveThisLaunch') + expect(source).toContain('onTurnOffGpuFallback: turnOffGpuFallback') }) it('reads and writes the GPU fallback marker through the same userData path', () => { @@ -184,11 +306,19 @@ describe('startup ordering', () => { // Why: a split between reader and writer paths turns the whole fallback into // a silent no-op — the marker lands somewhere the next launch never looks. expect(source.slice(readerStart, readerEnd)).toContain('getCanonicalUserDataPath()') - // Why here: a kill between the marker's write and rename orphans a temp file, - // and this is the only launch-time site that reclaims them. - expect(source.slice(readerStart, readerEnd)).toContain( - 'sweepStaleGpuFallbackMarkerTempFiles(userDataPath)' - ) + // Why here: a kill between a write and its rename orphans a temp file, and + // this is the only launch-time site that reclaims them. + const reader = source.slice(readerStart, readerEnd) + expect(reader).toContain('sweepStaleGpuFallbackMarkerTempFiles(userDataPath)') + expect(reader).toContain('sweepStaleGpuCrashHistoryTempFiles(userDataPath)') + // Why gated: the sweep readdirs all of userData (Cache / GPUCache / Local + // Storage) on a pre-whenReady path that runs on every Windows launch. + expect(reader).toContain('if (sweepForOrphans) {') + expect(reader).toContain('gpuFallbackMarkerFileExists(userDataPath)') + expect(reader).toContain('gpuCrashHistoryFileExists(userDataPath)') + // Why platform-gated: safe graphics is a Windows-only remedy, and + // enableMainProcessGpuFeatures() carries the macOS Graphite fix it skips. + expect(reader).toContain("process.platform !== 'win32'") expect(source.slice(readerEnd, writerEnd)).toContain('getCanonicalUserDataPath()') expect(source.slice(readerStart, writerEnd)).not.toContain("app.getPath('userData')") }) diff --git a/src/main/startup/gpu-crash-history.test.ts b/src/main/startup/gpu-crash-history.test.ts new file mode 100644 index 00000000000..8416d04a72a --- /dev/null +++ b/src/main/startup/gpu-crash-history.test.ts @@ -0,0 +1,404 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync +} from 'node:fs' +import os from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD } from '../crash-reporting/gpu-crash-fallback-decision' +import { + DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS, + GPU_CRASH_HISTORY_FILE, + inertGpuCrashHistoryDecision, + MAX_GPU_CRASH_HISTORY_ENTRIES, + appendGpuCrashTime, + clearGpuCrashHistory, + gpuCrashHistoryFileExists, + pruneGpuCrashTimes, + readGpuCrashHistory, + evaluateGpuCrashHistory, + persistGpuCrashTimes, + sweepStaleGpuCrashHistoryTempFiles +} from './gpu-crash-history' +import type { GpuFallbackEnvironment } from './gpu-fallback-marker' + +const WINDOW = DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS +const THRESHOLD = DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD +const NOW = 1_800_000_000_000 + +describe('pruneGpuCrashTimes', () => { + it('keeps only times inside the window, oldest first', () => { + expect(pruneGpuCrashTimes([NOW - 100, NOW - WINDOW - 1, NOW - 50], NOW, WINDOW)).toEqual([ + NOW - 100, + NOW - 50 + ]) + }) + + it('keeps a time exactly on the window edge', () => { + expect(pruneGpuCrashTimes([NOW - WINDOW], NOW, WINDOW)).toEqual([NOW - WINDOW]) + }) + + it('drops values a clock jump or a corrupt file made nonsense', () => { + // Why: these come off disk, so a NaN or a year-3000 stamp would otherwise pin + // the count permanently over the threshold. + const junk = [Number.NaN, Number.POSITIVE_INFINITY, -1, NOW + 86_400_000, '5', null, {}] + expect(pruneGpuCrashTimes([...junk, NOW - 10], NOW, WINDOW)).toEqual([NOW - 10]) + }) + + it('tolerates a small forward clock skew', () => { + // NTP corrections move the wall clock by seconds; discarding those would drop + // a crash that really did just happen. + expect(pruneGpuCrashTimes([NOW + 5_000], NOW, WINDOW)).toEqual([NOW + 5_000]) + }) + + it('returns nothing when `now` itself is unusable', () => { + expect(pruneGpuCrashTimes([NOW], Number.NaN, WINDOW)).toEqual([]) + }) + + it('drops entries stranded in the future by a backwards clock jump', () => { + // Why this matters most: a rolling window only ever prunes the OLD end, so a + // future-dated entry would sit in the window forever and two later crashes + // would latch build-sticky safe graphics with no burst and no consent. + const stored = [NOW - 30_000, NOW - 15_000, NOW] + for (const jumpMs of [ + 90_000, // NTP correction past the skew tolerance + 3_600_000, // hibernate / VM resume + 5 * 3_600_000 // dual-boot RTC skew: Windows local time vs Linux UTC + ]) { + const rewound = NOW - jumpMs + const kept = pruneGpuCrashTimes(stored, rewound, WINDOW) + // Nothing survives beyond the skew tolerance, so a rewind can never leave + // enough entries to reach the threshold on its own. + expect([jumpMs, kept.every((time) => time <= rewound + 60_000)]).toEqual([jumpMs, true]) + expect([jumpMs, kept.length < THRESHOLD]).toEqual([jumpMs, true]) + } + + // A rewind past the tolerance strands everything. + expect(pruneGpuCrashTimes(stored, NOW - 3_600_000, WINDOW)).toEqual([]) + // Inside the tolerance, only what is genuinely near-present survives. + expect(pruneGpuCrashTimes(stored, NOW - 90_000, WINDOW)).toEqual([NOW - 30_000]) + }) + + it('cannot be pushed over the threshold by future-dated entries', () => { + const rewound = NOW - 4 * 3_600_000 + const result = appendGpuCrashTime([NOW - 20_000, NOW - 10_000], rewound, { + windowMs: WINDOW, + threshold: THRESHOLD + }) + // Only the crash that actually just happened survives. + expect(result).toEqual({ + crashTimes: [rewound], + crashesInWindow: 1, + crossesThreshold: false + }) + }) + + it('does not let future-dated entries fill the cap and crowd out real crashes', () => { + const rewound = NOW - 3_600_000 + const future = Array.from({ length: MAX_GPU_CRASH_HISTORY_ENTRIES * 2 }, (_, i) => NOW + i) + const real = [rewound - 30_000, rewound - 15_000] + const result = appendGpuCrashTime([...future, ...real], rewound, { + windowMs: WINDOW, + threshold: THRESHOLD + }) + expect(result.crashTimes).toEqual([rewound - 30_000, rewound - 15_000, rewound]) + expect(result.crossesThreshold).toBe(true) + }) + + it('evicts oldest-first and still crosses the threshold at a full cap', () => { + // Why: a cap that evicted the newest, or that sat below the threshold, would + // silently suppress a machine crashing hard enough to overflow it. + const many = Array.from({ length: MAX_GPU_CRASH_HISTORY_ENTRIES + 40 }, (_, i) => NOW - 100 + i) + const result = appendGpuCrashTime(many, NOW, { windowMs: WINDOW, threshold: THRESHOLD }) + expect(result.crashTimes).toHaveLength(MAX_GPU_CRASH_HISTORY_ENTRIES) + expect(result.crashTimes[0]).toBeGreaterThan(many[0]) + expect(result.crashTimes.at(-1)).toBe(NOW) + expect(result.crossesThreshold).toBe(true) + expect(MAX_GPU_CRASH_HISTORY_ENTRIES).toBeGreaterThan(THRESHOLD) + }) +}) + +describe('DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS', () => { + it('spans the measured crash-loop shape without reaching back hours', () => { + // Measured: 18-24s median gap between consecutive crashing launches; the first + // 8 crashes on F0BN5HZL8FJ span 96s. Three crashes need ~40-50s. + expect(DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS).toBeGreaterThan(2 * 24_000 * 3) + // Every extra minute is false-positive surface, and the marker is written + // before the user consents. + expect(DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS).toBeLessThanOrEqual(300_000) + }) +}) + +describe('appendGpuCrashTime', () => { + it('does not cross the threshold before the third crash', () => { + expect(appendGpuCrashTime([], NOW, { windowMs: WINDOW, threshold: THRESHOLD })).toMatchObject({ + crashesInWindow: 1, + crossesThreshold: false + }) + expect( + appendGpuCrashTime([NOW - 20_000], NOW, { windowMs: WINDOW, threshold: THRESHOLD }) + ).toMatchObject({ crashesInWindow: 2, crossesThreshold: false }) + }) + + it('crosses on the third crash spread across launches', () => { + // The measured shape: one GPU death per launch, launches ~20s apart. + const result = appendGpuCrashTime([NOW - 44_000, NOW - 22_000], NOW, { + windowMs: WINDOW, + threshold: THRESHOLD + }) + expect(result).toMatchObject({ crashesInWindow: 3, crossesThreshold: true }) + expect(result.crashTimes).toEqual([NOW - 44_000, NOW - 22_000, NOW]) + }) + + it('does not count crashes that aged out of the window', () => { + expect( + appendGpuCrashTime([NOW - WINDOW - 1, NOW - WINDOW - 2], NOW, { + windowMs: WINDOW, + threshold: THRESHOLD + }) + ).toMatchObject({ crashesInWindow: 1, crossesThreshold: false }) + }) + + it('bounds the array so a crash loop cannot grow the file', () => { + const many = Array.from({ length: 200 }, (_, index) => NOW - 200 + index) + const result = appendGpuCrashTime(many, NOW, { windowMs: WINDOW, threshold: THRESHOLD }) + expect(result.crashTimes).toHaveLength(MAX_GPU_CRASH_HISTORY_ENTRIES) + // Why the newest: the dropped entries are the ones about to age out anyway. + expect(result.crashTimes.at(-1)).toBe(NOW) + expect(result.crashesInWindow).toBe(MAX_GPU_CRASH_HISTORY_ENTRIES) + }) + + it('never counts a crash time it refused to keep', () => { + const result = appendGpuCrashTime([NOW - 10, NOW - 20], Number.NaN, { + windowMs: WINDOW, + threshold: THRESHOLD + }) + expect(result).toEqual({ crashTimes: [], crashesInWindow: 0, crossesThreshold: false }) + }) +}) + +describe('gpu-crash-history file', () => { + let userDataPath: string + const environment = { + appVersion: '1.4.163', + electronVersion: '42.3.3', + platform: 'win32' as const + } + /** The production sequence for a crash that does not fire: evaluate, then persist. */ + const record = (now: number, env: GpuFallbackEnvironment = environment) => { + const decision = evaluateGpuCrashHistory(userDataPath, env, { + now, + windowMs: WINDOW, + threshold: THRESHOLD + }) + persistGpuCrashTimes(userDataPath, env, decision.crashTimes) + return decision + } + beforeEach(() => { + userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-crash-history-test-')) + }) + + afterEach(() => { + rmSync(userDataPath, { recursive: true, force: true }) + }) + + it('accumulates across separate calls and fires on the third', () => { + // The bug this exists for: three launches, one GPU death each, ~20s apart. + expect(record(NOW).crossesThreshold).toBe(false) + expect(record(NOW + 20_000).crossesThreshold).toBe(false) + const third = record(NOW + 41_000) + expect(third.crossesThreshold).toBe(true) + expect(third.crashesInWindow).toBe(3) + }) + + it('evaluates without writing, so the fsync never precedes the marker write', () => { + // Why load-bearing: the marker write races a kill that can land ~7ms after the + // GPU failure, so the firing crash skips this write entirely. + record(NOW) + record(NOW + 1_000) + const before = readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8') + + const decision = evaluateGpuCrashHistory(userDataPath, environment, { + now: NOW + 2_000, + windowMs: WINDOW, + threshold: THRESHOLD + }) + + expect(decision.crossesThreshold).toBe(true) + expect(readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8')).toBe(before) + }) + + it('is truncated to nothing by persisting an inert decision', () => { + // Why asserted rather than commented: an excluded reason (`launch-failed`) makes + // handleGpuChildCrash evaluate to inertGpuCrashHistoryDecision(), whose crashTimes + // is []. Persisting that rewrites the file empty, so a single excluded GPU death + // erases every real crash counted so far and the fallback never reaches three. + // The persist must therefore stay gated on the reason counting durably. + record(NOW) + record(NOW + 1_000) + expect(readGpuCrashHistory(userDataPath, environment)).toHaveLength(2) + + persistGpuCrashTimes(userDataPath, environment, inertGpuCrashHistoryDecision().crashTimes) + + expect(readGpuCrashHistory(userDataPath, environment)).toEqual([]) + }) + + it('round-trips the persisted times', () => { + record(NOW) + record(NOW + 1_000) + expect(readGpuCrashHistory(userDataPath, environment)).toEqual([NOW, NOW + 1_000]) + expect(gpuCrashHistoryFileExists(userDataPath)).toBe(true) + }) + + it('leaves no temp file behind', () => { + record(NOW) + record(NOW + 1) + expect(readdirSync(userDataPath).filter((name) => name.endsWith('.tmp'))).toEqual([]) + }) + + it('discards history from a different app build', () => { + record(NOW) + record(NOW + 1_000) + + const upgraded = { ...environment, appVersion: '1.4.164' } + expect(readGpuCrashHistory(userDataPath, upgraded)).toEqual([]) + // A new build gets one clean slate, the same policy the marker uses. + expect(record(NOW + 2_000, upgraded).crashesInWindow).toBe(1) + }) + + it('discards history from a different Electron build', () => { + record(NOW) + expect( + readGpuCrashHistory(userDataPath, { ...environment, electronVersion: '43.0.0' }) + ).toEqual([]) + }) + + it('discards a corrupt or wrong-version file instead of trusting it', () => { + writeFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), '{ not json') + expect(readGpuCrashHistory(userDataPath, environment)).toEqual([]) + + writeFileSync( + join(userDataPath, GPU_CRASH_HISTORY_FILE), + JSON.stringify({ schemeVersion: 999, crashTimes: [NOW, NOW, NOW], ...environment }) + ) + expect(readGpuCrashHistory(userDataPath, environment)).toEqual([]) + expect(record(NOW).crossesThreshold).toBe(false) + }) + + it('ignores a crashTimes value that is not an array', () => { + writeFileSync( + join(userDataPath, GPU_CRASH_HISTORY_FILE), + JSON.stringify({ schemeVersion: 1, crashTimes: 'lots', ...environment }) + ) + expect(readGpuCrashHistory(userDataPath, environment)).toEqual([]) + }) + + it('does not latch after the wall clock steps backwards', () => { + // Two real crashes, then the clock rewinds 5 hours (dual-boot RTC skew). The + // stored entries are now future-dated; a rolling window never prunes that end, + // so without the future check they would sit there and the next crash would be + // the "third" — safe graphics latched with no burst and no consent. + record(NOW) + record(NOW + 20_000) + expect(readGpuCrashHistory(userDataPath, environment)).toHaveLength(2) + + const rewound = NOW - 5 * 3_600_000 + const afterJump = record(rewound) + + expect(afterJump.crossesThreshold).toBe(false) + expect(afterJump.crashesInWindow).toBe(1) + expect(readGpuCrashHistory(userDataPath, environment)).toEqual([rewound]) + }) + + it('does not fire on a stale count from hours earlier', () => { + record(NOW) + record(NOW + 1_000) + expect(record(NOW + 4 * 3_600_000).crossesThreshold).toBe(false) + }) + + it('caps the persisted array', () => { + for (let index = 0; index < MAX_GPU_CRASH_HISTORY_ENTRIES + 20; index += 1) { + record(NOW + index) + } + expect(readGpuCrashHistory(userDataPath, environment)).toHaveLength( + MAX_GPU_CRASH_HISTORY_ENTRIES + ) + }) + + it('clears the history so an engaged fallback cannot immediately re-fire', () => { + record(NOW) + record(NOW + 1_000) + clearGpuCrashHistory(userDataPath) + + expect(gpuCrashHistoryFileExists(userDataPath)).toBe(false) + expect(record(NOW + 2_000).crossesThreshold).toBe(false) + }) + + it('clearing a history that is already gone is not an error', () => { + expect(() => clearGpuCrashHistory(userDataPath)).not.toThrow() + }) + + it('hands out a fresh inert decision rather than one shared mutable array', () => { + const first = inertGpuCrashHistoryDecision() + first.crashTimes.push(NOW) + expect(inertGpuCrashHistoryDecision()).toEqual({ + crashTimes: [], + crashesInWindow: 0, + crossesThreshold: false + }) + }) + + it('never throws when the file cannot be written', () => { + // A directory at the target path fails both the durable and the direct write. + mkdirSync(join(userDataPath, GPU_CRASH_HISTORY_FILE)) + + // Why: this runs on the path racing Chromium's kill, so a throw here would + // take out the marker write that is the actual remedy. + expect(() => record(NOW)).not.toThrow() + expect(readdirSync(userDataPath).filter((name) => name.endsWith('.tmp'))).toEqual([]) + }) + + it('sweeps temp files orphaned by a kill between write and rename', async () => { + record(NOW) + const orphan = join(userDataPath, `${GPU_CRASH_HISTORY_FILE}.999999.123.abc.tmp`) + writeFileSync(orphan, '{}') + + await sweepStaleGpuCrashHistoryTempFiles(userDataPath) + + expect(existsSync(orphan)).toBe(false) + // The history itself shares the prefix and must survive the sweep. + expect(readGpuCrashHistory(userDataPath, environment)).toEqual([NOW]) + }) + + for (const platform of ['darwin', 'linux'] as const) { + it(`writes nothing and never fires on ${platform}`, () => { + const foreign = { ...environment, platform } + + for (let index = 0; index < 10; index += 1) { + expect(record(NOW + index * 1_000, foreign)).toEqual({ + crashTimes: [], + crashesInWindow: 0, + crossesThreshold: false + }) + } + + expect(existsSync(join(userDataPath, GPU_CRASH_HISTORY_FILE))).toBe(false) + expect(readdirSync(userDataPath)).toEqual([]) + }) + + it(`discards a history file that followed a profile onto ${platform}`, () => { + record(NOW) + record(NOW + 1_000) + const persisted = readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8') + expect(persisted).toContain('win32') + + expect(readGpuCrashHistory(userDataPath, { ...environment, platform })).toEqual([]) + expect(existsSync(join(userDataPath, GPU_CRASH_HISTORY_FILE))).toBe(false) + }) + } +}) diff --git a/src/main/startup/gpu-crash-history.ts b/src/main/startup/gpu-crash-history.ts new file mode 100644 index 00000000000..7e4bb788a6f --- /dev/null +++ b/src/main/startup/gpu-crash-history.ts @@ -0,0 +1,242 @@ +import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { + durableWriteTempPath, + removeStaleDurableWriteTempFiles, + writeFileDurableSync +} from '../durable-file-write' +import type { GpuFallbackEnvironment } from './gpu-fallback-marker' + +/** + * Cross-launch record of GPU child-process deaths. + * + * Why durable: `GpuCrashFallbackTracker` counts crashes in memory, so its count + * resets to zero on every launch. In the 39 Windows bundles, 53 of the 55 launches + * that recorded a GPU death recorded exactly one (median 1.4s after `app_started`, + * range 0.8-510s), and none recorded three inside the tracker's 30s window. + * + * That per-launch count is a floor rather than a measurement: `process_gone_suppressed` + * coalesces identical repeats on a 30s key, and the same bundles carry 36 + * `gpu_fallback_applied` breadcrumbs, so the in-process tracker does still reach three + * and engage. Counting across launches is what makes the threshold reachable from the + * deaths that are observable. + * + * Not established by that data: that engaging helps. 30 of the 56 attributable deaths + * landed on a launch that had already applied safe graphics, all STATUS_BREAKPOINT — + * but every one predates `--in-process-gpu` (#11295, v1.4.163, tagged 5h after the last + * of them), which is what finally removed the GPU child the old fallback left running. + * Efficacy of the current fallback is unmeasured in both directions. + * + * Why a standalone file (not the Store): same reason as gpu-fallback-marker.ts — + * this is read and written before app.whenReady(), where the Store does not exist. + */ + +export const GPU_CRASH_HISTORY_FILE = 'gpu-crash-history.json' +export const GPU_CRASH_HISTORY_SCHEME_VERSION = 1 + +/** + * Why 5 minutes, derived from the measured distribution rather than rounded up: + * the gap between consecutive crashing launches was 18-24s (median), and the + * first 8 crashes on F0BN5HZL8FJ span 96s (12s/crash). Three crashes therefore + * need ~40-50s. Five minutes tolerates ~2.5 min on each of the two inter-crash + * gaps — 6-8x the observed median, enough for a user who reads the Windows crash + * dialog or tries a driver update before relaunching — and still covers ~12 + * consecutive launches at the observed rate. + * + * Why not 10 minutes: false-positive surface scales linearly with the window, + * and the marker is written before the user consents. Too short only delays the + * remedy to the next tight burst; too long latches safe graphics on a machine + * that was never in the loop. The asymmetry says pick the shorter number. + */ +export const DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS = 300_000 + +/** + * Why 32: an order of magnitude above the threshold of 3, so the cap can never + * suppress a crossing (that would need >32 crashes inside the window, which is + * 10x over threshold and already firing). Eviction is oldest-first — entries + * nearest to ageing out anyway. Future-dated entries are dropped before the cap + * applies, so a bad clock cannot fill it and crowd out real crashes. + */ +export const MAX_GPU_CRASH_HISTORY_ENTRIES = 32 + +// Why tolerated at all: NTP corrections move the wall clock by seconds. Anything +// further ahead is a bad clock, and trusting it would pin a stale crash in the +// window until the clock caught up. +const GPU_CRASH_HISTORY_FUTURE_SKEW_MS = 60_000 + +export type GpuCrashHistoryDecision = { + /** Times to persist, oldest first. */ + crashTimes: number[] + crashesInWindow: number + /** True once the durable count reaches the fallback threshold. */ + crossesThreshold: boolean +} + +/** Epoch-ms crash times still inside the window, dropping anything a clock jump made nonsense. */ +export function pruneGpuCrashTimes( + times: readonly unknown[], + now: number, + windowMs: number +): number[] { + if (!Number.isFinite(now)) { + return [] + } + const oldest = now - windowMs + const newest = now + GPU_CRASH_HISTORY_FUTURE_SKEW_MS + return times + .filter( + (time): time is number => + typeof time === 'number' && + Number.isFinite(time) && + time >= 0 && + time >= oldest && + time <= newest + ) + .sort((a, b) => a - b) +} + +/** Adds `now` to `times` and reports whether the durable threshold is now met. */ +export function appendGpuCrashTime( + times: readonly unknown[], + now: number, + options: { windowMs: number; threshold: number; maxEntries?: number } +): GpuCrashHistoryDecision { + const pruned = pruneGpuCrashTimes([...times, now], now, options.windowMs) + // Keep the newest: the entries dropped here are the ones about to age out anyway. + const crashTimes = pruned.slice(-(options.maxEntries ?? MAX_GPU_CRASH_HISTORY_ENTRIES)) + return { + crashTimes, + crashesInWindow: crashTimes.length, + crossesThreshold: crashTimes.length >= options.threshold + } +} + +/** Why a factory: a shared instance would hand every caller the same mutable array. */ +export function inertGpuCrashHistoryDecision(): GpuCrashHistoryDecision { + return { crashTimes: [], crashesInWindow: 0, crossesThreshold: false } +} + +function historyPath(userDataPath: string): string { + return join(userDataPath, GPU_CRASH_HISTORY_FILE) +} + +/** Why exported: the startup temp sweep is gated on a fallback file having existed. */ +export function gpuCrashHistoryFileExists(userDataPath: string): boolean { + return existsSync(historyPath(userDataPath)) +} + +/** Why no retry: a stale history costs at most one extra loop iteration. */ +function clearHistoryBestEffort(userDataPath: string): void { + try { + rmSync(historyPath(userDataPath), { force: true }) + } catch { + // The next launch revalidates it anyway. + } +} + +export function clearGpuCrashHistory(userDataPath: string): void { + clearHistoryBestEffort(userDataPath) +} + +/** + * Stored crash times for this exact build. A mismatched build or platform discards + * them, the same policy the marker uses: a new build gets one clean slate. + */ +export function readGpuCrashHistory( + userDataPath: string, + environment: GpuFallbackEnvironment +): number[] { + if (environment.platform !== 'win32') { + clearHistoryBestEffort(userDataPath) + return [] + } + let parsed: Record + try { + parsed = JSON.parse(readFileSync(historyPath(userDataPath), 'utf-8')) as Record + } catch { + // Missing or corrupt means no history; a corrupt file is overwritten by the next write. + return [] + } + if ( + parsed.schemeVersion !== GPU_CRASH_HISTORY_SCHEME_VERSION || + parsed.platform !== 'win32' || + parsed.appVersion !== environment.appVersion || + parsed.electronVersion !== environment.electronVersion + ) { + clearHistoryBestEffort(userDataPath) + return [] + } + const stored: unknown[] = Array.isArray(parsed.crashTimes) ? parsed.crashTimes : [] + // Range checks belong to pruneGpuCrashTimes, which needs `now`; this is only the + // shape boundary, so a hand-edited file cannot put a string into the array. + return stored.filter( + (time): time is number => typeof time === 'number' && Number.isFinite(time) && time >= 0 + ) +} + +/** + * Persists `crashTimes`. Never throws: this runs on the crash path racing + * Chromium's kill, and a lost entry only costs one more iteration of the loop. + * + * Why separate from the evaluation: the caller skips this entirely on the crash + * that fires the fallback, so its fsync never lands in front of the marker write. + */ +export function persistGpuCrashTimes( + userDataPath: string, + environment: GpuFallbackEnvironment, + crashTimes: readonly number[] +): void { + if (environment.platform !== 'win32') { + return + } + const target = historyPath(userDataPath) + const payload = JSON.stringify({ + schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION, + crashTimes, + appVersion: environment.appVersion, + electronVersion: environment.electronVersion, + platform: 'win32' + }) + const tmp = durableWriteTempPath(target) + try { + writeFileDurableSync(tmp, target, payload) + return + } catch { + // Windows AV/indexer holds make renameSync EPERM where a direct write still lands. + } + try { + writeFileSync(target, payload) + } catch { + // Nothing left to try; the in-process tracker is still armed for this session. + } + try { + rmSync(tmp, { force: true }) + } catch { + // Swept next launch alongside the marker's orphans. + } +} + +/** + * Reads the stored history, adds a GPU child death at `options.now` (epoch ms), + * and reports whether the durable count has reached the threshold. Writes nothing + * — the caller decides, so a fallback that fires can go straight to the marker. + * Inert off Windows. + */ +export function evaluateGpuCrashHistory( + userDataPath: string, + environment: GpuFallbackEnvironment, + options: { now: number; windowMs: number; threshold: number } +): GpuCrashHistoryDecision { + if (environment.platform !== 'win32') { + return inertGpuCrashHistoryDecision() + } + return appendGpuCrashTime(readGpuCrashHistory(userDataPath, environment), options.now, { + windowMs: options.windowMs, + threshold: options.threshold + }) +} + +/** Why: the kill this history exists to survive can land between write and rename. */ +export async function sweepStaleGpuCrashHistoryTempFiles(userDataPath: string): Promise { + await removeStaleDurableWriteTempFiles(historyPath(userDataPath)) +} diff --git a/src/main/startup/gpu-fallback-marker.test.ts b/src/main/startup/gpu-fallback-marker.test.ts index b4d130285a2..de239c21ec4 100644 --- a/src/main/startup/gpu-fallback-marker.test.ts +++ b/src/main/startup/gpu-fallback-marker.test.ts @@ -44,6 +44,7 @@ vi.mock('../durable-file-write', async (importOriginal) => { import { GPU_FALLBACK_MARKER_FILE, clearGpuFallbackMarker, + gpuFallbackMarkerFileExists, readActiveGpuFallbackMarker, readGpuFallbackMarker, sweepStaleGpuFallbackMarkerTempFiles, @@ -73,20 +74,71 @@ describe('gpu-fallback-marker', () => { }) it('round-trips a written marker', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 123, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 123, crashesInWindow: 3, consented: false }, + environment + ) expect(readGpuFallbackMarker(userDataPath)).toEqual({ - schemeVersion: 2, + schemeVersion: 3, engagedAt: 123, crashesInWindow: 3, + consented: false, appVersion: '1.2.3', electronVersion: '42.3.3', platform: 'win32' }) }) + it('round-trips the consent flag that separates a latch from a chosen restart', () => { + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 123, crashesInWindow: 3, consented: true }, + environment + ) + expect(readGpuFallbackMarker(userDataPath)?.consented).toBe(true) + }) + + it('rejects a scheme-2 marker that predates the consent flag', () => { + // Why: without `consented` there is no way to tell a user-confirmed latch from + // one the crash imposed, so the bump must invalidate the old shape outright. + writeFileSync( + join(userDataPath, GPU_FALLBACK_MARKER_FILE), + JSON.stringify({ + schemeVersion: 2, + engagedAt: 1, + crashesInWindow: 3, + ...environment + }) + ) + expect(readGpuFallbackMarker(userDataPath)).toBeNull() + }) + + it('rejects a marker whose consent flag is not a boolean', () => { + writeFileSync( + join(userDataPath, GPU_FALLBACK_MARKER_FILE), + JSON.stringify({ + schemeVersion: 3, + engagedAt: 1, + crashesInWindow: 3, + consented: 'yes', + ...environment + }) + ) + expect(readGpuFallbackMarker(userDataPath)).toBeNull() + }) + it('overwrites an existing marker and leaves no temp file behind', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment) - writeGpuFallbackMarker(userDataPath, { engagedAt: 2, crashesInWindow: 5 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 3, consented: false }, + environment + ) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 2, crashesInWindow: 5, consented: false }, + environment + ) expect(readGpuFallbackMarker(userDataPath)?.engagedAt).toBe(2) // Why: the durable write renames through a temp file; an orphan would accumulate @@ -99,7 +151,11 @@ describe('gpu-fallback-marker', () => { // lands. This path must never end up worse than a plain writeFileSync. durableWrite.failWith = Object.assign(new Error('EPERM: rename'), { code: 'EPERM' }) - writeGpuFallbackMarker(userDataPath, { engagedAt: 7, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 7, crashesInWindow: 3, consented: false }, + environment + ) // Why assert the call: an ineffective mock would leave this green while // covering nothing but the happy path. @@ -116,7 +172,11 @@ describe('gpu-fallback-marker', () => { // Why the durable error, not the fallback's: it names the real cause. expect(() => - writeGpuFallbackMarker(userDataPath, { engagedAt: 9, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 9, crashesInWindow: 3, consented: false }, + environment + ) ).toThrow(durableError) }) @@ -126,7 +186,11 @@ describe('gpu-fallback-marker', () => { }) it('keeps an active marker for repeated launches on the same build', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 4, consented: false }, + environment + ) expect(existsSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE))).toBe(true) const firstRead = readActiveGpuFallbackMarker(userDataPath, environment) @@ -138,7 +202,11 @@ describe('gpu-fallback-marker', () => { }) it('clears an active marker when the app build changes', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 4, consented: false }, + environment + ) expect( readActiveGpuFallbackMarker(userDataPath, { @@ -150,7 +218,11 @@ describe('gpu-fallback-marker', () => { }) it('clears an active marker outside Windows', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 4, consented: false }, + environment + ) expect( readActiveGpuFallbackMarker(userDataPath, { @@ -165,7 +237,11 @@ describe('gpu-fallback-marker', () => { // carries the macOS disable-skia-graphite fix. A marker that survived on darwin would silently // strip the fix from the Macs it targets, so pin the platform gate for darwin specifically. it('clears an active marker on macOS so the Graphite fix is never skipped', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 4, consented: false }, + environment + ) expect( readActiveGpuFallbackMarker(userDataPath, { @@ -191,8 +267,20 @@ describe('gpu-fallback-marker', () => { expect(existsSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE))).toBe(false) }) + it('reports a marker file that exists but does not parse, so the sweep still runs', () => { + // Why: the startup sweep is gated on this, and gating it on a *readable* marker + // would skip the reclaim in exactly the torn-write case it exists for. + expect(gpuFallbackMarkerFileExists(userDataPath)).toBe(false) + writeFileSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE), '{ not json') + expect(gpuFallbackMarkerFileExists(userDataPath)).toBe(true) + }) + it('sweeps temp files orphaned by a kill between write and rename', async () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 3, consented: false }, + environment + ) // A foreign pid: the sweeper deliberately skips this process's own in-flight temps. const orphan = join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.999999.123.abc.tmp`) writeFileSync(orphan, '{}') @@ -205,7 +293,11 @@ describe('gpu-fallback-marker', () => { }) it('retries a marker delete that is briefly refused', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 3, consented: false }, + environment + ) rmControl.calls = 0 rmControl.failuresLeft = 2 @@ -216,7 +308,11 @@ describe('gpu-fallback-marker', () => { }) it('reports a marker delete that never succeeds, after backing off', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 3, consented: false }, + environment + ) rmControl.failuresLeft = Number.MAX_SAFE_INTEGER const startedAt = Date.now() @@ -231,7 +327,11 @@ describe('gpu-fallback-marker', () => { }) it('never blocks startup revalidation on an undeletable marker', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 3, consented: false }, + environment + ) rmControl.failuresLeft = Number.MAX_SAFE_INTEGER const startedAt = Date.now() @@ -261,7 +361,11 @@ describe('gpu-fallback-marker', () => { rmControl.failuresLeft = Number.MAX_SAFE_INTEGER const startedAt = Date.now() - writeGpuFallbackMarker(userDataPath, { engagedAt: 5, crashesInWindow: 3 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 5, crashesInWindow: 3, consented: false }, + environment + ) const elapsed = Date.now() - startedAt expect(readGpuFallbackMarker(userDataPath)?.engagedAt).toBe(5) @@ -271,7 +375,11 @@ describe('gpu-fallback-marker', () => { }) it('can explicitly clear the marker', () => { - writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment) + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 4, consented: false }, + environment + ) clearGpuFallbackMarker(userDataPath) expect(readGpuFallbackMarker(userDataPath)).toBeNull() }) diff --git a/src/main/startup/gpu-fallback-marker.ts b/src/main/startup/gpu-fallback-marker.ts index c901f5113c4..11da0e2680d 100644 --- a/src/main/startup/gpu-fallback-marker.ts +++ b/src/main/startup/gpu-fallback-marker.ts @@ -16,7 +16,8 @@ import { */ export const GPU_FALLBACK_MARKER_FILE = 'gpu-fallback.json' -export const GPU_FALLBACK_SCHEME_VERSION = 2 +// Bumped to 3 for `consented`, which self-invalidates markers written without it. +export const GPU_FALLBACK_SCHEME_VERSION = 3 export type GpuFallbackEnvironment = { appVersion: string @@ -30,6 +31,8 @@ export type GpuFallbackMarker = { schemeVersion: number engagedAt: number crashesInWindow: number + /** False while the marker is only a pre-prompt latch; true once the user chose to restart. */ + consented: boolean appVersion: string electronVersion: string platform: 'win32' @@ -39,6 +42,11 @@ function markerPath(userDataPath: string): string { return join(userDataPath, GPU_FALLBACK_MARKER_FILE) } +/** Why exported: the startup temp sweep is gated on a fallback file having existed. */ +export function gpuFallbackMarkerFileExists(userDataPath: string): boolean { + return existsSync(markerPath(userDataPath)) +} + export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker | null { try { const parsed = JSON.parse(readFileSync(markerPath(userDataPath), 'utf-8')) as Partial< @@ -52,6 +60,7 @@ export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker | !Number.isFinite(parsed.engagedAt) || typeof parsed.crashesInWindow !== 'number' || !Number.isFinite(parsed.crashesInWindow) || + typeof parsed.consented !== 'boolean' || typeof parsed.appVersion !== 'string' || typeof parsed.electronVersion !== 'string' || parsed.platform !== 'win32' @@ -62,6 +71,7 @@ export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker | schemeVersion: GPU_FALLBACK_SCHEME_VERSION, engagedAt: parsed.engagedAt, crashesInWindow: parsed.crashesInWindow, + consented: parsed.consented, appVersion: parsed.appVersion, electronVersion: parsed.electronVersion, platform: parsed.platform @@ -74,13 +84,14 @@ export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker | export function writeGpuFallbackMarker( userDataPath: string, - info: { engagedAt: number; crashesInWindow: number }, + info: { engagedAt: number; crashesInWindow: number; consented: boolean }, environment: WindowsGpuFallbackEnvironment ): void { const marker: GpuFallbackMarker = { schemeVersion: GPU_FALLBACK_SCHEME_VERSION, engagedAt: info.engagedAt, crashesInWindow: info.crashesInWindow, + consented: info.consented, appVersion: environment.appVersion, electronVersion: environment.electronVersion, platform: 'win32' diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index a648e68d409..9f5a239b8b0 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -67,6 +67,7 @@ "exploreOrca": "Explore Orca", "gettingStarted": "Getting Started with Orca", "reportCrash": "Report Crash...", + "turnOffSafeGraphics": "Turn Off Safe Graphics Mode", "file": "File", "exit": "Exit", "edit": "Edit",