mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
fix(windows): count GPU crashes across launches so the fallback can engage
The safe-graphics fallback engages on the 3rd GPU child crash inside a 30s window, but the crash shape it exists for kills the app ~0.8s after launch. The in-memory tracker therefore resets to zero every launch and the same-session threshold is structurally unreachable: across 39 Windows bundles, 53 of the 55 launches that recorded a GPU death recorded exactly one, and none reached three. Add gpu-crash-history.json, a build-scoped record of GPU death timestamps read and written alongside the existing marker, so the count survives the launch that dies. Three deaths inside 5 minutes now engage the fallback whether they land in one session or across a dozen. Only `crashed` and `abnormal-exit` accumulate durably. `launch-failed` clusters across launches for recoverable reasons — a driver update in flight, an RDP transition, a monitor hotplug — and durable counting would latch build-sticky safe graphics on a condition that clears itself. The in-process tracker still counts it, since three launch failures inside one 30s session is a broken GPU either way. The marker is written before the user consents, so add a Help-menu "Turn Off Safe Graphics Mode" escape hatch that clears both the marker and the count; clearing only one leaves the user latched on the launch after.
This commit is contained in:
@@ -5,6 +5,8 @@ import { recordDurableCrashBreadcrumb } from './crash-reporting/durable-crash-br
|
||||
export type AppRelaunchReason =
|
||||
| 'admin-restart'
|
||||
| 'gpu-fallback'
|
||||
/** User turned safe graphics back off from the Help menu. */
|
||||
| 'gpu-fallback-opt-out'
|
||||
| 'profile-switch'
|
||||
| 'profile-transfer'
|
||||
| 'renderer-request'
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
|
||||
DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
|
||||
GpuCrashFallbackTracker,
|
||||
countsTowardDurableGpuCrashHistory,
|
||||
isGpuChildProcessType,
|
||||
isGpuFallbackCrashCandidate
|
||||
} from './gpu-crash-fallback-decision'
|
||||
@@ -172,3 +173,26 @@ describe('isGpuFallbackCrashCandidate', () => {
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('countsTowardDurableGpuCrashHistory', () => {
|
||||
it('counts the reasons the crash loop actually produces', () => {
|
||||
// All 57 distinct GPU child deaths across the 39 cluster-E bundles are
|
||||
// `crashed` (53 STATUS_BREAKPOINT) or `killed` — none are `launch-failed`.
|
||||
expect(countsTowardDurableGpuCrashHistory('crashed')).toBe(true)
|
||||
expect(countsTowardDurableGpuCrashHistory('abnormal-exit')).toBe(true)
|
||||
})
|
||||
|
||||
it('refuses launch-failed, which clusters across launches for recoverable reasons', () => {
|
||||
// A driver update, an RDP session transition, or a monitor hotplug can fail the
|
||||
// GPU launch on several consecutive launches. Counting those durably would latch
|
||||
// build-sticky safe graphics on a condition that clears itself; the in-process
|
||||
// tracker still catches 3 inside one 30s session.
|
||||
expect(countsTowardDurableGpuCrashHistory('launch-failed')).toBe(false)
|
||||
})
|
||||
|
||||
it('ignores reasons that are not fallback candidates at all', () => {
|
||||
for (const reason of ['killed', 'clean-exit', 'oom', 'integrity-failure', '']) {
|
||||
expect([reason, countsTowardDurableGpuCrashHistory(reason)]).toEqual([reason, false])
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -7,6 +7,26 @@ export type GpuCrashFallbackOptions = {
|
||||
|
||||
const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch-failed'])
|
||||
|
||||
/**
|
||||
* Reasons that may accumulate in the *cross-launch* history.
|
||||
*
|
||||
* Why `launch-failed` is excluded here but kept in-process: a GPU that fails to
|
||||
* launch clusters across launches for recoverable reasons — a driver update in
|
||||
* flight, an RDP session transition, a monitor hotplug — and durable counting
|
||||
* would latch build-sticky safe graphics on a condition that clears itself. Three
|
||||
* launch failures inside one 30s session is still a broken GPU, so the in-process
|
||||
* tracker keeps counting them.
|
||||
*
|
||||
* Costs nothing on the observed data: the 39 cluster-E bundles hold 57 distinct
|
||||
* GPU child deaths (deduped by pid and timestamp) — 56 `crashed`, 53 of those
|
||||
* STATUS_BREAKPOINT, one `killed`, and zero `launch-failed`.
|
||||
*/
|
||||
const DURABLE_GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed'])
|
||||
|
||||
export function countsTowardDurableGpuCrashHistory(reason: string): boolean {
|
||||
return DURABLE_GPU_FALLBACK_CRASH_REASONS.has(reason)
|
||||
}
|
||||
|
||||
// Why: on old/flaky GPU drivers the GPU child process crashes (STATUS_BREAKPOINT
|
||||
// / ANGLE-D3D init failure) repeatedly - Windows clusters F0BDNADU79Q and
|
||||
// F0BDNRZ5MDG. GPU child deaths are intentionally suppressed as recoverable
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { GpuFallbackEngagementOutcome } from './gpu-fallback-engagement'
|
||||
import { resolveGpuFallbackEngagement } from './gpu-fallback-engagement-resolution'
|
||||
|
||||
const USER_DATA = '/tmp/orca-userdata'
|
||||
|
||||
describe('resolveGpuFallbackEngagement', () => {
|
||||
// Why exhaustive: every field here is a one-token change away from defeating the
|
||||
// feature silently, and this table is the only thing that can see it.
|
||||
const expected: Record<
|
||||
GpuFallbackEngagementOutcome,
|
||||
{
|
||||
provisionalMarkerPath: string | null
|
||||
rePersistConsentedMarker: boolean
|
||||
clearDurableHistory: boolean
|
||||
relaunch: boolean
|
||||
}
|
||||
> = {
|
||||
restart: {
|
||||
provisionalMarkerPath: null,
|
||||
rePersistConsentedMarker: true,
|
||||
clearDurableHistory: true,
|
||||
relaunch: true
|
||||
},
|
||||
'confirmed-quitting': {
|
||||
provisionalMarkerPath: null,
|
||||
rePersistConsentedMarker: true,
|
||||
clearDurableHistory: true,
|
||||
relaunch: false
|
||||
},
|
||||
deferred: {
|
||||
provisionalMarkerPath: null,
|
||||
rePersistConsentedMarker: false,
|
||||
clearDurableHistory: true,
|
||||
relaunch: false
|
||||
},
|
||||
'deferred-uncleared': {
|
||||
provisionalMarkerPath: USER_DATA,
|
||||
rePersistConsentedMarker: false,
|
||||
clearDurableHistory: true,
|
||||
relaunch: false
|
||||
},
|
||||
latched: {
|
||||
provisionalMarkerPath: USER_DATA,
|
||||
rePersistConsentedMarker: false,
|
||||
clearDurableHistory: true,
|
||||
relaunch: false
|
||||
},
|
||||
'marker-failed': {
|
||||
provisionalMarkerPath: null,
|
||||
rePersistConsentedMarker: false,
|
||||
clearDurableHistory: false,
|
||||
relaunch: false
|
||||
}
|
||||
}
|
||||
|
||||
for (const [outcome, resolution] of Object.entries(expected)) {
|
||||
it(`resolves ${outcome}`, () => {
|
||||
expect(
|
||||
resolveGpuFallbackEngagement(outcome as GpuFallbackEngagementOutcome, USER_DATA)
|
||||
).toEqual(resolution)
|
||||
})
|
||||
}
|
||||
|
||||
it('stands the shutdown drop down for every settled outcome', () => {
|
||||
// Why its own assertion: keeping the provisional path on a settled outcome
|
||||
// deletes the consented marker on the way out, so the relaunch comes back on
|
||||
// the same broken GPU — the exact mutation the source-text tests could not see.
|
||||
for (const outcome of ['restart', 'confirmed-quitting', 'deferred', 'marker-failed'] as const) {
|
||||
expect([
|
||||
outcome,
|
||||
resolveGpuFallbackEngagement(outcome, USER_DATA).provisionalMarkerPath
|
||||
]).toEqual([outcome, null])
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps the shutdown drop armed only while the marker is unconfirmed', () => {
|
||||
for (const outcome of ['latched', 'deferred-uncleared'] as const) {
|
||||
expect([
|
||||
outcome,
|
||||
resolveGpuFallbackEngagement(outcome, USER_DATA).provisionalMarkerPath
|
||||
]).toEqual([outcome, USER_DATA])
|
||||
}
|
||||
})
|
||||
|
||||
it('records consent on both paths where the user actually answered "restart"', () => {
|
||||
// 'restart' latched pre-prompt with consented:false, so without the re-persist
|
||||
// the primary consent path would land on disk indistinguishable from a latch.
|
||||
expect(resolveGpuFallbackEngagement('restart', USER_DATA).rePersistConsentedMarker).toBe(true)
|
||||
expect(
|
||||
resolveGpuFallbackEngagement('confirmed-quitting', USER_DATA).rePersistConsentedMarker
|
||||
).toBe(true)
|
||||
expect(resolveGpuFallbackEngagement('latched', USER_DATA).rePersistConsentedMarker).toBe(false)
|
||||
expect(resolveGpuFallbackEngagement('deferred', USER_DATA).rePersistConsentedMarker).toBe(false)
|
||||
})
|
||||
|
||||
it('relaunches only on an answered restart that no quit already claimed', () => {
|
||||
expect(resolveGpuFallbackEngagement('restart', USER_DATA).relaunch).toBe(true)
|
||||
for (const outcome of [
|
||||
'confirmed-quitting',
|
||||
'deferred',
|
||||
'deferred-uncleared',
|
||||
'latched',
|
||||
'marker-failed'
|
||||
] as const) {
|
||||
expect([outcome, resolveGpuFallbackEngagement(outcome, USER_DATA).relaunch]).toEqual([
|
||||
outcome,
|
||||
false
|
||||
])
|
||||
}
|
||||
})
|
||||
|
||||
it('drops the cross-launch count once the prompt has been answered', () => {
|
||||
// Why: after 'deferred' the app keeps running with gpuFallbackActiveThisLaunch
|
||||
// false, so handleGpuChildCrash still runs — but the stored 2 entries would make
|
||||
// the next GPU crash inside the 5-minute window the "third" and re-latch on a
|
||||
// prompt the user just declined.
|
||||
for (const outcome of [
|
||||
'restart',
|
||||
'confirmed-quitting',
|
||||
'deferred',
|
||||
'deferred-uncleared',
|
||||
'latched'
|
||||
] as const) {
|
||||
expect([
|
||||
outcome,
|
||||
resolveGpuFallbackEngagement(outcome, USER_DATA).clearDurableHistory
|
||||
]).toEqual([outcome, true])
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps the count armed when the marker write is what failed', () => {
|
||||
// 'marker-failed' never prompted, so the crashes are still unaddressed. Clearing
|
||||
// would demand three fresh crashes before retrying a transient EPERM.
|
||||
expect(resolveGpuFallbackEngagement('marker-failed', USER_DATA).clearDurableHistory).toBe(false)
|
||||
})
|
||||
|
||||
it('echoes back the caller path so a split userData cannot go unnoticed', () => {
|
||||
expect(resolveGpuFallbackEngagement('latched', '/other/path').provisionalMarkerPath).toBe(
|
||||
'/other/path'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
import {
|
||||
shouldKeepProvisionalGpuFallbackMarker,
|
||||
type GpuFallbackEngagementOutcome
|
||||
} from './gpu-fallback-engagement'
|
||||
|
||||
/**
|
||||
* Bookkeeping that must follow an `engageGpuFallback` outcome.
|
||||
*
|
||||
* Why its own module: this lived inline in index.ts, where the only coverage was
|
||||
* source-text matching. A mutation that kept the provisional path on *every*
|
||||
* outcome — which deletes the consented marker on the way out and brings the app
|
||||
* back on the same broken GPU — passed the entire suite.
|
||||
*/
|
||||
export type GpuFallbackEngagementResolution = {
|
||||
/** userData path the shutdown drop must keep retrying, or null to stand down. */
|
||||
provisionalMarkerPath: string | null
|
||||
/** Rewrite the marker with consent recorded, because the user answered "restart". */
|
||||
rePersistConsentedMarker: boolean
|
||||
/** Drop the cross-launch crash count, because this engagement settled it. */
|
||||
clearDurableHistory: boolean
|
||||
/** Relaunch into safe graphics now. */
|
||||
relaunch: boolean
|
||||
}
|
||||
|
||||
export function resolveGpuFallbackEngagement(
|
||||
outcome: GpuFallbackEngagementOutcome,
|
||||
userDataPath: string
|
||||
): GpuFallbackEngagementResolution {
|
||||
return {
|
||||
provisionalMarkerPath: shouldKeepProvisionalGpuFallbackMarker(outcome) ? userDataPath : null,
|
||||
// Why both outcomes: the pre-prompt persist recorded consented:false, and a
|
||||
// session-end drop can take the marker while the prompt is still open.
|
||||
rePersistConsentedMarker: outcome === 'restart' || outcome === 'confirmed-quitting',
|
||||
// Why every outcome but 'marker-failed': a declined prompt must not re-fire on
|
||||
// the very next GPU crash, and a consented one is about to relaunch. Only
|
||||
// 'marker-failed' never prompted at all, so its count stays armed to retry a
|
||||
// write that failed transiently rather than demanding three fresh crashes.
|
||||
clearDurableHistory: outcome !== 'marker-failed',
|
||||
relaunch: outcome === 'restart'
|
||||
}
|
||||
}
|
||||
+126
-29
@@ -130,6 +130,7 @@ import { enableRendererHeapHeadroom } from './startup/renderer-heap-headroom'
|
||||
import { ensureVirtualDisplayForHeadlessServe } from './startup/ensure-virtual-display'
|
||||
import {
|
||||
clearGpuFallbackMarker,
|
||||
gpuFallbackMarkerFileExists,
|
||||
readActiveGpuFallbackMarker,
|
||||
sweepStaleGpuFallbackMarkerTempFiles,
|
||||
writeGpuFallbackMarker,
|
||||
@@ -137,14 +138,22 @@ import {
|
||||
type WindowsGpuFallbackEnvironment
|
||||
} from './startup/gpu-fallback-marker'
|
||||
import {
|
||||
engageGpuFallback,
|
||||
shouldKeepProvisionalGpuFallbackMarker
|
||||
} from './crash-reporting/gpu-fallback-engagement'
|
||||
DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS,
|
||||
clearGpuCrashHistory,
|
||||
evaluateGpuCrashHistory,
|
||||
gpuCrashHistoryFileExists,
|
||||
inertGpuCrashHistoryDecision,
|
||||
persistGpuCrashTimes,
|
||||
sweepStaleGpuCrashHistoryTempFiles
|
||||
} from './startup/gpu-crash-history'
|
||||
import { engageGpuFallback } from './crash-reporting/gpu-fallback-engagement'
|
||||
import { resolveGpuFallbackEngagement } from './crash-reporting/gpu-fallback-engagement-resolution'
|
||||
import { applyGpuFallbackCommandLineSwitches } from './startup/gpu-fallback-switches'
|
||||
import {
|
||||
DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
|
||||
DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
|
||||
GpuCrashFallbackTracker,
|
||||
countsTowardDurableGpuCrashHistory,
|
||||
isGpuFallbackCrashCandidate
|
||||
} from './crash-reporting/gpu-crash-fallback-decision'
|
||||
import { promptForGpuFallbackRestart } from './crash-reporting/gpu-fallback-restart-prompt'
|
||||
@@ -369,6 +378,9 @@ const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({
|
||||
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD
|
||||
})
|
||||
let gpuFallbackActiveThisLaunch = false
|
||||
// Why a separate latch: the durable history and the in-process tracker can each
|
||||
// fire, so without this a later same-session crash opens a second restart prompt.
|
||||
let gpuFallbackEngagementStarted = false
|
||||
// Why: the safe-graphics marker is written before the user answers, so it must be
|
||||
// dropped again if the app reaches an orderly shutdown instead of a GPU kill.
|
||||
let provisionalGpuFallbackUserDataPath: string | null = null
|
||||
@@ -1565,51 +1577,100 @@ function maybeApplyGpuFallbackForThisLaunch(): void {
|
||||
// Why canonical: handleGpuChildCrash writes through the same helper, and a late
|
||||
// app.setName/setPath would silently split the read and write paths.
|
||||
const userDataPath = getCanonicalUserDataPath()
|
||||
// A kill between the marker's write and rename orphans a temp file next to it.
|
||||
void sweepStaleGpuFallbackMarkerTempFiles(userDataPath).catch(() => undefined)
|
||||
// A kill between a write and its rename orphans a temp file next to the target.
|
||||
// Why gated: the sweep readdirs all of userData — Cache, Code Cache, GPUCache,
|
||||
// Local Storage — on a path that runs before whenReady on every Windows launch,
|
||||
// and a temp can only be orphaned next to a file the fallback already wrote.
|
||||
const sweepForOrphans =
|
||||
gpuFallbackMarkerFileExists(userDataPath) || gpuCrashHistoryFileExists(userDataPath)
|
||||
const marker = readActiveGpuFallbackMarker(userDataPath, getGpuFallbackEnvironment())
|
||||
if (sweepForOrphans) {
|
||||
void sweepStaleGpuFallbackMarkerTempFiles(userDataPath).catch(() => undefined)
|
||||
void sweepStaleGpuCrashHistoryTempFiles(userDataPath).catch(() => undefined)
|
||||
}
|
||||
if (!marker) {
|
||||
return
|
||||
}
|
||||
app.disableHardwareAcceleration()
|
||||
const appliedSwitches = applyGpuFallbackCommandLineSwitches(app.commandLine, process.platform)
|
||||
gpuFallbackActiveThisLaunch = true
|
||||
// Why: the user is already in safe graphics, so pre-fallback crash times must not
|
||||
// re-fire the prompt on the first unrelated GPU hiccup of this launch.
|
||||
clearGpuCrashHistory(userDataPath)
|
||||
// Why: with no GPU child left, child-process-gone can't report a GPU fault, so
|
||||
// name the applied switches in the trail any later crash report carries.
|
||||
recordCrashBreadcrumb('gpu_fallback_applied', {
|
||||
crashesInWindow: marker.crashesInWindow,
|
||||
consented: marker.consented,
|
||||
switches: appliedSwitches.join(',')
|
||||
})
|
||||
}
|
||||
|
||||
// Why swallow: the marker is already on disk, so a tracer failure here must not
|
||||
// surface as 'marker-failed' and cancel the restart the write just earned.
|
||||
function recordGpuFallbackMarkerPersisted(
|
||||
consented: boolean,
|
||||
data: Record<string, string | number | boolean | null>
|
||||
): void {
|
||||
try {
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_marker_persisted', { ...data, consented })
|
||||
} catch {
|
||||
// Diagnostics only; the marker is what the next launch reads.
|
||||
}
|
||||
}
|
||||
|
||||
// Why: a burst of GPU child crashes means HW acceleration is unusable — persist a build-scoped marker and offer software rendering.
|
||||
async function handleGpuChildCrash(reason: string, exitCode: number | null): Promise<void> {
|
||||
// Software rendering already active or shutting down: nothing more to do.
|
||||
if (gpuFallbackActiveThisLaunch || isQuitting || isServeMode) {
|
||||
// Software rendering already active, engagement under way, or shutting down.
|
||||
if (gpuFallbackActiveThisLaunch || gpuFallbackEngagementStarted || isQuitting || isServeMode) {
|
||||
return
|
||||
}
|
||||
const result = gpuCrashFallbackTracker.recordGpuCrash(performance.now())
|
||||
if (!result.shouldEngageFallback) {
|
||||
return
|
||||
}
|
||||
recordCrashBreadcrumb('gpu_fallback_engaged', {
|
||||
reason,
|
||||
exitCode,
|
||||
crashesInWindow: result.crashesInWindow
|
||||
})
|
||||
const engagedAt = Date.now()
|
||||
// Why first: everything below this point must stay inert off Windows.
|
||||
const environment = getWindowsGpuFallbackEnvironment()
|
||||
if (!environment) {
|
||||
return
|
||||
}
|
||||
const fallbackData = {
|
||||
processReason: reason,
|
||||
exitCode,
|
||||
crashesInWindow: result.crashesInWindow
|
||||
}
|
||||
// Why canonical: the reader above resolves the same way, and a late app.setName
|
||||
// would otherwise move the write out from under it.
|
||||
const userDataPath = getCanonicalUserDataPath()
|
||||
const inProcess = gpuCrashFallbackTracker.recordGpuCrash(performance.now())
|
||||
// Why durable too: this failure kills the launch ~0.8s in, so the in-memory count
|
||||
// resets at 1 and the same-session threshold is structurally unreachable.
|
||||
// Why reason-gated: `launch-failed` clusters across launches for recoverable
|
||||
// reasons (driver update, RDP transition, monitor hotplug) — see the reason set.
|
||||
const countsDurably = countsTowardDurableGpuCrashHistory(reason)
|
||||
const durable = countsDurably
|
||||
? evaluateGpuCrashHistory(userDataPath, environment, {
|
||||
now: Date.now(),
|
||||
windowMs: DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS,
|
||||
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD
|
||||
})
|
||||
: inertGpuCrashHistoryDecision()
|
||||
if (!inProcess.shouldEngageFallback && !durable.crossesThreshold) {
|
||||
// Why only here: on the crash that fires, this fsync would land in front of the
|
||||
// marker write, which is racing a kill that can arrive ~7ms after the failure.
|
||||
// Why gated: an inert decision carries no times, so writing it would erase the
|
||||
// real ones an excluded reason is only supposed to be ignored by.
|
||||
if (countsDurably) {
|
||||
persistGpuCrashTimes(userDataPath, environment, durable.crashTimes)
|
||||
}
|
||||
return
|
||||
}
|
||||
gpuFallbackEngagementStarted = true
|
||||
const crashesInWindow = Math.max(inProcess.crashesInWindow, durable.crashesInWindow)
|
||||
recordCrashBreadcrumb('gpu_fallback_engaged', {
|
||||
reason,
|
||||
exitCode,
|
||||
crashesInWindow,
|
||||
crashesAcrossLaunches: durable.crashesInWindow,
|
||||
trigger: inProcess.shouldEngageFallback ? 'in-process' : 'durable'
|
||||
})
|
||||
const engagedAt = Date.now()
|
||||
const fallbackData = {
|
||||
processReason: reason,
|
||||
exitCode,
|
||||
crashesInWindow
|
||||
}
|
||||
const window = mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined
|
||||
// Why: Chromium fatally CHECKs the browser process on the 6th GPU failure, so
|
||||
// the marker must land before the prompt or an unattended crash never latches.
|
||||
@@ -1617,10 +1678,13 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
|
||||
persistMarker: () => {
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt, crashesInWindow: result.crashesInWindow },
|
||||
{ engagedAt, crashesInWindow, consented: false },
|
||||
environment
|
||||
)
|
||||
provisionalGpuFallbackUserDataPath = userDataPath
|
||||
// Why after the write: a kill lands here more often than anywhere else, and
|
||||
// this is the only record separating a provisional latch from a consented one.
|
||||
recordGpuFallbackMarkerPersisted(false, fallbackData)
|
||||
},
|
||||
clearMarker: () => clearGpuFallbackMarker(userDataPath),
|
||||
prompt: () => promptForGpuFallbackRestart(window),
|
||||
@@ -1639,23 +1703,29 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
|
||||
onRestartDeferred: () =>
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData)
|
||||
})
|
||||
if (!shouldKeepProvisionalGpuFallbackMarker(outcome)) {
|
||||
provisionalGpuFallbackUserDataPath = null
|
||||
// Why delegated: this mapping is unit-tested; deriving it inline here left the
|
||||
// whole feature defeatable by a one-token change no test could see.
|
||||
const resolution = resolveGpuFallbackEngagement(outcome, userDataPath)
|
||||
provisionalGpuFallbackUserDataPath = resolution.provisionalMarkerPath
|
||||
// Why here and not at fire time: a kill during the prompt must leave the count
|
||||
// armed to fire again, while a decline must not re-prompt on the very next crash.
|
||||
if (resolution.clearDurableHistory) {
|
||||
clearGpuCrashHistory(userDataPath)
|
||||
}
|
||||
if (outcome === 'confirmed-quitting') {
|
||||
// Why re-persist: a session-end drop can land while the prompt is still open.
|
||||
if (resolution.rePersistConsentedMarker) {
|
||||
try {
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt, crashesInWindow: result.crashesInWindow },
|
||||
{ engagedAt, crashesInWindow, consented: true },
|
||||
environment
|
||||
)
|
||||
recordGpuFallbackMarkerPersisted(true, fallbackData)
|
||||
} catch (error) {
|
||||
console.warn('[gpu-fallback] failed to re-persist marker after consent:', error)
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_marker_failed', fallbackData)
|
||||
}
|
||||
}
|
||||
if (outcome !== 'restart') {
|
||||
if (!resolution.relaunch) {
|
||||
return
|
||||
}
|
||||
isQuitting = true
|
||||
@@ -1665,12 +1735,37 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
|
||||
app.exit(0)
|
||||
}
|
||||
|
||||
// Why a user-visible escape hatch: the marker is build-sticky, so a false latch
|
||||
// would otherwise hold a working GPU in software rendering until the next update.
|
||||
function turnOffGpuFallback(): void {
|
||||
const userDataPath = getCanonicalUserDataPath()
|
||||
clearGpuCrashHistory(userDataPath)
|
||||
if (!clearGpuFallbackMarker(userDataPath)) {
|
||||
// Why not relaunch: the next launch would read the marker and come straight back.
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_marker_clear_failed', { phase: 'user-opt-out' })
|
||||
return
|
||||
}
|
||||
provisionalGpuFallbackUserDataPath = null
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_disabled_by_user')
|
||||
isQuitting = true
|
||||
relaunchApp('gpu-fallback-opt-out')
|
||||
destroySystemTray()
|
||||
app.exit(0)
|
||||
}
|
||||
|
||||
// Why: an orderly shutdown proves Chromium's GPU CHECK never landed, so a marker
|
||||
// the user never confirmed must not survive into the next launch.
|
||||
function dropUnconfirmedGpuFallbackMarker(): void {
|
||||
if (provisionalGpuFallbackUserDataPath === null) {
|
||||
return
|
||||
}
|
||||
// Why the count too: an orderly shutdown proves the burst was survivable, so
|
||||
// leaving it armed would re-latch on the next single GPU crash inside the window.
|
||||
// Why not hoisted above the null guard: only an engagement sets this path, and a
|
||||
// quit that clears the count unconditionally would wipe the 1-2 crashes recorded
|
||||
// by a loop the user quits out of by hand — the counting this file exists for.
|
||||
// The residual is bounded: an uncleared count ages out of the 5-minute window.
|
||||
clearGpuCrashHistory(provisionalGpuFallbackUserDataPath)
|
||||
if (!clearGpuFallbackMarker(provisionalGpuFallbackUserDataPath)) {
|
||||
// `quit` is the last JS event, so this is terminal: the next launch starts in
|
||||
// safe graphics and clears itself on the next Orca or Electron version.
|
||||
@@ -2693,6 +2788,8 @@ void app.whenReady().then(async () => {
|
||||
|
||||
registerAppMenu({
|
||||
appMenuLabel: devInstanceIdentity.name,
|
||||
isGpuFallbackActive: () => gpuFallbackActiveThisLaunch,
|
||||
onTurnOffGpuFallback: turnOffGpuFallback,
|
||||
onCheckForUpdates: (options) => runUserInitiatedUpdateCheck(options),
|
||||
onBeforeReload: ({ ignoreCache, webContentsId }) => {
|
||||
if (mainWindow?.webContents.id === webContentsId) {
|
||||
|
||||
@@ -417,4 +417,35 @@ describe('registerAppMenu', () => {
|
||||
expect(appearanceSubmenu.find((item) => item.label === leftLabel)?.accelerator).toBeUndefined()
|
||||
expect(appearanceSubmenu.find((item) => item.label === rightLabel)?.accelerator).toBeUndefined()
|
||||
})
|
||||
|
||||
it('offers the safe-graphics escape hatch only while the fallback is latched', () => {
|
||||
const label = 'Turn Off Safe Graphics Mode'
|
||||
|
||||
registerAppMenu(buildMenuOptions())
|
||||
expect(getSubmenu(getTemplate(), 'Help').map((item) => item.label)).not.toContain(label)
|
||||
|
||||
buildFromTemplateMock.mockReset()
|
||||
buildFromTemplateMock.mockImplementation((template) => ({ template }))
|
||||
const options = {
|
||||
...buildMenuOptions(),
|
||||
isGpuFallbackActive: vi.fn(() => true),
|
||||
onTurnOffGpuFallback: vi.fn()
|
||||
}
|
||||
registerAppMenu(options)
|
||||
|
||||
// Why it must exist: the marker is build-sticky, so without this a false latch
|
||||
// holds a working GPU in software rendering until the next app update.
|
||||
const item = getSubmenu(getTemplate(), 'Help').find((entry) => entry.label === label)
|
||||
expect(item).toBeDefined()
|
||||
item?.click?.({} as never, {} as never, {} as never)
|
||||
expect(options.onTurnOffGpuFallback).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('hides the escape hatch when no handler is wired, rather than a dead item', () => {
|
||||
registerAppMenu({ ...buildMenuOptions(), isGpuFallbackActive: vi.fn(() => true) })
|
||||
|
||||
expect(getSubmenu(getTemplate(), 'Help').map((item) => item.label)).not.toContain(
|
||||
'Turn Off Safe Graphics Mode'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -37,6 +37,10 @@ type RegisterAppMenuOptions = {
|
||||
onToggleAppearance: (key: AppearanceMenuKey) => void
|
||||
getAppearanceState: () => AppearanceMenuState
|
||||
getKeybindings?: () => KeybindingOverrides | undefined
|
||||
// Why an escape hatch: the safe-graphics marker is build-sticky, so without this
|
||||
// a false latch leaves a user in software rendering until the next app update.
|
||||
isGpuFallbackActive?: () => boolean
|
||||
onTurnOffGpuFallback?: () => void
|
||||
// Why: the macOS app-menu title. Passed the per-branch dev label since
|
||||
// app.name is now pinned to a stable value for Keychain-key stability.
|
||||
appMenuLabel?: string
|
||||
@@ -57,7 +61,9 @@ function buildAndApplyMenu(options: RegisterAppMenuOptions): void {
|
||||
onToggleRightSidebar,
|
||||
onToggleAppearance,
|
||||
getAppearanceState,
|
||||
getKeybindings
|
||||
getKeybindings,
|
||||
isGpuFallbackActive,
|
||||
onTurnOffGpuFallback
|
||||
} = options
|
||||
|
||||
const isMac = process.platform === 'darwin'
|
||||
@@ -292,10 +298,24 @@ function buildAndApplyMenu(options: RegisterAppMenuOptions): void {
|
||||
submenu: [{ role: 'minimize' }, { role: 'zoom' }]
|
||||
}
|
||||
|
||||
// Why only while latched: the item is meaningless otherwise, and the marker is
|
||||
// read before whenReady so this state cannot change during the launch.
|
||||
const gpuFallbackItems: Electron.MenuItemConstructorOptions[] =
|
||||
isGpuFallbackActive?.() && onTurnOffGpuFallback
|
||||
? [
|
||||
{ type: 'separator' },
|
||||
{
|
||||
label: translateMain('menu.turnOffSafeGraphics', 'Turn Off Safe Graphics Mode'),
|
||||
click: () => onTurnOffGpuFallback()
|
||||
}
|
||||
]
|
||||
: []
|
||||
|
||||
const helpMenu: Electron.MenuItemConstructorOptions = {
|
||||
label: translateMain('menu.help', 'Help'),
|
||||
submenu: [
|
||||
crashReportItem,
|
||||
...gpuFallbackItems,
|
||||
{ type: 'separator' },
|
||||
featureTourItem,
|
||||
setupGuideItem,
|
||||
|
||||
@@ -137,15 +137,109 @@ describe('startup ordering', () => {
|
||||
expect(persistBody).not.toMatch(/\bawait[\s(]/)
|
||||
|
||||
// Why exactly two writers: the pre-prompt persist, plus the re-persist that
|
||||
// recovers a consented restart whose marker a session-end drop already took.
|
||||
// records consent (and recovers a marker a session-end drop already took).
|
||||
expect(handler.split('writeGpuFallbackMarker(')).toHaveLength(3)
|
||||
const confirmedQuittingIndex = handler.indexOf("outcome === 'confirmed-quitting'")
|
||||
expect(confirmedQuittingIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(handler.lastIndexOf('writeGpuFallbackMarker(')).toBeGreaterThan(confirmedQuittingIndex)
|
||||
const resolutionIndex = handler.indexOf('resolveGpuFallbackEngagement(outcome')
|
||||
expect(resolutionIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(handler.lastIndexOf('writeGpuFallbackMarker(')).toBeGreaterThan(resolutionIndex)
|
||||
|
||||
// Why pinned: the outcome -> retry mapping is unit-tested, so the handler must
|
||||
// defer to it rather than re-deriving the outcome list inline.
|
||||
expect(handler).toContain('if (!shouldKeepProvisionalGpuFallbackMarker(outcome)) {')
|
||||
// Why pinned: the outcome -> bookkeeping mapping is unit-tested in
|
||||
// gpu-fallback-engagement-resolution, so re-deriving it inline here would
|
||||
// restore the zero-coverage gap this extraction closed.
|
||||
expect(handler).toContain(
|
||||
'provisionalGpuFallbackUserDataPath = resolution.provisionalMarkerPath'
|
||||
)
|
||||
expect(handler).toContain('if (resolution.rePersistConsentedMarker) {')
|
||||
expect(handler).toContain('if (!resolution.relaunch) {')
|
||||
expect(handler).not.toContain('shouldKeepProvisionalGpuFallbackMarker(')
|
||||
|
||||
// Why: consent is what separates a latch the user chose from one the crash
|
||||
// imposed, and without it on disk the fix's own efficacy is unmeasurable.
|
||||
expect(handler).toContain('consented: false')
|
||||
expect(handler).toContain('consented: true')
|
||||
})
|
||||
|
||||
it('counts GPU crashes across launches, not just within one', () => {
|
||||
const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8')
|
||||
const start = source.indexOf('async function handleGpuChildCrash(')
|
||||
const end = source.indexOf('function recordProcessGoneCrash(', start)
|
||||
const handler = source.slice(start, end)
|
||||
|
||||
expect(start).toBeGreaterThanOrEqual(0)
|
||||
expect(end).toBeGreaterThan(start)
|
||||
|
||||
// Why both: the in-process burst path is what catches a driver that fails
|
||||
// mid-session; the durable path is the only one reachable when the crash
|
||||
// kills the launch in 0.8s.
|
||||
const inProcessIndex = handler.indexOf('gpuCrashFallbackTracker.recordGpuCrash(')
|
||||
const durableIndex = handler.indexOf('evaluateGpuCrashHistory(')
|
||||
const gateIndex = handler.indexOf(
|
||||
'if (!inProcess.shouldEngageFallback && !durable.crossesThreshold) {'
|
||||
)
|
||||
const persistIndex = handler.indexOf('persistGpuCrashTimes(')
|
||||
expect(inProcessIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(durableIndex).toBeGreaterThan(inProcessIndex)
|
||||
expect(gateIndex).toBeGreaterThan(durableIndex)
|
||||
// Why inside the non-firing branch: on the crash that fires, this write's fsync
|
||||
// would land in front of the marker write that is racing Chromium's kill.
|
||||
expect(persistIndex).toBeGreaterThan(gateIndex)
|
||||
expect(persistIndex).toBeLessThan(handler.indexOf('gpuFallbackEngagementStarted = true'))
|
||||
expect(handler.split('persistGpuCrashTimes(')).toHaveLength(2)
|
||||
|
||||
// Why inert off Windows: the environment check must precede every write.
|
||||
const environmentIndex = handler.indexOf('getWindowsGpuFallbackEnvironment()')
|
||||
expect(environmentIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(environmentIndex).toBeLessThan(inProcessIndex)
|
||||
expect(handler.indexOf('getCanonicalUserDataPath()')).toBeGreaterThan(environmentIndex)
|
||||
|
||||
// Why cleared only after the prompt resolved: a kill while the modal is up must
|
||||
// leave the count armed to fire again, while a decline must not re-prompt on the
|
||||
// very next crash.
|
||||
const clearIndex = handler.indexOf('clearGpuCrashHistory(userDataPath)')
|
||||
expect(clearIndex).toBeGreaterThan(handler.indexOf('await engageGpuFallback({'))
|
||||
// Why latched: either counter can fire, so a later same-session crash would
|
||||
// otherwise stack a second modal on top of the first.
|
||||
expect(handler).toContain('gpuFallbackEngagementStarted = true')
|
||||
expect(handler).toContain('gpuFallbackEngagementStarted ||')
|
||||
})
|
||||
|
||||
// Why one test over three call sites: they are the whole lifecycle of the durable
|
||||
// count, each is a one-token deletion away from silently reverting the feature to
|
||||
// the in-process tracker, and each survived the suite until this test existed.
|
||||
it('writes and clears the cross-launch GPU crash count at exactly three places', () => {
|
||||
const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8')
|
||||
const slice = (from: string, to: string): string => {
|
||||
const start = source.indexOf(from)
|
||||
const end = source.indexOf(to, start)
|
||||
expect(start).toBeGreaterThanOrEqual(0)
|
||||
expect(end).toBeGreaterThan(start)
|
||||
return source.slice(start, end)
|
||||
}
|
||||
|
||||
// Why guarded: an excluded reason (`launch-failed`) evaluates to the inert
|
||||
// decision, whose crashTimes is []. Persisting that truncates the real count to
|
||||
// zero, so one excluded death silently erases the whole accumulated history.
|
||||
const handler = slice('async function handleGpuChildCrash(', 'function recordProcessGoneCrash(')
|
||||
expect(handler).toContain('const countsDurably = countsTowardDurableGpuCrashHistory(reason)')
|
||||
expect(handler).toContain(
|
||||
'if (countsDurably) {\n persistGpuCrashTimes(userDataPath, environment, durable.crashTimes)\n }'
|
||||
)
|
||||
|
||||
// Why cleared on a safe-graphics launch: the marker is already applied, so the
|
||||
// pre-fallback times must not re-fire the prompt on the first GPU hiccup here.
|
||||
const reader = slice(
|
||||
'function maybeApplyGpuFallbackForThisLaunch()',
|
||||
'function recordGpuFallbackMarkerPersisted('
|
||||
)
|
||||
expect(reader).toContain('clearGpuCrashHistory(userDataPath)')
|
||||
|
||||
// Why cleared on an orderly quit: a shutdown that reached 'quit' proves the
|
||||
// burst was survivable, so an unconfirmed marker's count must not outlive it.
|
||||
const drop = slice(
|
||||
'function dropUnconfirmedGpuFallbackMarker()',
|
||||
'function recordProcessGoneCrash('
|
||||
)
|
||||
expect(drop).toContain('clearGpuCrashHistory(provisionalGpuFallbackUserDataPath)')
|
||||
})
|
||||
|
||||
it('drops an unconfirmed GPU fallback marker only once the quit is committed', () => {
|
||||
@@ -166,9 +260,37 @@ describe('startup ordering', () => {
|
||||
'dropUnconfirmedGpuFallbackMarker'
|
||||
)
|
||||
|
||||
// Why source-wide: the marker may only be dropped by the shutdown helper and
|
||||
// by the explicit "Keep Running" answer.
|
||||
expect(source.split('clearGpuFallbackMarker(')).toHaveLength(3)
|
||||
// Why source-wide: the marker may only be dropped by the shutdown helper, by
|
||||
// the explicit "Keep Running" answer, and by the Help-menu opt-out.
|
||||
expect(source.split('clearGpuFallbackMarker(')).toHaveLength(4)
|
||||
expect(source).toContain('function turnOffGpuFallback()')
|
||||
})
|
||||
|
||||
it('gives a latched user a way out that does not come straight back', () => {
|
||||
const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8')
|
||||
const start = source.indexOf('function turnOffGpuFallback()')
|
||||
const end = source.indexOf('function dropUnconfirmedGpuFallbackMarker()', start)
|
||||
const optOut = source.slice(start, end)
|
||||
|
||||
expect(start).toBeGreaterThanOrEqual(0)
|
||||
expect(end).toBeGreaterThan(start)
|
||||
|
||||
// Why both: the marker latches the next launch and the count would re-latch the
|
||||
// one after it, so clearing only one of them leaves the user still stuck.
|
||||
expect(optOut).toContain('clearGpuCrashHistory(userDataPath)')
|
||||
expect(optOut).toContain('clearGpuFallbackMarker(userDataPath)')
|
||||
// Why gated on the clear: relaunching over a marker still on disk returns the
|
||||
// user straight to safe graphics, having promised otherwise.
|
||||
const clearIndex = optOut.indexOf('if (!clearGpuFallbackMarker(userDataPath)) {')
|
||||
const relaunchIndex = optOut.indexOf("relaunchApp('gpu-fallback-opt-out'")
|
||||
expect(clearIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(relaunchIndex).toBeGreaterThan(clearIndex)
|
||||
// Why canonical: a split path would clear a marker the next launch never reads.
|
||||
expect(optOut).toContain('getCanonicalUserDataPath()')
|
||||
|
||||
// Why surfaced only while latched: the menu item is meaningless otherwise.
|
||||
expect(source).toContain('isGpuFallbackActive: () => gpuFallbackActiveThisLaunch')
|
||||
expect(source).toContain('onTurnOffGpuFallback: turnOffGpuFallback')
|
||||
})
|
||||
|
||||
it('reads and writes the GPU fallback marker through the same userData path', () => {
|
||||
@@ -184,11 +306,19 @@ describe('startup ordering', () => {
|
||||
// Why: a split between reader and writer paths turns the whole fallback into
|
||||
// a silent no-op — the marker lands somewhere the next launch never looks.
|
||||
expect(source.slice(readerStart, readerEnd)).toContain('getCanonicalUserDataPath()')
|
||||
// Why here: a kill between the marker's write and rename orphans a temp file,
|
||||
// and this is the only launch-time site that reclaims them.
|
||||
expect(source.slice(readerStart, readerEnd)).toContain(
|
||||
'sweepStaleGpuFallbackMarkerTempFiles(userDataPath)'
|
||||
)
|
||||
// Why here: a kill between a write and its rename orphans a temp file, and
|
||||
// this is the only launch-time site that reclaims them.
|
||||
const reader = source.slice(readerStart, readerEnd)
|
||||
expect(reader).toContain('sweepStaleGpuFallbackMarkerTempFiles(userDataPath)')
|
||||
expect(reader).toContain('sweepStaleGpuCrashHistoryTempFiles(userDataPath)')
|
||||
// Why gated: the sweep readdirs all of userData (Cache / GPUCache / Local
|
||||
// Storage) on a pre-whenReady path that runs on every Windows launch.
|
||||
expect(reader).toContain('if (sweepForOrphans) {')
|
||||
expect(reader).toContain('gpuFallbackMarkerFileExists(userDataPath)')
|
||||
expect(reader).toContain('gpuCrashHistoryFileExists(userDataPath)')
|
||||
// Why platform-gated: safe graphics is a Windows-only remedy, and
|
||||
// enableMainProcessGpuFeatures() carries the macOS Graphite fix it skips.
|
||||
expect(reader).toContain("process.platform !== 'win32'")
|
||||
expect(source.slice(readerEnd, writerEnd)).toContain('getCanonicalUserDataPath()')
|
||||
expect(source.slice(readerStart, writerEnd)).not.toContain("app.getPath('userData')")
|
||||
})
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD } from '../crash-reporting/gpu-crash-fallback-decision'
|
||||
import {
|
||||
DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS,
|
||||
GPU_CRASH_HISTORY_FILE,
|
||||
inertGpuCrashHistoryDecision,
|
||||
MAX_GPU_CRASH_HISTORY_ENTRIES,
|
||||
appendGpuCrashTime,
|
||||
clearGpuCrashHistory,
|
||||
gpuCrashHistoryFileExists,
|
||||
pruneGpuCrashTimes,
|
||||
readGpuCrashHistory,
|
||||
evaluateGpuCrashHistory,
|
||||
persistGpuCrashTimes,
|
||||
sweepStaleGpuCrashHistoryTempFiles
|
||||
} from './gpu-crash-history'
|
||||
import type { GpuFallbackEnvironment } from './gpu-fallback-marker'
|
||||
|
||||
const WINDOW = DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS
|
||||
const THRESHOLD = DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD
|
||||
const NOW = 1_800_000_000_000
|
||||
|
||||
describe('pruneGpuCrashTimes', () => {
|
||||
it('keeps only times inside the window, oldest first', () => {
|
||||
expect(pruneGpuCrashTimes([NOW - 100, NOW - WINDOW - 1, NOW - 50], NOW, WINDOW)).toEqual([
|
||||
NOW - 100,
|
||||
NOW - 50
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps a time exactly on the window edge', () => {
|
||||
expect(pruneGpuCrashTimes([NOW - WINDOW], NOW, WINDOW)).toEqual([NOW - WINDOW])
|
||||
})
|
||||
|
||||
it('drops values a clock jump or a corrupt file made nonsense', () => {
|
||||
// Why: these come off disk, so a NaN or a year-3000 stamp would otherwise pin
|
||||
// the count permanently over the threshold.
|
||||
const junk = [Number.NaN, Number.POSITIVE_INFINITY, -1, NOW + 86_400_000, '5', null, {}]
|
||||
expect(pruneGpuCrashTimes([...junk, NOW - 10], NOW, WINDOW)).toEqual([NOW - 10])
|
||||
})
|
||||
|
||||
it('tolerates a small forward clock skew', () => {
|
||||
// NTP corrections move the wall clock by seconds; discarding those would drop
|
||||
// a crash that really did just happen.
|
||||
expect(pruneGpuCrashTimes([NOW + 5_000], NOW, WINDOW)).toEqual([NOW + 5_000])
|
||||
})
|
||||
|
||||
it('returns nothing when `now` itself is unusable', () => {
|
||||
expect(pruneGpuCrashTimes([NOW], Number.NaN, WINDOW)).toEqual([])
|
||||
})
|
||||
|
||||
it('drops entries stranded in the future by a backwards clock jump', () => {
|
||||
// Why this matters most: a rolling window only ever prunes the OLD end, so a
|
||||
// future-dated entry would sit in the window forever and two later crashes
|
||||
// would latch build-sticky safe graphics with no burst and no consent.
|
||||
const stored = [NOW - 30_000, NOW - 15_000, NOW]
|
||||
for (const jumpMs of [
|
||||
90_000, // NTP correction past the skew tolerance
|
||||
3_600_000, // hibernate / VM resume
|
||||
5 * 3_600_000 // dual-boot RTC skew: Windows local time vs Linux UTC
|
||||
]) {
|
||||
const rewound = NOW - jumpMs
|
||||
const kept = pruneGpuCrashTimes(stored, rewound, WINDOW)
|
||||
// Nothing survives beyond the skew tolerance, so a rewind can never leave
|
||||
// enough entries to reach the threshold on its own.
|
||||
expect([jumpMs, kept.every((time) => time <= rewound + 60_000)]).toEqual([jumpMs, true])
|
||||
expect([jumpMs, kept.length < THRESHOLD]).toEqual([jumpMs, true])
|
||||
}
|
||||
|
||||
// A rewind past the tolerance strands everything.
|
||||
expect(pruneGpuCrashTimes(stored, NOW - 3_600_000, WINDOW)).toEqual([])
|
||||
// Inside the tolerance, only what is genuinely near-present survives.
|
||||
expect(pruneGpuCrashTimes(stored, NOW - 90_000, WINDOW)).toEqual([NOW - 30_000])
|
||||
})
|
||||
|
||||
it('cannot be pushed over the threshold by future-dated entries', () => {
|
||||
const rewound = NOW - 4 * 3_600_000
|
||||
const result = appendGpuCrashTime([NOW - 20_000, NOW - 10_000], rewound, {
|
||||
windowMs: WINDOW,
|
||||
threshold: THRESHOLD
|
||||
})
|
||||
// Only the crash that actually just happened survives.
|
||||
expect(result).toEqual({
|
||||
crashTimes: [rewound],
|
||||
crashesInWindow: 1,
|
||||
crossesThreshold: false
|
||||
})
|
||||
})
|
||||
|
||||
it('does not let future-dated entries fill the cap and crowd out real crashes', () => {
|
||||
const rewound = NOW - 3_600_000
|
||||
const future = Array.from({ length: MAX_GPU_CRASH_HISTORY_ENTRIES * 2 }, (_, i) => NOW + i)
|
||||
const real = [rewound - 30_000, rewound - 15_000]
|
||||
const result = appendGpuCrashTime([...future, ...real], rewound, {
|
||||
windowMs: WINDOW,
|
||||
threshold: THRESHOLD
|
||||
})
|
||||
expect(result.crashTimes).toEqual([rewound - 30_000, rewound - 15_000, rewound])
|
||||
expect(result.crossesThreshold).toBe(true)
|
||||
})
|
||||
|
||||
it('evicts oldest-first and still crosses the threshold at a full cap', () => {
|
||||
// Why: a cap that evicted the newest, or that sat below the threshold, would
|
||||
// silently suppress a machine crashing hard enough to overflow it.
|
||||
const many = Array.from({ length: MAX_GPU_CRASH_HISTORY_ENTRIES + 40 }, (_, i) => NOW - 100 + i)
|
||||
const result = appendGpuCrashTime(many, NOW, { windowMs: WINDOW, threshold: THRESHOLD })
|
||||
expect(result.crashTimes).toHaveLength(MAX_GPU_CRASH_HISTORY_ENTRIES)
|
||||
expect(result.crashTimes[0]).toBeGreaterThan(many[0])
|
||||
expect(result.crashTimes.at(-1)).toBe(NOW)
|
||||
expect(result.crossesThreshold).toBe(true)
|
||||
expect(MAX_GPU_CRASH_HISTORY_ENTRIES).toBeGreaterThan(THRESHOLD)
|
||||
})
|
||||
})
|
||||
|
||||
describe('DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS', () => {
|
||||
it('spans the measured crash-loop shape without reaching back hours', () => {
|
||||
// Measured: 18-24s median gap between consecutive crashing launches; the first
|
||||
// 8 crashes on F0BN5HZL8FJ span 96s. Three crashes need ~40-50s.
|
||||
expect(DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS).toBeGreaterThan(2 * 24_000 * 3)
|
||||
// Every extra minute is false-positive surface, and the marker is written
|
||||
// before the user consents.
|
||||
expect(DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS).toBeLessThanOrEqual(300_000)
|
||||
})
|
||||
})
|
||||
|
||||
describe('appendGpuCrashTime', () => {
|
||||
it('does not cross the threshold before the third crash', () => {
|
||||
expect(appendGpuCrashTime([], NOW, { windowMs: WINDOW, threshold: THRESHOLD })).toMatchObject({
|
||||
crashesInWindow: 1,
|
||||
crossesThreshold: false
|
||||
})
|
||||
expect(
|
||||
appendGpuCrashTime([NOW - 20_000], NOW, { windowMs: WINDOW, threshold: THRESHOLD })
|
||||
).toMatchObject({ crashesInWindow: 2, crossesThreshold: false })
|
||||
})
|
||||
|
||||
it('crosses on the third crash spread across launches', () => {
|
||||
// The measured shape: one GPU death per launch, launches ~20s apart.
|
||||
const result = appendGpuCrashTime([NOW - 44_000, NOW - 22_000], NOW, {
|
||||
windowMs: WINDOW,
|
||||
threshold: THRESHOLD
|
||||
})
|
||||
expect(result).toMatchObject({ crashesInWindow: 3, crossesThreshold: true })
|
||||
expect(result.crashTimes).toEqual([NOW - 44_000, NOW - 22_000, NOW])
|
||||
})
|
||||
|
||||
it('does not count crashes that aged out of the window', () => {
|
||||
expect(
|
||||
appendGpuCrashTime([NOW - WINDOW - 1, NOW - WINDOW - 2], NOW, {
|
||||
windowMs: WINDOW,
|
||||
threshold: THRESHOLD
|
||||
})
|
||||
).toMatchObject({ crashesInWindow: 1, crossesThreshold: false })
|
||||
})
|
||||
|
||||
it('bounds the array so a crash loop cannot grow the file', () => {
|
||||
const many = Array.from({ length: 200 }, (_, index) => NOW - 200 + index)
|
||||
const result = appendGpuCrashTime(many, NOW, { windowMs: WINDOW, threshold: THRESHOLD })
|
||||
expect(result.crashTimes).toHaveLength(MAX_GPU_CRASH_HISTORY_ENTRIES)
|
||||
// Why the newest: the dropped entries are the ones about to age out anyway.
|
||||
expect(result.crashTimes.at(-1)).toBe(NOW)
|
||||
expect(result.crashesInWindow).toBe(MAX_GPU_CRASH_HISTORY_ENTRIES)
|
||||
})
|
||||
|
||||
it('never counts a crash time it refused to keep', () => {
|
||||
const result = appendGpuCrashTime([NOW - 10, NOW - 20], Number.NaN, {
|
||||
windowMs: WINDOW,
|
||||
threshold: THRESHOLD
|
||||
})
|
||||
expect(result).toEqual({ crashTimes: [], crashesInWindow: 0, crossesThreshold: false })
|
||||
})
|
||||
})
|
||||
|
||||
describe('gpu-crash-history file', () => {
|
||||
let userDataPath: string
|
||||
const environment = {
|
||||
appVersion: '1.4.163',
|
||||
electronVersion: '42.3.3',
|
||||
platform: 'win32' as const
|
||||
}
|
||||
/** The production sequence for a crash that does not fire: evaluate, then persist. */
|
||||
const record = (now: number, env: GpuFallbackEnvironment = environment) => {
|
||||
const decision = evaluateGpuCrashHistory(userDataPath, env, {
|
||||
now,
|
||||
windowMs: WINDOW,
|
||||
threshold: THRESHOLD
|
||||
})
|
||||
persistGpuCrashTimes(userDataPath, env, decision.crashTimes)
|
||||
return decision
|
||||
}
|
||||
beforeEach(() => {
|
||||
userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-crash-history-test-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('accumulates across separate calls and fires on the third', () => {
|
||||
// The bug this exists for: three launches, one GPU death each, ~20s apart.
|
||||
expect(record(NOW).crossesThreshold).toBe(false)
|
||||
expect(record(NOW + 20_000).crossesThreshold).toBe(false)
|
||||
const third = record(NOW + 41_000)
|
||||
expect(third.crossesThreshold).toBe(true)
|
||||
expect(third.crashesInWindow).toBe(3)
|
||||
})
|
||||
|
||||
it('evaluates without writing, so the fsync never precedes the marker write', () => {
|
||||
// Why load-bearing: the marker write races a kill that can land ~7ms after the
|
||||
// GPU failure, so the firing crash skips this write entirely.
|
||||
record(NOW)
|
||||
record(NOW + 1_000)
|
||||
const before = readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8')
|
||||
|
||||
const decision = evaluateGpuCrashHistory(userDataPath, environment, {
|
||||
now: NOW + 2_000,
|
||||
windowMs: WINDOW,
|
||||
threshold: THRESHOLD
|
||||
})
|
||||
|
||||
expect(decision.crossesThreshold).toBe(true)
|
||||
expect(readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8')).toBe(before)
|
||||
})
|
||||
|
||||
it('is truncated to nothing by persisting an inert decision', () => {
|
||||
// Why asserted rather than commented: an excluded reason (`launch-failed`) makes
|
||||
// handleGpuChildCrash evaluate to inertGpuCrashHistoryDecision(), whose crashTimes
|
||||
// is []. Persisting that rewrites the file empty, so a single excluded GPU death
|
||||
// erases every real crash counted so far and the fallback never reaches three.
|
||||
// The persist must therefore stay gated on the reason counting durably.
|
||||
record(NOW)
|
||||
record(NOW + 1_000)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toHaveLength(2)
|
||||
|
||||
persistGpuCrashTimes(userDataPath, environment, inertGpuCrashHistoryDecision().crashTimes)
|
||||
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toEqual([])
|
||||
})
|
||||
|
||||
it('round-trips the persisted times', () => {
|
||||
record(NOW)
|
||||
record(NOW + 1_000)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toEqual([NOW, NOW + 1_000])
|
||||
expect(gpuCrashHistoryFileExists(userDataPath)).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves no temp file behind', () => {
|
||||
record(NOW)
|
||||
record(NOW + 1)
|
||||
expect(readdirSync(userDataPath).filter((name) => name.endsWith('.tmp'))).toEqual([])
|
||||
})
|
||||
|
||||
it('discards history from a different app build', () => {
|
||||
record(NOW)
|
||||
record(NOW + 1_000)
|
||||
|
||||
const upgraded = { ...environment, appVersion: '1.4.164' }
|
||||
expect(readGpuCrashHistory(userDataPath, upgraded)).toEqual([])
|
||||
// A new build gets one clean slate, the same policy the marker uses.
|
||||
expect(record(NOW + 2_000, upgraded).crashesInWindow).toBe(1)
|
||||
})
|
||||
|
||||
it('discards history from a different Electron build', () => {
|
||||
record(NOW)
|
||||
expect(
|
||||
readGpuCrashHistory(userDataPath, { ...environment, electronVersion: '43.0.0' })
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
it('discards a corrupt or wrong-version file instead of trusting it', () => {
|
||||
writeFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), '{ not json')
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toEqual([])
|
||||
|
||||
writeFileSync(
|
||||
join(userDataPath, GPU_CRASH_HISTORY_FILE),
|
||||
JSON.stringify({ schemeVersion: 999, crashTimes: [NOW, NOW, NOW], ...environment })
|
||||
)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toEqual([])
|
||||
expect(record(NOW).crossesThreshold).toBe(false)
|
||||
})
|
||||
|
||||
it('ignores a crashTimes value that is not an array', () => {
|
||||
writeFileSync(
|
||||
join(userDataPath, GPU_CRASH_HISTORY_FILE),
|
||||
JSON.stringify({ schemeVersion: 1, crashTimes: 'lots', ...environment })
|
||||
)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toEqual([])
|
||||
})
|
||||
|
||||
it('does not latch after the wall clock steps backwards', () => {
|
||||
// Two real crashes, then the clock rewinds 5 hours (dual-boot RTC skew). The
|
||||
// stored entries are now future-dated; a rolling window never prunes that end,
|
||||
// so without the future check they would sit there and the next crash would be
|
||||
// the "third" — safe graphics latched with no burst and no consent.
|
||||
record(NOW)
|
||||
record(NOW + 20_000)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toHaveLength(2)
|
||||
|
||||
const rewound = NOW - 5 * 3_600_000
|
||||
const afterJump = record(rewound)
|
||||
|
||||
expect(afterJump.crossesThreshold).toBe(false)
|
||||
expect(afterJump.crashesInWindow).toBe(1)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toEqual([rewound])
|
||||
})
|
||||
|
||||
it('does not fire on a stale count from hours earlier', () => {
|
||||
record(NOW)
|
||||
record(NOW + 1_000)
|
||||
expect(record(NOW + 4 * 3_600_000).crossesThreshold).toBe(false)
|
||||
})
|
||||
|
||||
it('caps the persisted array', () => {
|
||||
for (let index = 0; index < MAX_GPU_CRASH_HISTORY_ENTRIES + 20; index += 1) {
|
||||
record(NOW + index)
|
||||
}
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toHaveLength(
|
||||
MAX_GPU_CRASH_HISTORY_ENTRIES
|
||||
)
|
||||
})
|
||||
|
||||
it('clears the history so an engaged fallback cannot immediately re-fire', () => {
|
||||
record(NOW)
|
||||
record(NOW + 1_000)
|
||||
clearGpuCrashHistory(userDataPath)
|
||||
|
||||
expect(gpuCrashHistoryFileExists(userDataPath)).toBe(false)
|
||||
expect(record(NOW + 2_000).crossesThreshold).toBe(false)
|
||||
})
|
||||
|
||||
it('clearing a history that is already gone is not an error', () => {
|
||||
expect(() => clearGpuCrashHistory(userDataPath)).not.toThrow()
|
||||
})
|
||||
|
||||
it('hands out a fresh inert decision rather than one shared mutable array', () => {
|
||||
const first = inertGpuCrashHistoryDecision()
|
||||
first.crashTimes.push(NOW)
|
||||
expect(inertGpuCrashHistoryDecision()).toEqual({
|
||||
crashTimes: [],
|
||||
crashesInWindow: 0,
|
||||
crossesThreshold: false
|
||||
})
|
||||
})
|
||||
|
||||
it('never throws when the file cannot be written', () => {
|
||||
// A directory at the target path fails both the durable and the direct write.
|
||||
mkdirSync(join(userDataPath, GPU_CRASH_HISTORY_FILE))
|
||||
|
||||
// Why: this runs on the path racing Chromium's kill, so a throw here would
|
||||
// take out the marker write that is the actual remedy.
|
||||
expect(() => record(NOW)).not.toThrow()
|
||||
expect(readdirSync(userDataPath).filter((name) => name.endsWith('.tmp'))).toEqual([])
|
||||
})
|
||||
|
||||
it('sweeps temp files orphaned by a kill between write and rename', async () => {
|
||||
record(NOW)
|
||||
const orphan = join(userDataPath, `${GPU_CRASH_HISTORY_FILE}.999999.123.abc.tmp`)
|
||||
writeFileSync(orphan, '{}')
|
||||
|
||||
await sweepStaleGpuCrashHistoryTempFiles(userDataPath)
|
||||
|
||||
expect(existsSync(orphan)).toBe(false)
|
||||
// The history itself shares the prefix and must survive the sweep.
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toEqual([NOW])
|
||||
})
|
||||
|
||||
for (const platform of ['darwin', 'linux'] as const) {
|
||||
it(`writes nothing and never fires on ${platform}`, () => {
|
||||
const foreign = { ...environment, platform }
|
||||
|
||||
for (let index = 0; index < 10; index += 1) {
|
||||
expect(record(NOW + index * 1_000, foreign)).toEqual({
|
||||
crashTimes: [],
|
||||
crashesInWindow: 0,
|
||||
crossesThreshold: false
|
||||
})
|
||||
}
|
||||
|
||||
expect(existsSync(join(userDataPath, GPU_CRASH_HISTORY_FILE))).toBe(false)
|
||||
expect(readdirSync(userDataPath)).toEqual([])
|
||||
})
|
||||
|
||||
it(`discards a history file that followed a profile onto ${platform}`, () => {
|
||||
record(NOW)
|
||||
record(NOW + 1_000)
|
||||
const persisted = readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8')
|
||||
expect(persisted).toContain('win32')
|
||||
|
||||
expect(readGpuCrashHistory(userDataPath, { ...environment, platform })).toEqual([])
|
||||
expect(existsSync(join(userDataPath, GPU_CRASH_HISTORY_FILE))).toBe(false)
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,242 @@
|
||||
import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import {
|
||||
durableWriteTempPath,
|
||||
removeStaleDurableWriteTempFiles,
|
||||
writeFileDurableSync
|
||||
} from '../durable-file-write'
|
||||
import type { GpuFallbackEnvironment } from './gpu-fallback-marker'
|
||||
|
||||
/**
|
||||
* Cross-launch record of GPU child-process deaths.
|
||||
*
|
||||
* Why durable: `GpuCrashFallbackTracker` counts crashes in memory, so its count
|
||||
* resets to zero on every launch. In the 39 Windows bundles, 53 of the 55 launches
|
||||
* that recorded a GPU death recorded exactly one (median 1.4s after `app_started`,
|
||||
* range 0.8-510s), and none recorded three inside the tracker's 30s window.
|
||||
*
|
||||
* That per-launch count is a floor rather than a measurement: `process_gone_suppressed`
|
||||
* coalesces identical repeats on a 30s key, and the same bundles carry 36
|
||||
* `gpu_fallback_applied` breadcrumbs, so the in-process tracker does still reach three
|
||||
* and engage. Counting across launches is what makes the threshold reachable from the
|
||||
* deaths that are observable.
|
||||
*
|
||||
* Not established by that data: that engaging helps. 30 of the 56 attributable deaths
|
||||
* landed on a launch that had already applied safe graphics, all STATUS_BREAKPOINT —
|
||||
* but every one predates `--in-process-gpu` (#11295, v1.4.163, tagged 5h after the last
|
||||
* of them), which is what finally removed the GPU child the old fallback left running.
|
||||
* Efficacy of the current fallback is unmeasured in both directions.
|
||||
*
|
||||
* Why a standalone file (not the Store): same reason as gpu-fallback-marker.ts —
|
||||
* this is read and written before app.whenReady(), where the Store does not exist.
|
||||
*/
|
||||
|
||||
export const GPU_CRASH_HISTORY_FILE = 'gpu-crash-history.json'
|
||||
export const GPU_CRASH_HISTORY_SCHEME_VERSION = 1
|
||||
|
||||
/**
|
||||
* Why 5 minutes, derived from the measured distribution rather than rounded up:
|
||||
* the gap between consecutive crashing launches was 18-24s (median), and the
|
||||
* first 8 crashes on F0BN5HZL8FJ span 96s (12s/crash). Three crashes therefore
|
||||
* need ~40-50s. Five minutes tolerates ~2.5 min on each of the two inter-crash
|
||||
* gaps — 6-8x the observed median, enough for a user who reads the Windows crash
|
||||
* dialog or tries a driver update before relaunching — and still covers ~12
|
||||
* consecutive launches at the observed rate.
|
||||
*
|
||||
* Why not 10 minutes: false-positive surface scales linearly with the window,
|
||||
* and the marker is written before the user consents. Too short only delays the
|
||||
* remedy to the next tight burst; too long latches safe graphics on a machine
|
||||
* that was never in the loop. The asymmetry says pick the shorter number.
|
||||
*/
|
||||
export const DEFAULT_GPU_CRASH_DURABLE_WINDOW_MS = 300_000
|
||||
|
||||
/**
|
||||
* Why 32: an order of magnitude above the threshold of 3, so the cap can never
|
||||
* suppress a crossing (that would need >32 crashes inside the window, which is
|
||||
* 10x over threshold and already firing). Eviction is oldest-first — entries
|
||||
* nearest to ageing out anyway. Future-dated entries are dropped before the cap
|
||||
* applies, so a bad clock cannot fill it and crowd out real crashes.
|
||||
*/
|
||||
export const MAX_GPU_CRASH_HISTORY_ENTRIES = 32
|
||||
|
||||
// Why tolerated at all: NTP corrections move the wall clock by seconds. Anything
|
||||
// further ahead is a bad clock, and trusting it would pin a stale crash in the
|
||||
// window until the clock caught up.
|
||||
const GPU_CRASH_HISTORY_FUTURE_SKEW_MS = 60_000
|
||||
|
||||
export type GpuCrashHistoryDecision = {
|
||||
/** Times to persist, oldest first. */
|
||||
crashTimes: number[]
|
||||
crashesInWindow: number
|
||||
/** True once the durable count reaches the fallback threshold. */
|
||||
crossesThreshold: boolean
|
||||
}
|
||||
|
||||
/** Epoch-ms crash times still inside the window, dropping anything a clock jump made nonsense. */
|
||||
export function pruneGpuCrashTimes(
|
||||
times: readonly unknown[],
|
||||
now: number,
|
||||
windowMs: number
|
||||
): number[] {
|
||||
if (!Number.isFinite(now)) {
|
||||
return []
|
||||
}
|
||||
const oldest = now - windowMs
|
||||
const newest = now + GPU_CRASH_HISTORY_FUTURE_SKEW_MS
|
||||
return times
|
||||
.filter(
|
||||
(time): time is number =>
|
||||
typeof time === 'number' &&
|
||||
Number.isFinite(time) &&
|
||||
time >= 0 &&
|
||||
time >= oldest &&
|
||||
time <= newest
|
||||
)
|
||||
.sort((a, b) => a - b)
|
||||
}
|
||||
|
||||
/** Adds `now` to `times` and reports whether the durable threshold is now met. */
|
||||
export function appendGpuCrashTime(
|
||||
times: readonly unknown[],
|
||||
now: number,
|
||||
options: { windowMs: number; threshold: number; maxEntries?: number }
|
||||
): GpuCrashHistoryDecision {
|
||||
const pruned = pruneGpuCrashTimes([...times, now], now, options.windowMs)
|
||||
// Keep the newest: the entries dropped here are the ones about to age out anyway.
|
||||
const crashTimes = pruned.slice(-(options.maxEntries ?? MAX_GPU_CRASH_HISTORY_ENTRIES))
|
||||
return {
|
||||
crashTimes,
|
||||
crashesInWindow: crashTimes.length,
|
||||
crossesThreshold: crashTimes.length >= options.threshold
|
||||
}
|
||||
}
|
||||
|
||||
/** Why a factory: a shared instance would hand every caller the same mutable array. */
|
||||
export function inertGpuCrashHistoryDecision(): GpuCrashHistoryDecision {
|
||||
return { crashTimes: [], crashesInWindow: 0, crossesThreshold: false }
|
||||
}
|
||||
|
||||
function historyPath(userDataPath: string): string {
|
||||
return join(userDataPath, GPU_CRASH_HISTORY_FILE)
|
||||
}
|
||||
|
||||
/** Why exported: the startup temp sweep is gated on a fallback file having existed. */
|
||||
export function gpuCrashHistoryFileExists(userDataPath: string): boolean {
|
||||
return existsSync(historyPath(userDataPath))
|
||||
}
|
||||
|
||||
/** Why no retry: a stale history costs at most one extra loop iteration. */
|
||||
function clearHistoryBestEffort(userDataPath: string): void {
|
||||
try {
|
||||
rmSync(historyPath(userDataPath), { force: true })
|
||||
} catch {
|
||||
// The next launch revalidates it anyway.
|
||||
}
|
||||
}
|
||||
|
||||
export function clearGpuCrashHistory(userDataPath: string): void {
|
||||
clearHistoryBestEffort(userDataPath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stored crash times for this exact build. A mismatched build or platform discards
|
||||
* them, the same policy the marker uses: a new build gets one clean slate.
|
||||
*/
|
||||
export function readGpuCrashHistory(
|
||||
userDataPath: string,
|
||||
environment: GpuFallbackEnvironment
|
||||
): number[] {
|
||||
if (environment.platform !== 'win32') {
|
||||
clearHistoryBestEffort(userDataPath)
|
||||
return []
|
||||
}
|
||||
let parsed: Record<string, unknown>
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(historyPath(userDataPath), 'utf-8')) as Record<string, unknown>
|
||||
} catch {
|
||||
// Missing or corrupt means no history; a corrupt file is overwritten by the next write.
|
||||
return []
|
||||
}
|
||||
if (
|
||||
parsed.schemeVersion !== GPU_CRASH_HISTORY_SCHEME_VERSION ||
|
||||
parsed.platform !== 'win32' ||
|
||||
parsed.appVersion !== environment.appVersion ||
|
||||
parsed.electronVersion !== environment.electronVersion
|
||||
) {
|
||||
clearHistoryBestEffort(userDataPath)
|
||||
return []
|
||||
}
|
||||
const stored: unknown[] = Array.isArray(parsed.crashTimes) ? parsed.crashTimes : []
|
||||
// Range checks belong to pruneGpuCrashTimes, which needs `now`; this is only the
|
||||
// shape boundary, so a hand-edited file cannot put a string into the array.
|
||||
return stored.filter(
|
||||
(time): time is number => typeof time === 'number' && Number.isFinite(time) && time >= 0
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Persists `crashTimes`. Never throws: this runs on the crash path racing
|
||||
* Chromium's kill, and a lost entry only costs one more iteration of the loop.
|
||||
*
|
||||
* Why separate from the evaluation: the caller skips this entirely on the crash
|
||||
* that fires the fallback, so its fsync never lands in front of the marker write.
|
||||
*/
|
||||
export function persistGpuCrashTimes(
|
||||
userDataPath: string,
|
||||
environment: GpuFallbackEnvironment,
|
||||
crashTimes: readonly number[]
|
||||
): void {
|
||||
if (environment.platform !== 'win32') {
|
||||
return
|
||||
}
|
||||
const target = historyPath(userDataPath)
|
||||
const payload = JSON.stringify({
|
||||
schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION,
|
||||
crashTimes,
|
||||
appVersion: environment.appVersion,
|
||||
electronVersion: environment.electronVersion,
|
||||
platform: 'win32'
|
||||
})
|
||||
const tmp = durableWriteTempPath(target)
|
||||
try {
|
||||
writeFileDurableSync(tmp, target, payload)
|
||||
return
|
||||
} catch {
|
||||
// Windows AV/indexer holds make renameSync EPERM where a direct write still lands.
|
||||
}
|
||||
try {
|
||||
writeFileSync(target, payload)
|
||||
} catch {
|
||||
// Nothing left to try; the in-process tracker is still armed for this session.
|
||||
}
|
||||
try {
|
||||
rmSync(tmp, { force: true })
|
||||
} catch {
|
||||
// Swept next launch alongside the marker's orphans.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the stored history, adds a GPU child death at `options.now` (epoch ms),
|
||||
* and reports whether the durable count has reached the threshold. Writes nothing
|
||||
* — the caller decides, so a fallback that fires can go straight to the marker.
|
||||
* Inert off Windows.
|
||||
*/
|
||||
export function evaluateGpuCrashHistory(
|
||||
userDataPath: string,
|
||||
environment: GpuFallbackEnvironment,
|
||||
options: { now: number; windowMs: number; threshold: number }
|
||||
): GpuCrashHistoryDecision {
|
||||
if (environment.platform !== 'win32') {
|
||||
return inertGpuCrashHistoryDecision()
|
||||
}
|
||||
return appendGpuCrashTime(readGpuCrashHistory(userDataPath, environment), options.now, {
|
||||
windowMs: options.windowMs,
|
||||
threshold: options.threshold
|
||||
})
|
||||
}
|
||||
|
||||
/** Why: the kill this history exists to survive can land between write and rename. */
|
||||
export async function sweepStaleGpuCrashHistoryTempFiles(userDataPath: string): Promise<void> {
|
||||
await removeStaleDurableWriteTempFiles(historyPath(userDataPath))
|
||||
}
|
||||
@@ -44,6 +44,7 @@ vi.mock('../durable-file-write', async (importOriginal) => {
|
||||
import {
|
||||
GPU_FALLBACK_MARKER_FILE,
|
||||
clearGpuFallbackMarker,
|
||||
gpuFallbackMarkerFileExists,
|
||||
readActiveGpuFallbackMarker,
|
||||
readGpuFallbackMarker,
|
||||
sweepStaleGpuFallbackMarkerTempFiles,
|
||||
@@ -73,20 +74,71 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('round-trips a written marker', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 123, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 123, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
expect(readGpuFallbackMarker(userDataPath)).toEqual({
|
||||
schemeVersion: 2,
|
||||
schemeVersion: 3,
|
||||
engagedAt: 123,
|
||||
crashesInWindow: 3,
|
||||
consented: false,
|
||||
appVersion: '1.2.3',
|
||||
electronVersion: '42.3.3',
|
||||
platform: 'win32'
|
||||
})
|
||||
})
|
||||
|
||||
it('round-trips the consent flag that separates a latch from a chosen restart', () => {
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 123, crashesInWindow: 3, consented: true },
|
||||
environment
|
||||
)
|
||||
expect(readGpuFallbackMarker(userDataPath)?.consented).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects a scheme-2 marker that predates the consent flag', () => {
|
||||
// Why: without `consented` there is no way to tell a user-confirmed latch from
|
||||
// one the crash imposed, so the bump must invalidate the old shape outright.
|
||||
writeFileSync(
|
||||
join(userDataPath, GPU_FALLBACK_MARKER_FILE),
|
||||
JSON.stringify({
|
||||
schemeVersion: 2,
|
||||
engagedAt: 1,
|
||||
crashesInWindow: 3,
|
||||
...environment
|
||||
})
|
||||
)
|
||||
expect(readGpuFallbackMarker(userDataPath)).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects a marker whose consent flag is not a boolean', () => {
|
||||
writeFileSync(
|
||||
join(userDataPath, GPU_FALLBACK_MARKER_FILE),
|
||||
JSON.stringify({
|
||||
schemeVersion: 3,
|
||||
engagedAt: 1,
|
||||
crashesInWindow: 3,
|
||||
consented: 'yes',
|
||||
...environment
|
||||
})
|
||||
)
|
||||
expect(readGpuFallbackMarker(userDataPath)).toBeNull()
|
||||
})
|
||||
|
||||
it('overwrites an existing marker and leaves no temp file behind', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 2, crashesInWindow: 5 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 2, crashesInWindow: 5, consented: false },
|
||||
environment
|
||||
)
|
||||
|
||||
expect(readGpuFallbackMarker(userDataPath)?.engagedAt).toBe(2)
|
||||
// Why: the durable write renames through a temp file; an orphan would accumulate
|
||||
@@ -99,7 +151,11 @@ describe('gpu-fallback-marker', () => {
|
||||
// lands. This path must never end up worse than a plain writeFileSync.
|
||||
durableWrite.failWith = Object.assign(new Error('EPERM: rename'), { code: 'EPERM' })
|
||||
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 7, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 7, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
|
||||
// Why assert the call: an ineffective mock would leave this green while
|
||||
// covering nothing but the happy path.
|
||||
@@ -116,7 +172,11 @@ describe('gpu-fallback-marker', () => {
|
||||
|
||||
// Why the durable error, not the fallback's: it names the real cause.
|
||||
expect(() =>
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 9, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 9, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
).toThrow(durableError)
|
||||
})
|
||||
|
||||
@@ -126,7 +186,11 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('keeps an active marker for repeated launches on the same build', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 4, consented: false },
|
||||
environment
|
||||
)
|
||||
expect(existsSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE))).toBe(true)
|
||||
|
||||
const firstRead = readActiveGpuFallbackMarker(userDataPath, environment)
|
||||
@@ -138,7 +202,11 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('clears an active marker when the app build changes', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 4, consented: false },
|
||||
environment
|
||||
)
|
||||
|
||||
expect(
|
||||
readActiveGpuFallbackMarker(userDataPath, {
|
||||
@@ -150,7 +218,11 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('clears an active marker outside Windows', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 4, consented: false },
|
||||
environment
|
||||
)
|
||||
|
||||
expect(
|
||||
readActiveGpuFallbackMarker(userDataPath, {
|
||||
@@ -165,7 +237,11 @@ describe('gpu-fallback-marker', () => {
|
||||
// carries the macOS disable-skia-graphite fix. A marker that survived on darwin would silently
|
||||
// strip the fix from the Macs it targets, so pin the platform gate for darwin specifically.
|
||||
it('clears an active marker on macOS so the Graphite fix is never skipped', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 4, consented: false },
|
||||
environment
|
||||
)
|
||||
|
||||
expect(
|
||||
readActiveGpuFallbackMarker(userDataPath, {
|
||||
@@ -191,8 +267,20 @@ describe('gpu-fallback-marker', () => {
|
||||
expect(existsSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE))).toBe(false)
|
||||
})
|
||||
|
||||
it('reports a marker file that exists but does not parse, so the sweep still runs', () => {
|
||||
// Why: the startup sweep is gated on this, and gating it on a *readable* marker
|
||||
// would skip the reclaim in exactly the torn-write case it exists for.
|
||||
expect(gpuFallbackMarkerFileExists(userDataPath)).toBe(false)
|
||||
writeFileSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE), '{ not json')
|
||||
expect(gpuFallbackMarkerFileExists(userDataPath)).toBe(true)
|
||||
})
|
||||
|
||||
it('sweeps temp files orphaned by a kill between write and rename', async () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
// A foreign pid: the sweeper deliberately skips this process's own in-flight temps.
|
||||
const orphan = join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.999999.123.abc.tmp`)
|
||||
writeFileSync(orphan, '{}')
|
||||
@@ -205,7 +293,11 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('retries a marker delete that is briefly refused', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
rmControl.calls = 0
|
||||
rmControl.failuresLeft = 2
|
||||
|
||||
@@ -216,7 +308,11 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('reports a marker delete that never succeeds, after backing off', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
rmControl.failuresLeft = Number.MAX_SAFE_INTEGER
|
||||
|
||||
const startedAt = Date.now()
|
||||
@@ -231,7 +327,11 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('never blocks startup revalidation on an undeletable marker', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
rmControl.failuresLeft = Number.MAX_SAFE_INTEGER
|
||||
|
||||
const startedAt = Date.now()
|
||||
@@ -261,7 +361,11 @@ describe('gpu-fallback-marker', () => {
|
||||
rmControl.failuresLeft = Number.MAX_SAFE_INTEGER
|
||||
|
||||
const startedAt = Date.now()
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 5, crashesInWindow: 3 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 5, crashesInWindow: 3, consented: false },
|
||||
environment
|
||||
)
|
||||
const elapsed = Date.now() - startedAt
|
||||
|
||||
expect(readGpuFallbackMarker(userDataPath)?.engagedAt).toBe(5)
|
||||
@@ -271,7 +375,11 @@ describe('gpu-fallback-marker', () => {
|
||||
})
|
||||
|
||||
it('can explicitly clear the marker', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 4 }, environment)
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 4, consented: false },
|
||||
environment
|
||||
)
|
||||
clearGpuFallbackMarker(userDataPath)
|
||||
expect(readGpuFallbackMarker(userDataPath)).toBeNull()
|
||||
})
|
||||
|
||||
@@ -16,7 +16,8 @@ import {
|
||||
*/
|
||||
|
||||
export const GPU_FALLBACK_MARKER_FILE = 'gpu-fallback.json'
|
||||
export const GPU_FALLBACK_SCHEME_VERSION = 2
|
||||
// Bumped to 3 for `consented`, which self-invalidates markers written without it.
|
||||
export const GPU_FALLBACK_SCHEME_VERSION = 3
|
||||
|
||||
export type GpuFallbackEnvironment = {
|
||||
appVersion: string
|
||||
@@ -30,6 +31,8 @@ export type GpuFallbackMarker = {
|
||||
schemeVersion: number
|
||||
engagedAt: number
|
||||
crashesInWindow: number
|
||||
/** False while the marker is only a pre-prompt latch; true once the user chose to restart. */
|
||||
consented: boolean
|
||||
appVersion: string
|
||||
electronVersion: string
|
||||
platform: 'win32'
|
||||
@@ -39,6 +42,11 @@ function markerPath(userDataPath: string): string {
|
||||
return join(userDataPath, GPU_FALLBACK_MARKER_FILE)
|
||||
}
|
||||
|
||||
/** Why exported: the startup temp sweep is gated on a fallback file having existed. */
|
||||
export function gpuFallbackMarkerFileExists(userDataPath: string): boolean {
|
||||
return existsSync(markerPath(userDataPath))
|
||||
}
|
||||
|
||||
export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker | null {
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(markerPath(userDataPath), 'utf-8')) as Partial<
|
||||
@@ -52,6 +60,7 @@ export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker |
|
||||
!Number.isFinite(parsed.engagedAt) ||
|
||||
typeof parsed.crashesInWindow !== 'number' ||
|
||||
!Number.isFinite(parsed.crashesInWindow) ||
|
||||
typeof parsed.consented !== 'boolean' ||
|
||||
typeof parsed.appVersion !== 'string' ||
|
||||
typeof parsed.electronVersion !== 'string' ||
|
||||
parsed.platform !== 'win32'
|
||||
@@ -62,6 +71,7 @@ export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker |
|
||||
schemeVersion: GPU_FALLBACK_SCHEME_VERSION,
|
||||
engagedAt: parsed.engagedAt,
|
||||
crashesInWindow: parsed.crashesInWindow,
|
||||
consented: parsed.consented,
|
||||
appVersion: parsed.appVersion,
|
||||
electronVersion: parsed.electronVersion,
|
||||
platform: parsed.platform
|
||||
@@ -74,13 +84,14 @@ export function readGpuFallbackMarker(userDataPath: string): GpuFallbackMarker |
|
||||
|
||||
export function writeGpuFallbackMarker(
|
||||
userDataPath: string,
|
||||
info: { engagedAt: number; crashesInWindow: number },
|
||||
info: { engagedAt: number; crashesInWindow: number; consented: boolean },
|
||||
environment: WindowsGpuFallbackEnvironment
|
||||
): void {
|
||||
const marker: GpuFallbackMarker = {
|
||||
schemeVersion: GPU_FALLBACK_SCHEME_VERSION,
|
||||
engagedAt: info.engagedAt,
|
||||
crashesInWindow: info.crashesInWindow,
|
||||
consented: info.consented,
|
||||
appVersion: environment.appVersion,
|
||||
electronVersion: environment.electronVersion,
|
||||
platform: 'win32'
|
||||
|
||||
@@ -67,6 +67,7 @@
|
||||
"exploreOrca": "Explore Orca",
|
||||
"gettingStarted": "Getting Started with Orca",
|
||||
"reportCrash": "Report Crash...",
|
||||
"turnOffSafeGraphics": "Turn Off Safe Graphics Mode",
|
||||
"file": "File",
|
||||
"exit": "Exit",
|
||||
"edit": "Edit",
|
||||
|
||||
Reference in New Issue
Block a user