diff --git a/src/main/crash-reporting/process-gone-recorder.test.ts b/src/main/crash-reporting/process-gone-recorder.test.ts index 0226186fc90..eec093c7ddc 100644 --- a/src/main/crash-reporting/process-gone-recorder.test.ts +++ b/src/main/crash-reporting/process-gone-recorder.test.ts @@ -9,7 +9,11 @@ vi.mock('electron', () => ({ import { clearCrashBreadcrumbsForTest, getCrashBreadcrumbSnapshot } from './crash-breadcrumb-store' import { ProcessGoneDedupe } from './process-gone-dedupe' -import { recordProcessGoneCrash, type ProcessGoneCrashEvent } from './process-gone-recorder' +import { + recordProcessGoneCrash, + resetGpuFallbackCrashReportBudgetForTesting, + type ProcessGoneCrashEvent +} from './process-gone-recorder' import { _resetTracerForTests, setActiveSink, type TracerSink } from '../observability/tracer' type CapturingSink = TracerSink & { records: unknown[]; flushMock: ReturnType } @@ -44,6 +48,7 @@ beforeEach(() => { sink = capturingSink() setActiveSink(sink) clearCrashBreadcrumbsForTest() + resetGpuFallbackCrashReportBudgetForTesting() }) afterEach(() => { @@ -206,6 +211,129 @@ describe('recordProcessGoneCrash', () => { ) }) + // Why: a GPU crash loop under the fallback must not rewrite crash-reports.json + // every dedupe window for the rest of the session. + it('caps GPU-under-fallback reports per launch and durably records the overflow', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + const dedupe = new ProcessGoneDedupe() + const gpuCrash = (exitCode: number): ProcessGoneCrashEvent => + event({ + source: 'child', + processType: 'GPU', + reason: 'crashed', + exitCode, + gpuFallbackActive: true, + details: { type: 'GPU' } + }) + + for (const exitCode of [1, 2, 3]) { + recordProcessGoneCrash({ record } as never, gpuCrash(exitCode), dedupe) + } + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(3)) + + recordProcessGoneCrash({ record } as never, gpuCrash(4), dedupe) + + expect(record).toHaveBeenCalledTimes(3) + expect(getCrashBreadcrumbSnapshot()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'process_gone_suppressed', + data: expect.objectContaining({ suppressedBy: 'gpu_fallback_report_budget' }) + }) + ]) + ) + }) + + it('does not spend GPU-fallback budget on deduped repeats', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + const dedupe = new ProcessGoneDedupe() + const gpuCrash = (exitCode: number): ProcessGoneCrashEvent => + event({ + source: 'child', + processType: 'GPU', + reason: 'crashed', + exitCode, + gpuFallbackActive: true, + details: { type: 'GPU' } + }) + + recordProcessGoneCrash({ record } as never, gpuCrash(1), dedupe) + recordProcessGoneCrash({ record } as never, gpuCrash(2), dedupe) + recordProcessGoneCrash({ record } as never, gpuCrash(2), dedupe) + recordProcessGoneCrash({ record } as never, gpuCrash(3), dedupe) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(3)) + expect(getCrashBreadcrumbSnapshot()).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ suppressedBy: 'gpu_fallback_report_budget' }) + }) + ]) + ) + }) + + it('releases GPU-fallback budget when persistence fails', async () => { + const record = vi + .fn() + .mockRejectedValueOnce(new Error('disk unavailable')) + .mockResolvedValue({ id: 'report' }) + const dedupe = new ProcessGoneDedupe() + vi.spyOn(console, 'error').mockImplementation(() => {}) + const gpuCrash = (exitCode: number): ProcessGoneCrashEvent => + event({ + source: 'child', + processType: 'GPU', + reason: 'crashed', + exitCode, + gpuFallbackActive: true, + details: { type: 'GPU' } + }) + + recordProcessGoneCrash({ record } as never, gpuCrash(1), dedupe) + await vi.waitFor(() => + expect(getCrashBreadcrumbSnapshot()).toEqual( + expect.arrayContaining([expect.objectContaining({ name: 'crash_report_persist_failed' })]) + ) + ) + + for (const exitCode of [2, 3, 4]) { + recordProcessGoneCrash({ record } as never, gpuCrash(exitCode), dedupe) + } + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(4)) + expect(getCrashBreadcrumbSnapshot()).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ suppressedBy: 'gpu_fallback_report_budget' }) + }) + ]) + ) + }) + + it('still records renderer crashes after the GPU-fallback budget is spent', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + const dedupe = new ProcessGoneDedupe() + + for (const exitCode of [1, 2, 3]) { + recordProcessGoneCrash( + { record } as never, + event({ + source: 'child', + processType: 'GPU', + reason: 'crashed', + exitCode, + gpuFallbackActive: true, + details: { type: 'GPU' } + }), + dedupe + ) + } + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(3)) + + recordProcessGoneCrash({ record } as never, event({ gpuFallbackActive: true }), dedupe) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(4)) + }) + it('allows the same renderer crash to retry after persistence fails', async () => { const record = vi .fn() diff --git a/src/main/crash-reporting/process-gone-recorder.ts b/src/main/crash-reporting/process-gone-recorder.ts index b6a8abd8c9e..557c8afa15b 100644 --- a/src/main/crash-reporting/process-gone-recorder.ts +++ b/src/main/crash-reporting/process-gone-recorder.ts @@ -35,6 +35,24 @@ export type ProcessGoneCrashEvent = { type CrashReportRecorderStore = Pick +// Why: a GPU crash loop under the fallback would rewrite crash-reports.json every +// dedupe window for the rest of the session; the first few reports carry all the +// triage signal (driver identity, tier), the rest are pure disk churn. +const MAX_GPU_FALLBACK_CRASH_REPORTS_PER_LAUNCH = 3 +let gpuFallbackCrashReportsThisLaunch = 0 + +export function resetGpuFallbackCrashReportBudgetForTesting(): void { + gpuFallbackCrashReportsThisLaunch = 0 +} + +function countsAgainstGpuFallbackReportBudget(event: ProcessGoneCrashEvent): boolean { + return ( + event.gpuFallbackActive === true && + event.source === 'child' && + event.processType.toLowerCase() === 'gpu' + ) +} + function processGoneBreadcrumbData(event: ProcessGoneCrashEvent) { return buildSuppressedProcessGoneBreadcrumbData(event) } @@ -84,11 +102,26 @@ export function recordProcessGoneCrash( return } + const gpuFallbackBudgeted = countsAgainstGpuFallbackReportBudget(event) + if ( + gpuFallbackBudgeted && + gpuFallbackCrashReportsThisLaunch >= MAX_GPU_FALLBACK_CRASH_REPORTS_PER_LAUNCH + ) { + recordDurableCrashBreadcrumb('process_gone_suppressed', { + ...processGoneBreadcrumbData(event), + suppressedBy: 'gpu_fallback_report_budget' + }) + return + } + const key = getProcessGoneDedupeKey(event.source, event.processType, event.reason, event.exitCode) const claim = dedupe.tryClaim(key) if (!claim) { return } + if (gpuFallbackBudgeted) { + gpuFallbackCrashReportsThisLaunch += 1 + } const mainProcessLifecycle = getMainProcessLifecycleIdentity() // Why: GPU and renderer deaths are the ones triage needs driver identity for; // every other child type would just pad the report. @@ -145,6 +178,9 @@ export function recordProcessGoneCrash( }) .catch((error) => { dedupe.release(claim) + if (gpuFallbackBudgeted) { + gpuFallbackCrashReportsThisLaunch -= 1 + } console.error('[crash-reporting] Failed to persist crash report:', error) const data = persistFailureData(event, error) recordDurableCrashBreadcrumb( diff --git a/src/main/index.ts b/src/main/index.ts index 5a7e5be27a6..0d450f31c33 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -100,7 +100,7 @@ import { import { NO_GPU_FALLBACK_TIER, getGpuFallbackTierSwitches, - getNextGpuFallbackTier, + resolveGpuFallbackEscalation, type GpuFallbackTier } from './startup/gpu-fallback-tiers' import { purgeGpuCaches } from './startup/gpu-cache-purge' @@ -1443,9 +1443,9 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void { // Why: after an update the build-scoped marker is gone but the machine is not. // Resuming at the tier it already needed is the difference between one relaunch // per update and re-walking the whole ladder every time. - const resumeTier = - currentTier === NO_GPU_FALLBACK_TIER ? getResumeGpuFallbackTier(userDataPath) : null - const nextTier = resumeTier ?? getNextGpuFallbackTier(currentTier) + const { nextTier, resumedFromHistory } = resolveGpuFallbackEscalation(currentTier, () => + getResumeGpuFallbackTier(userDataPath) + ) if (currentTier !== NO_GPU_FALLBACK_TIER) { // Why: 20 of 21 crashed launches on the reported machine already carried // gpu_fallback_applied. Without this breadcrumb the failure looked like a fix. @@ -1473,7 +1473,7 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void { exitCode, tier: nextTier, previousTier: currentTier, - resumedFromHistory: resumeTier !== null, + resumedFromHistory, crashesInWindow: result.crashesInWindow }) const engagedAt = Date.now() @@ -1869,11 +1869,15 @@ app.whenReady().then(async () => { logStartupMilestone('app-ready') // Why: driver identity for crash details. Fire-and-forget and bounded — on the // machines this exists for, the GPU process never initializes and this never resolves. - void captureGpuInfoSnapshot(() => app.getGPUInfo('complete'), GPU_INFO_CAPTURE_TIMEOUT_MS).then( - (snapshot) => { - recordCrashBreadcrumb('gpu_info_captured', snapshot) - } - ) + // Skipped in headless serve: no crash dialog to feed, and 'complete' can provoke a + // GPU child that Xvfb environments have no reason to spawn. + if (!isServeMode) { + void captureGpuInfoSnapshot(() => app.getGPUInfo('complete'), GPU_INFO_CAPTURE_TIMEOUT_MS).then( + (snapshot) => { + recordCrashBreadcrumb('gpu_info_captured', snapshot) + } + ) + } // Why: install certificate decisions before any webview or headless window issues its first TLS request. app.on( 'certificate-error', diff --git a/src/main/startup/gpu-fallback-tiers.test.ts b/src/main/startup/gpu-fallback-tiers.test.ts index 78cb5da15a6..82254047994 100644 --- a/src/main/startup/gpu-fallback-tiers.test.ts +++ b/src/main/startup/gpu-fallback-tiers.test.ts @@ -4,7 +4,8 @@ import { clampGpuFallbackTier, getGpuFallbackTierSwitches, getNextGpuFallbackTier, - isGpuFallbackTier + isGpuFallbackTier, + resolveGpuFallbackEscalation } from './gpu-fallback-tiers' describe('gpu-fallback-tiers', () => { @@ -63,3 +64,42 @@ describe('gpu-fallback-tiers', () => { } }) }) + +describe('resolveGpuFallbackEscalation', () => { + it('starts a machine with no history at the bottom of the ladder', () => { + expect(resolveGpuFallbackEscalation(0, () => null)).toEqual({ + nextTier: 1, + resumedFromHistory: false + }) + }) + + // Why: this is what turns "unusable after every update" into one relaunch per update. + it('resumes a hardware launch at the tier the machine already needed', () => { + expect(resolveGpuFallbackEscalation(0, () => 2)).toEqual({ + nextTier: 2, + resumedFromHistory: true + }) + expect(resolveGpuFallbackEscalation(0, () => 1)).toEqual({ + nextTier: 1, + resumedFromHistory: true + }) + }) + + it('ignores history once a tier is applied and escalates one rung', () => { + let historyReads = 0 + expect( + resolveGpuFallbackEscalation(1, () => { + historyReads += 1 + return 2 + }) + ).toEqual({ nextTier: 2, resumedFromHistory: false }) + expect(historyReads).toBe(0) + }) + + it('reports an exhausted ladder at the top rung', () => { + expect(resolveGpuFallbackEscalation(2, () => 2)).toEqual({ + nextTier: null, + resumedFromHistory: false + }) + }) +}) diff --git a/src/main/startup/gpu-fallback-tiers.ts b/src/main/startup/gpu-fallback-tiers.ts index 847eba028db..62c8670f99a 100644 --- a/src/main/startup/gpu-fallback-tiers.ts +++ b/src/main/startup/gpu-fallback-tiers.ts @@ -72,3 +72,25 @@ export function getNextGpuFallbackTier(currentTier: number): GpuFallbackTier | n const next = Math.trunc(currentTier) + 1 return isGpuFallbackTier(next) ? next : null } + +export type GpuFallbackEscalation = { + nextTier: GpuFallbackTier | null + resumedFromHistory: boolean +} + +/** + * Tier for the relaunch after a crash burst. History (the strongest tier this + * machine ever needed) is consulted only when no tier is applied this launch — + * i.e. the post-update hardware probe just failed. Once on the ladder, + * escalation is strictly one rung at a time. + */ +export function resolveGpuFallbackEscalation( + currentTier: number, + readRequiredTierHistory: () => GpuFallbackTier | null +): GpuFallbackEscalation { + const resumeTier = currentTier === NO_GPU_FALLBACK_TIER ? readRequiredTierHistory() : null + return { + nextTier: resumeTier ?? getNextGpuFallbackTier(currentTier), + resumedFromHistory: resumeTier !== null + } +}