From fb6c2800ee46fbd9eb5341d475c20a791a4562df Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sat, 29 Aug 2026 02:41:53 -0700 Subject: [PATCH] fix(gpu): capture hardware identity in crash reports (#16973) * fix(gpu): capture hardware identity in crash reports * fix(gpu): order bounded crash diagnostics before fallback * fix(gpu): keep fallback persistence ahead of diagnostics --- .../gpu-crash-diagnostics.test.ts | 241 ++++++++++++++++++ .../crash-reporting/gpu-crash-diagnostics.ts | 222 ++++++++++++++++ .../gpu-crash-fallback-field-sessions.test.ts | 6 +- src/main/index.ts | 24 +- 4 files changed, 488 insertions(+), 5 deletions(-) create mode 100644 src/main/crash-reporting/gpu-crash-diagnostics.test.ts create mode 100644 src/main/crash-reporting/gpu-crash-diagnostics.ts diff --git a/src/main/crash-reporting/gpu-crash-diagnostics.test.ts b/src/main/crash-reporting/gpu-crash-diagnostics.test.ts new file mode 100644 index 00000000000..0cb6c02cf3a --- /dev/null +++ b/src/main/crash-reporting/gpu-crash-diagnostics.test.ts @@ -0,0 +1,241 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { buildGpuCrashDiagnostics, GpuCrashDiagnosticsRecorder } from './gpu-crash-diagnostics' + +const FEATURE_STATUS = { + gpu_compositing: 'enabled', + rasterization: 'enabled', + webgl: 'enabled', + webgl2: 'enabled', + video_decode: 'enabled' +} + +const BASIC_INFO = { + gpuDevice: [ + { + active: false, + vendorId: 0x8086, + deviceId: 0x9a49, + vendorString: 'Intel', + deviceString: 'Integrated GPU' + }, + { + active: true, + vendorId: 0x10de, + deviceId: 0x2684, + vendorString: 'NVIDIA', + deviceString: 'Discrete GPU', + driverVendor: 'NVIDIA', + driverVersion: '32.0.15.6094' + } + ], + auxAttributes: { + glVendor: 'Google Inc.', + glRenderer: 'ANGLE (NVIDIA, D3D11)', + glVersion: 'OpenGL ES 3.0' + } +} + +function deferred(): { + promise: Promise + resolve: (value: T) => void +} { + let resolvePromise: ((value: T) => void) | undefined + const promise = new Promise((resolve) => { + resolvePromise = resolve + }) + return { + promise, + resolve: (value) => resolvePromise?.(value) + } +} + +describe('buildGpuCrashDiagnostics', () => { + it('keeps only the active GPU identity, driver, rendering backend, and feature status', () => { + expect( + buildGpuCrashDiagnostics({ info: BASIC_INFO, level: 'complete' }, FEATURE_STATUS) + ).toEqual({ + gpuInfoLevel: 'complete', + gpuDeviceCount: 2, + gpuVendorId: 0x10de, + gpuDeviceId: 0x2684, + gpuVendor: 'NVIDIA', + gpuDevice: 'Discrete GPU', + gpuDriverVendor: 'NVIDIA', + gpuDriverVersion: '32.0.15.6094', + gpuGlVendor: 'Google Inc.', + gpuGlRenderer: 'ANGLE (NVIDIA, D3D11)', + gpuGlVersion: 'OpenGL ES 3.0', + gpuCompositingStatus: 'enabled', + gpuRasterizationStatus: 'enabled', + gpuWebglStatus: 'enabled', + gpuWebgl2Status: 'enabled', + gpuVideoDecodeStatus: 'enabled' + }) + }) + + it('degrades malformed or unavailable GPU info without copying arbitrary fields', () => { + expect( + buildGpuCrashDiagnostics( + { + level: 'basic', + info: { + gpuDevice: [{ active: true, vendorId: Number.NaN, secret: 'do not copy' }], + machineModelName: 'do not copy' + } + }, + { webgl: 'unavailable', unexpected: 'do not copy' } + ) + ).toEqual({ + gpuInfoLevel: 'basic', + gpuDeviceCount: 1, + gpuWebglStatus: 'unavailable' + }) + expect(buildGpuCrashDiagnostics(null, null)).toEqual({ gpuInfoLevel: 'unavailable' }) + }) +}) + +describe('GpuCrashDiagnosticsRecorder', () => { + it('warms only complete info and records one breadcrumb across a crash burst', async () => { + const recordBreadcrumb = vi.fn() + const provider = { + getGPUInfo: vi.fn(async (level: 'basic' | 'complete') => ({ + ...BASIC_INFO, + gpuDevice: BASIC_INFO.gpuDevice.map((device) => ({ + ...device, + ...(level === 'complete' ? { driverVersion: 'complete-driver' } : {}) + })) + })), + getGPUFeatureStatus: vi.fn(() => FEATURE_STATUS) + } + const recorder = new GpuCrashDiagnosticsRecorder({ provider, recordBreadcrumb }) + + recorder.warm() + await vi.waitFor(() => { + expect(provider.getGPUInfo).toHaveBeenCalledTimes(1) + }) + await Promise.resolve() + await recorder.record() + await recorder.record() + + expect(provider.getGPUInfo).toHaveBeenCalledOnce() + expect(provider.getGPUInfo).toHaveBeenCalledWith('complete') + + expect(recordBreadcrumb).toHaveBeenCalledTimes(1) + expect(recordBreadcrumb).toHaveBeenCalledWith( + expect.objectContaining({ + gpuInfoLevel: 'complete', + gpuVendorId: 0x10de, + gpuDeviceId: 0x2684, + gpuDriverVersion: 'complete-driver' + }) + ) + }) + + it('uses promptly available basic info when complete collection is still pending', async () => { + const complete = deferred() + const recordBreadcrumb = vi.fn() + const provider = { + getGPUInfo: vi.fn((level: 'basic' | 'complete') => + level === 'basic' ? Promise.resolve(BASIC_INFO) : complete.promise + ), + getGPUFeatureStatus: vi.fn(() => FEATURE_STATUS) + } + const recorder = new GpuCrashDiagnosticsRecorder({ provider, recordBreadcrumb }) + + recorder.warm() + expect(provider.getGPUInfo).toHaveBeenCalledOnce() + expect(provider.getGPUInfo).toHaveBeenCalledWith('complete') + await recorder.record() + + expect(recordBreadcrumb).toHaveBeenCalledWith( + expect.objectContaining({ + gpuInfoLevel: 'basic', + gpuVendorId: 0x10de, + gpuDriverVersion: '32.0.15.6094' + }) + ) + complete.resolve(BASIC_INFO) + }) + + it('shares pending capture work and releases it when complete info arrives first', async () => { + const complete = deferred() + const basic = deferred() + const recordBreadcrumb = vi.fn() + const provider = { + getGPUInfo: vi.fn((level: 'basic' | 'complete') => + level === 'basic' ? basic.promise : complete.promise + ), + getGPUFeatureStatus: vi.fn(() => FEATURE_STATUS) + } + const recorder = new GpuCrashDiagnosticsRecorder({ provider, recordBreadcrumb }) + + recorder.warm() + const first = recorder.record() + const second = recorder.record() + + expect(second).toBe(first) + expect(recordBreadcrumb).not.toHaveBeenCalled() + complete.resolve(BASIC_INFO) + await first + expect(recordBreadcrumb).toHaveBeenCalledOnce() + expect(recordBreadcrumb).toHaveBeenCalledWith( + expect.objectContaining({ gpuInfoLevel: 'complete' }) + ) + }) + + it('does not let stalled GPU info block crash recovery', async () => { + const never = Promise.withResolvers().promise + const recordBreadcrumb = vi.fn() + const provider = { + getGPUInfo: vi.fn(() => never), + getGPUFeatureStatus: vi.fn(() => FEATURE_STATUS) + } + const recorder = new GpuCrashDiagnosticsRecorder({ + provider, + recordBreadcrumb, + recordTimeoutMs: 0 + }) + + await recorder.record() + + expect(recordBreadcrumb).toHaveBeenCalledWith( + expect.objectContaining({ gpuInfoLevel: 'unavailable' }) + ) + }) + + it('still records collection status when Electron throws during both GPU info calls', async () => { + const recordBreadcrumb = vi.fn() + const provider = { + getGPUInfo: vi.fn(() => { + throw new Error('GPU access disabled') + }), + getGPUFeatureStatus: vi.fn(() => { + throw new Error('GPU teardown') + }) + } + const recorder = new GpuCrashDiagnosticsRecorder({ provider, recordBreadcrumb }) + + recorder.warm() + await recorder.record() + + expect(recordBreadcrumb).toHaveBeenCalledWith({ gpuInfoLevel: 'unavailable' }) + }) +}) + +describe('GPU crash diagnostics production wiring', () => { + it('starts diagnostics without delaying safe-graphics fallback', () => { + const source = readFileSync(join(__dirname, '..', 'index.ts'), 'utf8') + const listenerStart = source.indexOf("app.on('child-process-gone'") + expect(listenerStart).toBeGreaterThan(0) + const listener = source.slice(listenerStart, source.indexOf('\n })', listenerStart)) + expect(source).toMatch( + /recordBreadcrumb: \(data\) =>\s*recordDurableCrashBreadcrumb\('gpu_crash_hardware', data\)/ + ) + expect(listener).toMatch( + /const crashedAt = performance\.now\(\)[\s\S]*?void gpuCrashDiagnostics\?\.record\(\)[\s\S]*?void handleGpuChildCrash\(details\.reason, details\.exitCode \?\? null, crashedAt\)/ + ) + expect(listener).not.toMatch(/gpuCrashDiagnostics\?\.record\(\)[\s\S]*?\.then\(/) + }) +}) diff --git a/src/main/crash-reporting/gpu-crash-diagnostics.ts b/src/main/crash-reporting/gpu-crash-diagnostics.ts new file mode 100644 index 00000000000..055ec2f1b29 --- /dev/null +++ b/src/main/crash-reporting/gpu-crash-diagnostics.ts @@ -0,0 +1,222 @@ +import type { CrashReportBreadcrumbData } from '../../shared/crash-reporting' + +type GpuInfoLevel = 'basic' | 'complete' +const DEFAULT_GPU_CRASH_DIAGNOSTICS_WAIT_MS = 1_000 + +type GpuInfoProvider = { + getGPUInfo(infoType: GpuInfoLevel): Promise + getGPUFeatureStatus(): unknown +} + +type GpuCrashDiagnosticsRecorderOptions = { + provider: GpuInfoProvider + recordBreadcrumb: (data: CrashReportBreadcrumbData) => void + recordTimeoutMs?: number +} + +type GpuInfoSnapshot = { + info: unknown + level: GpuInfoLevel +} + +async function waitAtMost(promise: Promise, timeoutMs: number): Promise { + const timeoutGate = Promise.withResolvers() + const timeout = setTimeout(timeoutGate.resolve, timeoutMs) + try { + await Promise.race([promise, timeoutGate.promise]) + } finally { + clearTimeout(timeout) + } +} + +function waitForFirstAvailable(promises: Promise[]): Promise { + const availableGate = Promise.withResolvers() + let remaining = promises.length + for (const promise of promises) { + void promise.then((available) => { + remaining -= 1 + if (available || remaining === 0) { + availableGate.resolve() + } + }) + } + return availableGate.promise +} + +function recordValue(value: unknown): Record | null { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : null +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +function addString(target: CrashReportBreadcrumbData, key: string, value: unknown): void { + const safe = nonEmptyString(value) + if (safe !== undefined) { + target[key] = safe + } +} + +function addNumber(target: CrashReportBreadcrumbData, key: string, value: unknown): void { + const safe = finiteNumber(value) + if (safe !== undefined) { + target[key] = safe + } +} + +function activeGpuDevice(info: Record): { + device: Record | null + count: number +} { + const devices = Array.isArray(info.gpuDevice) + ? info.gpuDevice.map(recordValue).filter((device) => device !== null) + : [] + return { + device: devices.find((device) => device.active === true) ?? devices[0] ?? null, + count: devices.length + } +} + +function addFeatureStatuses(details: CrashReportBreadcrumbData, featureStatus: unknown): void { + const status = recordValue(featureStatus) + if (!status) { + return + } + addString(details, 'gpuCompositingStatus', status.gpu_compositing) + addString(details, 'gpuRasterizationStatus', status.rasterization) + addString(details, 'gpuWebglStatus', status.webgl) + addString(details, 'gpuWebgl2Status', status.webgl2) + addString(details, 'gpuVideoDecodeStatus', status.video_decode) +} + +export function buildGpuCrashDiagnostics( + snapshot: GpuInfoSnapshot | null, + featureStatus: unknown +): CrashReportBreadcrumbData { + const details: CrashReportBreadcrumbData = { + gpuInfoLevel: snapshot?.level ?? 'unavailable' + } + addFeatureStatuses(details, featureStatus) + const info = recordValue(snapshot?.info) + if (!info) { + return details + } + + const { device, count } = activeGpuDevice(info) + details.gpuDeviceCount = count + if (device) { + addNumber(details, 'gpuVendorId', device.vendorId) + addNumber(details, 'gpuDeviceId', device.deviceId) + addString(details, 'gpuVendor', device.vendorString) + addString(details, 'gpuDevice', device.deviceString) + addString(details, 'gpuDriverVendor', device.driverVendor) + addString(details, 'gpuDriverVersion', device.driverVersion) + } + + const aux = recordValue(info.auxAttributes) + if (aux) { + addString(details, 'gpuGlVendor', aux.glVendor) + addString(details, 'gpuGlRenderer', aux.glRenderer) + addString(details, 'gpuGlVersion', aux.glVersion) + } + return details +} + +/** Captures GPU identity before a crash and emits it once when a Windows GPU burst starts. */ +export class GpuCrashDiagnosticsRecorder { + private readonly provider: GpuInfoProvider + private readonly recordBreadcrumb: (data: CrashReportBreadcrumbData) => void + private readonly recordTimeoutMs: number + private basicInfoPromise: Promise | null = null + private completeInfoPromise: Promise | null = null + private recordingPromise: Promise | null = null + private basicInfo: unknown = null + private completeInfo: unknown = null + + constructor(options: GpuCrashDiagnosticsRecorderOptions) { + this.provider = options.provider + this.recordBreadcrumb = options.recordBreadcrumb + this.recordTimeoutMs = options.recordTimeoutMs ?? DEFAULT_GPU_CRASH_DIAGNOSTICS_WAIT_MS + } + + warm(): void { + void this.ensureCompleteInfo() + } + + record(): Promise { + this.recordingPromise ??= this.recordOnce() + return this.recordingPromise + } + + private async recordOnce(): Promise { + let featureStatus: unknown = null + try { + featureStatus = this.provider.getGPUFeatureStatus() + } catch { + // GPU teardown can race this read; device identity is still useful. + } + if (this.completeInfo === null) { + await waitAtMost( + waitForFirstAvailable([this.ensureBasicInfo(), this.ensureCompleteInfo()]), + this.recordTimeoutMs + ) + } + const snapshot = this.preferredSnapshot() + try { + this.recordBreadcrumb(buildGpuCrashDiagnostics(snapshot, featureStatus)) + } catch { + // Diagnostics must never block safe-graphics recovery. + } + } + + private ensureBasicInfo(): Promise { + if (this.basicInfoPromise === null) { + try { + this.basicInfoPromise = this.provider.getGPUInfo('basic').then( + (info) => { + this.basicInfo = info + return true + }, + () => false + ) + } catch { + this.basicInfoPromise = Promise.resolve(false) + } + } + return this.basicInfoPromise + } + + private ensureCompleteInfo(): Promise { + if (this.completeInfoPromise === null) { + try { + this.completeInfoPromise = this.provider.getGPUInfo('complete').then( + (info) => { + this.completeInfo = info + return true + }, + () => false + ) + } catch { + this.completeInfoPromise = Promise.resolve(false) + } + } + return this.completeInfoPromise + } + + private preferredSnapshot(): GpuInfoSnapshot | null { + if (this.completeInfo !== null) { + return { info: this.completeInfo, level: 'complete' } + } + if (this.basicInfo !== null) { + return { info: this.basicInfo, level: 'basic' } + } + return null + } +} diff --git a/src/main/crash-reporting/gpu-crash-fallback-field-sessions.test.ts b/src/main/crash-reporting/gpu-crash-fallback-field-sessions.test.ts index b0112b7175e..3df9849505d 100644 --- a/src/main/crash-reporting/gpu-crash-fallback-field-sessions.test.ts +++ b/src/main/crash-reporting/gpu-crash-fallback-field-sessions.test.ts @@ -82,12 +82,14 @@ describe('1.4.190 win32 GPU-child crash cluster', () => { const guardStart = listener.indexOf('isGpuFallbackCrashCandidate(') expect(guardStart).toBeGreaterThan(0) expect(listener.slice(0, guardStart).match(/\bif\s*\(/g) ?? []).toHaveLength(1) - expect(listener).toMatch(/isGpuFallbackCrashCandidate\([\s\S]*?void handleGpuChildCrash\(/) + expect(listener).toMatch( + /isGpuFallbackCrashCandidate\([\s\S]*?gpuCrashDiagnostics\?\.record\(\)[\s\S]*?handleGpuChildCrash\(/ + ) // The `if (` count alone still allows `recorded && isGpuFallbackCrashCandidate(...)`, which // re-couples recovery to the suppression decision, so pin the guard to that check alone. const recoveryGuard = listener.slice( listener.lastIndexOf('if (', guardStart), - listener.indexOf('void handleGpuChildCrash(') + listener.indexOf('handleGpuChildCrash(') ) expect(recoveryGuard).not.toMatch(/&&|\|\|/) }) diff --git a/src/main/index.ts b/src/main/index.ts index 69cb4d33c78..7df14b894a1 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -187,6 +187,7 @@ import { } from './crash-reporting/gpu-crash-fallback-decision' import { promptForGpuFallbackRestart } from './crash-reporting/gpu-fallback-restart-prompt' import { engageGpuFallbackAfterCrashBurst } from './crash-reporting/gpu-fallback-engagement' +import { GpuCrashDiagnosticsRecorder } from './crash-reporting/gpu-crash-diagnostics' import { handleGpuFallbackRecoveredLaunch, promptForGpuFallbackRecoveredLaunch @@ -470,6 +471,16 @@ const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({ let activeGpuFallbackMarker: GpuFallbackMarker | null = null let gpuFallbackActiveThisLaunch = false let gpuFeatureStatus: Electron.GPUFeatureStatus | null = null +const gpuCrashDiagnostics = + process.platform === 'win32' + ? new GpuCrashDiagnosticsRecorder({ + provider: { + getGPUInfo: (infoType) => app.getGPUInfo(infoType), + getGPUFeatureStatus: () => app.getGPUFeatureStatus() + }, + recordBreadcrumb: (data) => recordDurableCrashBreadcrumb('gpu_crash_hardware', data) + }) + : null let localPtyStartupReady: Promise = Promise.resolve() let localPtyProviderStartupReady: Promise = Promise.resolve() const AGENT_STATE_CRASH_BREADCRUMB_MIN_INTERVAL_MS = 30_000 @@ -556,6 +567,7 @@ function updateGpuAccelerationAboutPanel(): void { app.on('gpu-info-update', () => { gpuFeatureStatus = app.getGPUFeatureStatus() + gpuCrashDiagnostics?.warm() if (app.isReady()) { updateGpuAccelerationAboutPanel() } @@ -1945,12 +1957,16 @@ async function presentGpuFallbackRecoveredLaunchPrompt(window: BrowserWindow): P } // Why: a burst of GPU child crashes means HW acceleration is unusable — persist a build-scoped marker and offer software rendering. -async function handleGpuChildCrash(reason: string, exitCode: number | null): Promise { +async function handleGpuChildCrash( + reason: string, + exitCode: number | null, + crashedAt: number +): Promise { // Software rendering already active or shutting down: nothing more to do. if (gpuFallbackActiveThisLaunch || isQuitting || isServeMode) { return } - const result = gpuCrashFallbackTracker.recordGpuCrash(performance.now()) + const result = gpuCrashFallbackTracker.recordGpuCrash(crashedAt) if (!result.shouldEngageFallback) { return } @@ -3219,7 +3235,9 @@ void app.whenReady().then(async () => { reason: details.reason }) ) { - void handleGpuChildCrash(details.reason, details.exitCode ?? null) + const crashedAt = performance.now() + void gpuCrashDiagnostics?.record() + void handleGpuChildCrash(details.reason, details.exitCode ?? null, crashedAt) } })