From 81dfa1db557c3ce65d4370774a6ab080cfcb4eea Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sat, 25 Jul 2026 12:15:00 -0700 Subject: [PATCH] feat(gpu): add the GPU fallback ladder, cache purge, and driver-identity modules `--disable-gpu` does not stop Chromium from spawning a GPU child for the Viz display compositor, so a driver that CHECK-crashes at GPU init keeps killing every launch even with the fallback applied. These are the pieces a real remediation needs, added standalone with no callers yet: - a bounded, additive tier ladder (disable-gpu -> +compositing/ANGLE swiftshader -> +in-process-gpu, which removes the crashing child entirely) - a GPU/shader cache purge, so a cache written by the crashing driver is not replayed into the tier we just escalated to - a summarizer for app.getGPUInfo('complete'); nothing in src/main called it, so every GPU crash bundle arrived with no vendor, device, or driver version Co-authored-by: Orca --- .../crash-reporting/gpu-info-snapshot.test.ts | 115 ++++++++++++++ src/main/crash-reporting/gpu-info-snapshot.ts | 143 ++++++++++++++++++ src/main/startup/gpu-cache-purge.test.ts | 50 ++++++ src/main/startup/gpu-cache-purge.ts | 43 ++++++ src/main/startup/gpu-fallback-tiers.test.ts | 64 ++++++++ src/main/startup/gpu-fallback-tiers.ts | 77 ++++++++++ 6 files changed, 492 insertions(+) create mode 100644 src/main/crash-reporting/gpu-info-snapshot.test.ts create mode 100644 src/main/crash-reporting/gpu-info-snapshot.ts create mode 100644 src/main/startup/gpu-cache-purge.test.ts create mode 100644 src/main/startup/gpu-cache-purge.ts create mode 100644 src/main/startup/gpu-fallback-tiers.test.ts create mode 100644 src/main/startup/gpu-fallback-tiers.ts diff --git a/src/main/crash-reporting/gpu-info-snapshot.test.ts b/src/main/crash-reporting/gpu-info-snapshot.test.ts new file mode 100644 index 00000000000..f8376caef7d --- /dev/null +++ b/src/main/crash-reporting/gpu-info-snapshot.test.ts @@ -0,0 +1,115 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + captureGpuInfoSnapshot, + getGpuInfoSnapshot, + setGpuInfoSnapshotForTesting, + summarizeGpuInfo +} from './gpu-info-snapshot' + +const COMPLETE_INFO = { + auxAttributes: { + glVendor: 'Google Inc. (Intel)', + glRenderer: 'ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)', + glVersion: 'OpenGL ES 2.0.0', + glImplementation: 'EGL_ANGLE', + glExtensions: 'GL_EXT_a '.repeat(500), + basicInfoState: 1, + contextInfoState: 1, + initializationTime: 42.5, + inProcessGpu: false, + sandboxed: true, + directComposition: true, + passthroughCmdDecoder: true + }, + gpuDevice: [ + { active: false, vendorId: 4318, deviceId: 7938, driverVendor: 'NVIDIA' }, + { + active: true, + vendorId: 32902, + deviceId: 15130, + driverVendor: 'Intel Corporation', + driverVersion: '31.0.101.2111', + deviceString: 'Intel(R) UHD Graphics 620' + } + ], + machineModelName: 'Latitude 5400', + machineModelVersion: '1.0' +} + +describe('summarizeGpuInfo', () => { + it('flattens the active device and aux attributes', () => { + const summary = summarizeGpuInfo(COMPLETE_INFO) + + expect(summary).toMatchObject({ + gpuInfoAvailable: true, + gpuDeviceCount: 2, + gpuVendorId: '0x8086', + gpuDeviceId: '0x3b1a', + gpuDriverVendor: 'Intel Corporation', + gpuDriverVersion: '31.0.101.2111', + gpuGlVendor: 'Google Inc. (Intel)', + gpuAnglePlatform: 'EGL_ANGLE', + gpuInProcess: false, + gpuSandboxed: true, + gpuBasicInfoState: 1, + gpuInitializationTimeMs: 42.5, + gpuMachineModel: 'Latitude 5400' + }) + }) + + // Why: glExtensions is kilobytes long and would dominate every crash report. + it('never copies the extension list and bounds long strings', () => { + const summary = summarizeGpuInfo(COMPLETE_INFO) + + expect(Object.keys(summary)).not.toContain('gpuGlExtensions') + for (const value of Object.values(summary)) { + if (typeof value === 'string') { + expect(value.length).toBeLessThanOrEqual(160) + } + } + }) + + it('falls back to the first device when none is marked active', () => { + const summary = summarizeGpuInfo({ gpuDevice: [{ vendorId: 4318, deviceId: 7938 }] }) + expect(summary).toMatchObject({ gpuVendorId: '0x10de', gpuDeviceCount: 1 }) + }) + + it('reports unavailable for non-object payloads', () => { + expect(summarizeGpuInfo(null)).toEqual({ gpuInfoAvailable: false }) + expect(summarizeGpuInfo('nope')).toEqual({ gpuInfoAvailable: false }) + }) + + it('omits fields the payload does not carry', () => { + const summary = summarizeGpuInfo({ gpuDevice: [] }) + expect(summary).toEqual({ gpuInfoAvailable: true, gpuDeviceCount: 0 }) + }) +}) + +describe('captureGpuInfoSnapshot', () => { + afterEach(() => { + setGpuInfoSnapshotForTesting(null) + vi.useRealTimers() + }) + + it('caches a successful capture', async () => { + const snapshot = await captureGpuInfoSnapshot(async () => COMPLETE_INFO, 1_000) + expect(snapshot.gpuVendorId).toBe('0x8086') + expect(getGpuInfoSnapshot()).toBe(snapshot) + }) + + it('records the error shape when getGPUInfo rejects', async () => { + const snapshot = await captureGpuInfoSnapshot(async () => { + throw new Error('GPU process crashed') + }, 1_000) + expect(snapshot).toMatchObject({ gpuInfoAvailable: false, gpuInfoError: 'GPU process crashed' }) + expect(getGpuInfoSnapshot()).toBe(snapshot) + }) + + // Why: on the machines this exists for, the GPU child never initializes and the promise never settles. + it('times out instead of hanging forever', async () => { + vi.useFakeTimers() + const pending = captureGpuInfoSnapshot(() => new Promise(() => {}), 10_000) + await vi.advanceTimersByTimeAsync(10_000) + await expect(pending).resolves.toEqual({ gpuInfoAvailable: false, gpuInfoError: 'timeout' }) + }) +}) diff --git a/src/main/crash-reporting/gpu-info-snapshot.ts b/src/main/crash-reporting/gpu-info-snapshot.ts new file mode 100644 index 00000000000..0f78fe155a8 --- /dev/null +++ b/src/main/crash-reporting/gpu-info-snapshot.ts @@ -0,0 +1,143 @@ +import { + sanitizeCrashReportDetails, + type CrashReportBreadcrumbData +} from '../../shared/crash-reporting' + +/** + * Driver identity for crash details. + * + * Nothing in the main process called app.getGPUInfo() before, so every GPU + * crash bundle arrived without a vendor, device or driver version — triage + * could not tell a broken driver from a broken machine. + */ + +/** Bounded so a pathological glExtensions-style string can't dominate the report. */ +const MAX_GPU_STRING_LENGTH = 160 + +type UnknownRecord = Record + +function asRecord(value: unknown): UnknownRecord | null { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as UnknownRecord) + : null +} + +function pickString(source: UnknownRecord | null, key: string): string | undefined { + const value = source?.[key] + return typeof value === 'string' && value.length > 0 + ? value.slice(0, MAX_GPU_STRING_LENGTH) + : undefined +} + +function pickNumber(source: UnknownRecord | null, key: string): number | undefined { + const value = source?.[key] + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +function pickBoolean(source: UnknownRecord | null, key: string): boolean | undefined { + const value = source?.[key] + return typeof value === 'boolean' ? value : undefined +} + +/** Vendor/device ids arrive as numbers; hex is what driver bug reports are indexed by. */ +function formatPciId(value: unknown): string | undefined { + if (typeof value === 'number' && Number.isFinite(value)) { + return `0x${Math.trunc(value).toString(16).padStart(4, '0')}` + } + return typeof value === 'string' && value.length > 0 + ? value.slice(0, MAX_GPU_STRING_LENGTH) + : undefined +} + +function selectGpuDevice(raw: UnknownRecord | null): UnknownRecord | null { + const devices = raw?.gpuDevice + if (!Array.isArray(devices) || devices.length === 0) { + return null + } + const records = devices.map(asRecord).filter((device): device is UnknownRecord => device !== null) + return records.find((device) => device.active === true) ?? records[0] ?? null +} + +/** + * Flattens `app.getGPUInfo('complete')` into crash-detail scalars. Pure so the + * shape can be tested without an Electron GPU process. + */ +export function summarizeGpuInfo(raw: unknown): CrashReportBreadcrumbData { + const root = asRecord(raw) + if (!root) { + return { gpuInfoAvailable: false } + } + const aux = asRecord(root.auxAttributes) + const device = selectGpuDevice(root) + const devices = Array.isArray(root.gpuDevice) ? root.gpuDevice.length : 0 + + return sanitizeCrashReportDetails({ + gpuInfoAvailable: true, + gpuDeviceCount: devices, + gpuVendorId: formatPciId(device?.vendorId), + gpuDeviceId: formatPciId(device?.deviceId), + gpuDriverVendor: pickString(device, 'driverVendor'), + gpuDriverVersion: pickString(device, 'driverVersion'), + gpuDeviceString: pickString(device, 'deviceString'), + gpuGlVendor: pickString(aux, 'glVendor'), + gpuGlRenderer: pickString(aux, 'glRenderer'), + gpuGlVersion: pickString(aux, 'glVersion'), + gpuAnglePlatform: pickString(aux, 'glImplementation'), + gpuBasicInfoState: pickNumber(aux, 'basicInfoState'), + gpuContextInfoState: pickNumber(aux, 'contextInfoState'), + gpuInitializationTimeMs: pickNumber(aux, 'initializationTime'), + gpuInProcess: pickBoolean(aux, 'inProcessGpu'), + gpuSandboxed: pickBoolean(aux, 'sandboxed'), + gpuDirectComposition: pickBoolean(aux, 'directComposition'), + gpuPassthroughCmdDecoder: pickBoolean(aux, 'passthroughCmdDecoder'), + gpuMachineModel: pickString(root, 'machineModelName'), + gpuMachineModelVersion: pickString(root, 'machineModelVersion') + }) +} + +let gpuInfoSnapshot: CrashReportBreadcrumbData | null = null + +export function getGpuInfoSnapshot(): CrashReportBreadcrumbData | null { + return gpuInfoSnapshot +} + +export function setGpuInfoSnapshotForTesting(snapshot: CrashReportBreadcrumbData | null): void { + gpuInfoSnapshot = snapshot +} + +/** + * Captures the snapshot once, off the startup critical path. On a machine whose + * GPU child CHECK-crashes at init this call can hang forever, so it is bounded + * and records the failure shape rather than nothing. + */ +export async function captureGpuInfoSnapshot( + getGpuInfo: () => Promise, + timeoutMs: number +): Promise { + let timer: NodeJS.Timeout | undefined + try { + const snapshot = await Promise.race([ + getGpuInfo().then(summarizeGpuInfo), + new Promise((resolve) => { + timer = setTimeout( + () => resolve({ gpuInfoAvailable: false, gpuInfoError: 'timeout' }), + timeoutMs + ) + timer.unref?.() + }) + ]) + gpuInfoSnapshot = snapshot + return snapshot + } catch (error) { + const snapshot = sanitizeCrashReportDetails({ + gpuInfoAvailable: false, + gpuInfoError: error instanceof Error ? error.message : String(error) + }) + gpuInfoSnapshot = snapshot + return snapshot + } finally { + if (timer) { + clearTimeout(timer) + } + } +} diff --git a/src/main/startup/gpu-cache-purge.test.ts b/src/main/startup/gpu-cache-purge.test.ts new file mode 100644 index 00000000000..c817e7c3656 --- /dev/null +++ b/src/main/startup/gpu-cache-purge.test.ts @@ -0,0 +1,50 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { GPU_CACHE_DIRECTORY_NAMES, purgeGpuCaches } from './gpu-cache-purge' + +describe('purgeGpuCaches', () => { + let userDataPath: string + + beforeEach(() => { + userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-cache-purge-test-')) + }) + + afterEach(() => { + rmSync(userDataPath, { recursive: true, force: true }) + }) + + it('removes populated GPU cache directories', () => { + for (const name of GPU_CACHE_DIRECTORY_NAMES) { + mkdirSync(join(userDataPath, name, 'nested'), { recursive: true }) + writeFileSync(join(userDataPath, name, 'nested', 'entry.bin'), 'shader') + } + + const result = purgeGpuCaches(userDataPath) + + expect(result.removed).toEqual([...GPU_CACHE_DIRECTORY_NAMES]) + expect(result.failed).toEqual([]) + for (const name of GPU_CACHE_DIRECTORY_NAMES) { + expect(existsSync(join(userDataPath, name))).toBe(false) + } + }) + + it('reports nothing removed when no caches exist', () => { + expect(purgeGpuCaches(userDataPath)).toEqual({ removed: [], failed: [] }) + }) + + it('leaves unrelated userData entries alone', () => { + mkdirSync(join(userDataPath, 'GPUCache'), { recursive: true }) + writeFileSync(join(userDataPath, 'orca-data.json'), '{}') + + purgeGpuCaches(userDataPath) + + expect(existsSync(join(userDataPath, 'orca-data.json'))).toBe(true) + }) + + // Why: this runs on the path to a relaunch — a missing userData dir must not throw. + it('does not throw for a missing userData directory', () => { + expect(() => purgeGpuCaches(join(userDataPath, 'absent'))).not.toThrow() + }) +}) diff --git a/src/main/startup/gpu-cache-purge.ts b/src/main/startup/gpu-cache-purge.ts new file mode 100644 index 00000000000..4d9c11fbbd1 --- /dev/null +++ b/src/main/startup/gpu-cache-purge.ts @@ -0,0 +1,43 @@ +import { existsSync, rmSync } from 'node:fs' +import { join } from 'node:path' + +/** + * Chromium GPU/shader caches under userData. + * + * A cache written by the driver that just CHECK-crashed is replayed on the next + * GPU init, so escalating the fallback tier without purging can reproduce the + * same crash on a launch that should have been safe. + */ +export const GPU_CACHE_DIRECTORY_NAMES = [ + 'GPUCache', + 'ShaderCache', + 'GrShaderCache', + 'DawnCache', + 'DawnGraphiteCache', + 'DawnWebGPUCache' +] as const + +export type GpuCachePurgeResult = { + removed: string[] + failed: string[] +} + +/** Best effort: a locked cache directory must never block the relaunch that follows. */ +export function purgeGpuCaches(userDataPath: string): GpuCachePurgeResult { + const removed: string[] = [] + const failed: string[] = [] + for (const name of GPU_CACHE_DIRECTORY_NAMES) { + const target = join(userDataPath, name) + try { + if (!existsSync(target)) { + continue + } + // Why: Windows can hold a brief lock on a cache file the dying GPU child owned. + rmSync(target, { recursive: true, force: true, maxRetries: 3, retryDelay: 25 }) + removed.push(name) + } catch { + failed.push(name) + } + } + return { removed, failed } +} diff --git a/src/main/startup/gpu-fallback-tiers.test.ts b/src/main/startup/gpu-fallback-tiers.test.ts new file mode 100644 index 00000000000..4723b1820ed --- /dev/null +++ b/src/main/startup/gpu-fallback-tiers.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_GPU_FALLBACK_TIER, + clampGpuFallbackTier, + getGpuFallbackTierSwitches, + getNextGpuFallbackTier, + isGpuFallbackTier +} from './gpu-fallback-tiers' + +describe('gpu-fallback-tiers', () => { + it('escalates one rung at a time and stops at the top', () => { + expect(getNextGpuFallbackTier(0)).toBe(1) + expect(getNextGpuFallbackTier(1)).toBe(2) + expect(getNextGpuFallbackTier(2)).toBe(3) + expect(getNextGpuFallbackTier(3)).toBeNull() + }) + + // Why: null is what bounds relaunches per build; a non-null answer here is a crash loop. + it('never escalates past the ladder for out-of-range input', () => { + expect(getNextGpuFallbackTier(99)).toBeNull() + expect(getNextGpuFallbackTier(Number.POSITIVE_INFINITY)).toBeNull() + expect(getNextGpuFallbackTier(Number.NaN)).toBe(1) + expect(getNextGpuFallbackTier(-5)).toBe(1) + }) + + it('clamps untrusted persisted tiers', () => { + expect(clampGpuFallbackTier(undefined)).toBe(1) + expect(clampGpuFallbackTier('2')).toBe(1) + expect(clampGpuFallbackTier(0)).toBe(1) + expect(clampGpuFallbackTier(2)).toBe(2) + expect(clampGpuFallbackTier(2.7)).toBe(2) + expect(clampGpuFallbackTier(999)).toBe(MAX_GPU_FALLBACK_TIER) + }) + + it('identifies ladder members', () => { + expect(isGpuFallbackTier(1)).toBe(true) + expect(isGpuFallbackTier(0)).toBe(false) + expect(isGpuFallbackTier(4)).toBe(false) + }) + + // Why: escalating must never drop a switch a lower tier already needed. + it('is additive across tiers', () => { + const names = ([1, 2, 3] as const).map( + (tier) => new Set(getGpuFallbackTierSwitches(tier).map((entry) => entry.name)) + ) + for (const name of names[0]) { + expect(names[1].has(name)).toBe(true) + } + for (const name of names[1]) { + expect(names[2].has(name)).toBe(true) + } + }) + + it('removes the GPU child process at the top tier', () => { + const tier1 = getGpuFallbackTierSwitches(1).map((entry) => entry.name) + expect(tier1).toEqual(['disable-gpu']) + + const tier2 = getGpuFallbackTierSwitches(2) + expect(tier2.map((entry) => entry.name)).toContain('disable-gpu-compositing') + expect(tier2.find((entry) => entry.name === 'use-angle')?.value).toBe('swiftshader') + + expect(getGpuFallbackTierSwitches(3).map((entry) => entry.name)).toContain('in-process-gpu') + }) +}) diff --git a/src/main/startup/gpu-fallback-tiers.ts b/src/main/startup/gpu-fallback-tiers.ts new file mode 100644 index 00000000000..fa1b5ddc4d9 --- /dev/null +++ b/src/main/startup/gpu-fallback-tiers.ts @@ -0,0 +1,77 @@ +/** + * Escalation ladder for the Windows GPU-crash fallback. + * + * `--disable-gpu` alone does not stop Chromium from spawning a GPU child for the + * Viz display compositor, so a driver that CHECK-crashes at GPU init keeps + * killing every launch even with the fallback applied. Each tier removes + * strictly more of the GPU process from the path; tier 3 removes the child + * process entirely, which is the only shape a GPU-init crash cannot survive. + * + * Tiers are additive on purpose: escalating never drops a switch that a lower + * tier already needed. + */ + +export const GPU_FALLBACK_TIERS = [1, 2, 3] as const + +export type GpuFallbackTier = (typeof GPU_FALLBACK_TIERS)[number] + +export const MIN_GPU_FALLBACK_TIER: GpuFallbackTier = 1 +export const MAX_GPU_FALLBACK_TIER: GpuFallbackTier = 3 + +/** Tier 0 means "no fallback applied to this launch". */ +export const NO_GPU_FALLBACK_TIER = 0 + +export type GpuFallbackSwitch = { name: string; value?: string } + +const TIER_SWITCHES: Record = { + 1: [{ name: 'disable-gpu' }], + 2: [ + { name: 'disable-gpu' }, + // Why: moves the display compositor off the GPU child, which tier 1 still spawned. + { name: 'disable-gpu-compositing' }, + // Why: keeps a broken vendor D3D11 driver DLL out of the process even when Chromium still initializes ANGLE. + { name: 'use-angle', value: 'swiftshader' } + ], + 3: [ + { name: 'disable-gpu' }, + { name: 'disable-gpu-compositing' }, + { name: 'use-angle', value: 'swiftshader' }, + // Why: last resort — no GPU child exists, so a GPU-init CHECK crash can no longer loop the app. + { name: 'in-process-gpu' } + ] +} + +export function isGpuFallbackTier(value: unknown): value is GpuFallbackTier { + return GPU_FALLBACK_TIERS.some((tier) => tier === value) +} + +/** Coerces a persisted/untrusted tier onto the ladder; anything unrecognized starts at tier 1. */ +export function clampGpuFallbackTier(value: unknown): GpuFallbackTier { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return MIN_GPU_FALLBACK_TIER + } + const rounded = Math.trunc(value) + if (rounded <= MIN_GPU_FALLBACK_TIER) { + return MIN_GPU_FALLBACK_TIER + } + return rounded >= MAX_GPU_FALLBACK_TIER ? MAX_GPU_FALLBACK_TIER : (rounded as GpuFallbackTier) +} + +export function getGpuFallbackTierSwitches(tier: GpuFallbackTier): readonly GpuFallbackSwitch[] { + return TIER_SWITCHES[tier] +} + +/** + * Next rung above `currentTier` (0 = nothing applied yet), or null once the + * ladder is exhausted. Null is what bounds relaunches to 3 per build. + */ +export function getNextGpuFallbackTier(currentTier: number): GpuFallbackTier | null { + if (Number.isNaN(currentTier) || currentTier < MIN_GPU_FALLBACK_TIER) { + return MIN_GPU_FALLBACK_TIER + } + if (currentTier >= MAX_GPU_FALLBACK_TIER) { + return null + } + const next = Math.trunc(currentTier) + 1 + return isGpuFallbackTier(next) ? next : null +}