feat(gpu): add the GPU fallback ladder, cache purge, and driver-identity modules

`--disable-gpu` does not stop Chromium from spawning a GPU child for the Viz
display compositor, so a driver that CHECK-crashes at GPU init keeps killing
every launch even with the fallback applied. These are the pieces a real
remediation needs, added standalone with no callers yet:

- a bounded, additive tier ladder (disable-gpu -> +compositing/ANGLE swiftshader
  -> +in-process-gpu, which removes the crashing child entirely)
- a GPU/shader cache purge, so a cache written by the crashing driver is not
  replayed into the tier we just escalated to
- a summarizer for app.getGPUInfo('complete'); nothing in src/main called it, so
  every GPU crash bundle arrived with no vendor, device, or driver version

Co-authored-by: Orca <help@stably.ai>

src/main/crash-reporting/gpu-info-snapshot.test.ts

@@ -0,0 +1,115 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+  captureGpuInfoSnapshot,
+  getGpuInfoSnapshot,
+  setGpuInfoSnapshotForTesting,
+  summarizeGpuInfo
+} from './gpu-info-snapshot'
+
+const COMPLETE_INFO = {
+  auxAttributes: {
+    glVendor: 'Google Inc. (Intel)',
+    glRenderer: 'ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)',
+    glVersion: 'OpenGL ES 2.0.0',
+    glImplementation: 'EGL_ANGLE',
+    glExtensions: 'GL_EXT_a '.repeat(500),
+    basicInfoState: 1,
+    contextInfoState: 1,
+    initializationTime: 42.5,
+    inProcessGpu: false,
+    sandboxed: true,
+    directComposition: true,
+    passthroughCmdDecoder: true
+  },
+  gpuDevice: [
+    { active: false, vendorId: 4318, deviceId: 7938, driverVendor: 'NVIDIA' },
+    {
+      active: true,
+      vendorId: 32902,
+      deviceId: 15130,
+      driverVendor: 'Intel Corporation',
+      driverVersion: '31.0.101.2111',
+      deviceString: 'Intel(R) UHD Graphics 620'
+    }
+  ],
+  machineModelName: 'Latitude 5400',
+  machineModelVersion: '1.0'
+}
+
+describe('summarizeGpuInfo', () => {
+  it('flattens the active device and aux attributes', () => {
+    const summary = summarizeGpuInfo(COMPLETE_INFO)
+
+    expect(summary).toMatchObject({
+      gpuInfoAvailable: true,
+      gpuDeviceCount: 2,
+      gpuVendorId: '0x8086',
+      gpuDeviceId: '0x3b1a',
+      gpuDriverVendor: 'Intel Corporation',
+      gpuDriverVersion: '31.0.101.2111',
+      gpuGlVendor: 'Google Inc. (Intel)',
+      gpuAnglePlatform: 'EGL_ANGLE',
+      gpuInProcess: false,
+      gpuSandboxed: true,
+      gpuBasicInfoState: 1,
+      gpuInitializationTimeMs: 42.5,
+      gpuMachineModel: 'Latitude 5400'
+    })
+  })
+
+  // Why: glExtensions is kilobytes long and would dominate every crash report.
+  it('never copies the extension list and bounds long strings', () => {
+    const summary = summarizeGpuInfo(COMPLETE_INFO)
+
+    expect(Object.keys(summary)).not.toContain('gpuGlExtensions')
+    for (const value of Object.values(summary)) {
+      if (typeof value === 'string') {
+        expect(value.length).toBeLessThanOrEqual(160)
+      }
+    }
+  })
+
+  it('falls back to the first device when none is marked active', () => {
+    const summary = summarizeGpuInfo({ gpuDevice: [{ vendorId: 4318, deviceId: 7938 }] })
+    expect(summary).toMatchObject({ gpuVendorId: '0x10de', gpuDeviceCount: 1 })
+  })
+
+  it('reports unavailable for non-object payloads', () => {
+    expect(summarizeGpuInfo(null)).toEqual({ gpuInfoAvailable: false })
+    expect(summarizeGpuInfo('nope')).toEqual({ gpuInfoAvailable: false })
+  })
+
+  it('omits fields the payload does not carry', () => {
+    const summary = summarizeGpuInfo({ gpuDevice: [] })
+    expect(summary).toEqual({ gpuInfoAvailable: true, gpuDeviceCount: 0 })
+  })
+})
+
+describe('captureGpuInfoSnapshot', () => {
+  afterEach(() => {
+    setGpuInfoSnapshotForTesting(null)
+    vi.useRealTimers()
+  })
+
+  it('caches a successful capture', async () => {
+    const snapshot = await captureGpuInfoSnapshot(async () => COMPLETE_INFO, 1_000)
+    expect(snapshot.gpuVendorId).toBe('0x8086')
+    expect(getGpuInfoSnapshot()).toBe(snapshot)
+  })
+
+  it('records the error shape when getGPUInfo rejects', async () => {
+    const snapshot = await captureGpuInfoSnapshot(async () => {
+      throw new Error('GPU process crashed')
+    }, 1_000)
+    expect(snapshot).toMatchObject({ gpuInfoAvailable: false, gpuInfoError: 'GPU process crashed' })
+    expect(getGpuInfoSnapshot()).toBe(snapshot)
+  })
+
+  // Why: on the machines this exists for, the GPU child never initializes and the promise never settles.
+  it('times out instead of hanging forever', async () => {
+    vi.useFakeTimers()
+    const pending = captureGpuInfoSnapshot(() => new Promise(() => {}), 10_000)
+    await vi.advanceTimersByTimeAsync(10_000)
+    await expect(pending).resolves.toEqual({ gpuInfoAvailable: false, gpuInfoError: 'timeout' })
+  })
+})

src/main/crash-reporting/gpu-info-snapshot.ts

@@ -0,0 +1,143 @@
+import {
+  sanitizeCrashReportDetails,
+  type CrashReportBreadcrumbData
+} from '../../shared/crash-reporting'
+
+/**
+ * Driver identity for crash details.
+ *
+ * Nothing in the main process called app.getGPUInfo() before, so every GPU
+ * crash bundle arrived without a vendor, device or driver version — triage
+ * could not tell a broken driver from a broken machine.
+ */
+
+/** Bounded so a pathological glExtensions-style string can't dominate the report. */
+const MAX_GPU_STRING_LENGTH = 160
+
+type UnknownRecord = Record<string, unknown>
+
+function asRecord(value: unknown): UnknownRecord | null {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+    ? (value as UnknownRecord)
+    : null
+}
+
+function pickString(source: UnknownRecord | null, key: string): string | undefined {
+  const value = source?.[key]
+  return typeof value === 'string' && value.length > 0
+    ? value.slice(0, MAX_GPU_STRING_LENGTH)
+    : undefined
+}
+
+function pickNumber(source: UnknownRecord | null, key: string): number | undefined {
+  const value = source?.[key]
+  return typeof value === 'number' && Number.isFinite(value) ? value : undefined
+}
+
+function pickBoolean(source: UnknownRecord | null, key: string): boolean | undefined {
+  const value = source?.[key]
+  return typeof value === 'boolean' ? value : undefined
+}
+
+/** Vendor/device ids arrive as numbers; hex is what driver bug reports are indexed by. */
+function formatPciId(value: unknown): string | undefined {
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    return `0x${Math.trunc(value).toString(16).padStart(4, '0')}`
+  }
+  return typeof value === 'string' && value.length > 0
+    ? value.slice(0, MAX_GPU_STRING_LENGTH)
+    : undefined
+}
+
+function selectGpuDevice(raw: UnknownRecord | null): UnknownRecord | null {
+  const devices = raw?.gpuDevice
+  if (!Array.isArray(devices) || devices.length === 0) {
+    return null
+  }
+  const records = devices.map(asRecord).filter((device): device is UnknownRecord => device !== null)
+  return records.find((device) => device.active === true) ?? records[0] ?? null
+}
+
+/**
+ * Flattens `app.getGPUInfo('complete')` into crash-detail scalars. Pure so the
+ * shape can be tested without an Electron GPU process.
+ */
+export function summarizeGpuInfo(raw: unknown): CrashReportBreadcrumbData {
+  const root = asRecord(raw)
+  if (!root) {
+    return { gpuInfoAvailable: false }
+  }
+  const aux = asRecord(root.auxAttributes)
+  const device = selectGpuDevice(root)
+  const devices = Array.isArray(root.gpuDevice) ? root.gpuDevice.length : 0
+
+  return sanitizeCrashReportDetails({
+    gpuInfoAvailable: true,
+    gpuDeviceCount: devices,
+    gpuVendorId: formatPciId(device?.vendorId),
+    gpuDeviceId: formatPciId(device?.deviceId),
+    gpuDriverVendor: pickString(device, 'driverVendor'),
+    gpuDriverVersion: pickString(device, 'driverVersion'),
+    gpuDeviceString: pickString(device, 'deviceString'),
+    gpuGlVendor: pickString(aux, 'glVendor'),
+    gpuGlRenderer: pickString(aux, 'glRenderer'),
+    gpuGlVersion: pickString(aux, 'glVersion'),
+    gpuAnglePlatform: pickString(aux, 'glImplementation'),
+    gpuBasicInfoState: pickNumber(aux, 'basicInfoState'),
+    gpuContextInfoState: pickNumber(aux, 'contextInfoState'),
+    gpuInitializationTimeMs: pickNumber(aux, 'initializationTime'),
+    gpuInProcess: pickBoolean(aux, 'inProcessGpu'),
+    gpuSandboxed: pickBoolean(aux, 'sandboxed'),
+    gpuDirectComposition: pickBoolean(aux, 'directComposition'),
+    gpuPassthroughCmdDecoder: pickBoolean(aux, 'passthroughCmdDecoder'),
+    gpuMachineModel: pickString(root, 'machineModelName'),
+    gpuMachineModelVersion: pickString(root, 'machineModelVersion')
+  })
+}
+
+let gpuInfoSnapshot: CrashReportBreadcrumbData | null = null
+
+export function getGpuInfoSnapshot(): CrashReportBreadcrumbData | null {
+  return gpuInfoSnapshot
+}
+
+export function setGpuInfoSnapshotForTesting(snapshot: CrashReportBreadcrumbData | null): void {
+  gpuInfoSnapshot = snapshot
+}
+
+/**
+ * Captures the snapshot once, off the startup critical path. On a machine whose
+ * GPU child CHECK-crashes at init this call can hang forever, so it is bounded
+ * and records the failure shape rather than nothing.
+ */
+export async function captureGpuInfoSnapshot(
+  getGpuInfo: () => Promise<unknown>,
+  timeoutMs: number
+): Promise<CrashReportBreadcrumbData> {
+  let timer: NodeJS.Timeout | undefined
+  try {
+    const snapshot = await Promise.race([
+      getGpuInfo().then(summarizeGpuInfo),
+      new Promise<CrashReportBreadcrumbData>((resolve) => {
+        timer = setTimeout(
+          () => resolve({ gpuInfoAvailable: false, gpuInfoError: 'timeout' }),
+          timeoutMs
+        )
+        timer.unref?.()
+      })
+    ])
+    gpuInfoSnapshot = snapshot
+    return snapshot
+  } catch (error) {
+    const snapshot = sanitizeCrashReportDetails({
+      gpuInfoAvailable: false,
+      gpuInfoError: error instanceof Error ? error.message : String(error)
+    })
+    gpuInfoSnapshot = snapshot
+    return snapshot
+  } finally {
+    if (timer) {
+      clearTimeout(timer)
+    }
+  }
+}

src/main/startup/gpu-cache-purge.test.ts

@@ -0,0 +1,50 @@
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import os from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { GPU_CACHE_DIRECTORY_NAMES, purgeGpuCaches } from './gpu-cache-purge'
+
+describe('purgeGpuCaches', () => {
+  let userDataPath: string
+
+  beforeEach(() => {
+    userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-cache-purge-test-'))
+  })
+
+  afterEach(() => {
+    rmSync(userDataPath, { recursive: true, force: true })
+  })
+
+  it('removes populated GPU cache directories', () => {
+    for (const name of GPU_CACHE_DIRECTORY_NAMES) {
+      mkdirSync(join(userDataPath, name, 'nested'), { recursive: true })
+      writeFileSync(join(userDataPath, name, 'nested', 'entry.bin'), 'shader')
+    }
+
+    const result = purgeGpuCaches(userDataPath)
+
+    expect(result.removed).toEqual([...GPU_CACHE_DIRECTORY_NAMES])
+    expect(result.failed).toEqual([])
+    for (const name of GPU_CACHE_DIRECTORY_NAMES) {
+      expect(existsSync(join(userDataPath, name))).toBe(false)
+    }
+  })
+
+  it('reports nothing removed when no caches exist', () => {
+    expect(purgeGpuCaches(userDataPath)).toEqual({ removed: [], failed: [] })
+  })
+
+  it('leaves unrelated userData entries alone', () => {
+    mkdirSync(join(userDataPath, 'GPUCache'), { recursive: true })
+    writeFileSync(join(userDataPath, 'orca-data.json'), '{}')
+
+    purgeGpuCaches(userDataPath)
+
+    expect(existsSync(join(userDataPath, 'orca-data.json'))).toBe(true)
+  })
+
+  // Why: this runs on the path to a relaunch — a missing userData dir must not throw.
+  it('does not throw for a missing userData directory', () => {
+    expect(() => purgeGpuCaches(join(userDataPath, 'absent'))).not.toThrow()
+  })
+})

src/main/startup/gpu-cache-purge.ts

@@ -0,0 +1,43 @@
+import { existsSync, rmSync } from 'node:fs'
+import { join } from 'node:path'
+
+/**
+ * Chromium GPU/shader caches under userData.
+ *
+ * A cache written by the driver that just CHECK-crashed is replayed on the next
+ * GPU init, so escalating the fallback tier without purging can reproduce the
+ * same crash on a launch that should have been safe.
+ */
+export const GPU_CACHE_DIRECTORY_NAMES = [
+  'GPUCache',
+  'ShaderCache',
+  'GrShaderCache',
+  'DawnCache',
+  'DawnGraphiteCache',
+  'DawnWebGPUCache'
+] as const
+
+export type GpuCachePurgeResult = {
+  removed: string[]
+  failed: string[]
+}
+
+/** Best effort: a locked cache directory must never block the relaunch that follows. */
+export function purgeGpuCaches(userDataPath: string): GpuCachePurgeResult {
+  const removed: string[] = []
+  const failed: string[] = []
+  for (const name of GPU_CACHE_DIRECTORY_NAMES) {
+    const target = join(userDataPath, name)
+    try {
+      if (!existsSync(target)) {
+        continue
+      }
+      // Why: Windows can hold a brief lock on a cache file the dying GPU child owned.
+      rmSync(target, { recursive: true, force: true, maxRetries: 3, retryDelay: 25 })
+      removed.push(name)
+    } catch {
+      failed.push(name)
+    }
+  }
+  return { removed, failed }
+}

src/main/startup/gpu-fallback-tiers.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from 'vitest'
+import {
+  MAX_GPU_FALLBACK_TIER,
+  clampGpuFallbackTier,
+  getGpuFallbackTierSwitches,
+  getNextGpuFallbackTier,
+  isGpuFallbackTier
+} from './gpu-fallback-tiers'
+
+describe('gpu-fallback-tiers', () => {
+  it('escalates one rung at a time and stops at the top', () => {
+    expect(getNextGpuFallbackTier(0)).toBe(1)
+    expect(getNextGpuFallbackTier(1)).toBe(2)
+    expect(getNextGpuFallbackTier(2)).toBe(3)
+    expect(getNextGpuFallbackTier(3)).toBeNull()
+  })
+
+  // Why: null is what bounds relaunches per build; a non-null answer here is a crash loop.
+  it('never escalates past the ladder for out-of-range input', () => {
+    expect(getNextGpuFallbackTier(99)).toBeNull()
+    expect(getNextGpuFallbackTier(Number.POSITIVE_INFINITY)).toBeNull()
+    expect(getNextGpuFallbackTier(Number.NaN)).toBe(1)
+    expect(getNextGpuFallbackTier(-5)).toBe(1)
+  })
+
+  it('clamps untrusted persisted tiers', () => {
+    expect(clampGpuFallbackTier(undefined)).toBe(1)
+    expect(clampGpuFallbackTier('2')).toBe(1)
+    expect(clampGpuFallbackTier(0)).toBe(1)
+    expect(clampGpuFallbackTier(2)).toBe(2)
+    expect(clampGpuFallbackTier(2.7)).toBe(2)
+    expect(clampGpuFallbackTier(999)).toBe(MAX_GPU_FALLBACK_TIER)
+  })
+
+  it('identifies ladder members', () => {
+    expect(isGpuFallbackTier(1)).toBe(true)
+    expect(isGpuFallbackTier(0)).toBe(false)
+    expect(isGpuFallbackTier(4)).toBe(false)
+  })
+
+  // Why: escalating must never drop a switch a lower tier already needed.
+  it('is additive across tiers', () => {
+    const names = ([1, 2, 3] as const).map(
+      (tier) => new Set(getGpuFallbackTierSwitches(tier).map((entry) => entry.name))
+    )
+    for (const name of names[0]) {
+      expect(names[1].has(name)).toBe(true)
+    }
+    for (const name of names[1]) {
+      expect(names[2].has(name)).toBe(true)
+    }
+  })
+
+  it('removes the GPU child process at the top tier', () => {
+    const tier1 = getGpuFallbackTierSwitches(1).map((entry) => entry.name)
+    expect(tier1).toEqual(['disable-gpu'])
+
+    const tier2 = getGpuFallbackTierSwitches(2)
+    expect(tier2.map((entry) => entry.name)).toContain('disable-gpu-compositing')
+    expect(tier2.find((entry) => entry.name === 'use-angle')?.value).toBe('swiftshader')
+
+    expect(getGpuFallbackTierSwitches(3).map((entry) => entry.name)).toContain('in-process-gpu')
+  })
+})

src/main/startup/gpu-fallback-tiers.ts

@@ -0,0 +1,77 @@
+/**
+ * Escalation ladder for the Windows GPU-crash fallback.
+ *
+ * `--disable-gpu` alone does not stop Chromium from spawning a GPU child for the
+ * Viz display compositor, so a driver that CHECK-crashes at GPU init keeps
+ * killing every launch even with the fallback applied. Each tier removes
+ * strictly more of the GPU process from the path; tier 3 removes the child
+ * process entirely, which is the only shape a GPU-init crash cannot survive.
+ *
+ * Tiers are additive on purpose: escalating never drops a switch that a lower
+ * tier already needed.
+ */
+
+export const GPU_FALLBACK_TIERS = [1, 2, 3] as const
+
+export type GpuFallbackTier = (typeof GPU_FALLBACK_TIERS)[number]
+
+export const MIN_GPU_FALLBACK_TIER: GpuFallbackTier = 1
+export const MAX_GPU_FALLBACK_TIER: GpuFallbackTier = 3
+
+/** Tier 0 means "no fallback applied to this launch". */
+export const NO_GPU_FALLBACK_TIER = 0
+
+export type GpuFallbackSwitch = { name: string; value?: string }
+
+const TIER_SWITCHES: Record<GpuFallbackTier, readonly GpuFallbackSwitch[]> = {
+  1: [{ name: 'disable-gpu' }],
+  2: [
+    { name: 'disable-gpu' },
+    // Why: moves the display compositor off the GPU child, which tier 1 still spawned.
+    { name: 'disable-gpu-compositing' },
+    // Why: keeps a broken vendor D3D11 driver DLL out of the process even when Chromium still initializes ANGLE.
+    { name: 'use-angle', value: 'swiftshader' }
+  ],
+  3: [
+    { name: 'disable-gpu' },
+    { name: 'disable-gpu-compositing' },
+    { name: 'use-angle', value: 'swiftshader' },
+    // Why: last resort — no GPU child exists, so a GPU-init CHECK crash can no longer loop the app.
+    { name: 'in-process-gpu' }
+  ]
+}
+
+export function isGpuFallbackTier(value: unknown): value is GpuFallbackTier {
+  return GPU_FALLBACK_TIERS.some((tier) => tier === value)
+}
+
+/** Coerces a persisted/untrusted tier onto the ladder; anything unrecognized starts at tier 1. */
+export function clampGpuFallbackTier(value: unknown): GpuFallbackTier {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return MIN_GPU_FALLBACK_TIER
+  }
+  const rounded = Math.trunc(value)
+  if (rounded <= MIN_GPU_FALLBACK_TIER) {
+    return MIN_GPU_FALLBACK_TIER
+  }
+  return rounded >= MAX_GPU_FALLBACK_TIER ? MAX_GPU_FALLBACK_TIER : (rounded as GpuFallbackTier)
+}
+
+export function getGpuFallbackTierSwitches(tier: GpuFallbackTier): readonly GpuFallbackSwitch[] {
+  return TIER_SWITCHES[tier]
+}
+
+/**
+ * Next rung above `currentTier` (0 = nothing applied yet), or null once the
+ * ladder is exhausted. Null is what bounds relaunches to 3 per build.
+ */
+export function getNextGpuFallbackTier(currentTier: number): GpuFallbackTier | null {
+  if (Number.isNaN(currentTier) || currentTier < MIN_GPU_FALLBACK_TIER) {
+    return MIN_GPU_FALLBACK_TIER
+  }
+  if (currentTier >= MAX_GPU_FALLBACK_TIER) {
+    return null
+  }
+  const next = Math.trunc(currentTier) + 1
+  return isGpuFallbackTier(next) ? next : null
+}