From c5539df5174f1f5d63d6103caab863c90fa9a455 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:57:24 -0700 Subject: [PATCH] Record GPU fallback escalation context in crash reports Add durable logging when GPU marker writes fail and enhance relaunch telemetry with escalation state. Ensure GPU identity snapshots reach crash details for triage, and gracefully handle locked cache directories. --- .../process-gone-recorder.test.ts | 72 +++++++++++++++++++ src/main/index.ts | 19 ++++- src/main/startup/gpu-cache-purge.test.ts | 48 ++++++++++++- src/main/startup/gpu-fallback-marker.test.ts | 34 +++++++++ src/main/startup/gpu-fallback-tiers.ts | 3 + 5 files changed, 172 insertions(+), 4 deletions(-) diff --git a/src/main/crash-reporting/process-gone-recorder.test.ts b/src/main/crash-reporting/process-gone-recorder.test.ts index eec093c7ddc..ef85104d32c 100644 --- a/src/main/crash-reporting/process-gone-recorder.test.ts +++ b/src/main/crash-reporting/process-gone-recorder.test.ts @@ -15,6 +15,7 @@ import { type ProcessGoneCrashEvent } from './process-gone-recorder' import { _resetTracerForTests, setActiveSink, type TracerSink } from '../observability/tracer' +import { setGpuInfoSnapshotForTesting } from './gpu-info-snapshot' type CapturingSink = TracerSink & { records: unknown[]; flushMock: ReturnType } @@ -55,6 +56,7 @@ afterEach(() => { vi.restoreAllMocks() _resetTracerForTests() clearCrashBreadcrumbsForTest() + setGpuInfoSnapshotForTesting(null) }) describe('recordProcessGoneCrash', () => { @@ -334,6 +336,76 @@ describe('recordProcessGoneCrash', () => { await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(4)) }) + // Why: driver identity is the whole point of the GPU snapshot — if it stops reaching + // crash details, triage silently loses the vendor/device data it was added for. + it('attaches the GPU identity snapshot to GPU-child crash details', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + + recordProcessGoneCrash( + { record } as never, + event({ + source: 'child', + processType: 'GPU', + gpuFallbackActive: true, + details: { type: 'GPU', gpuFallbackTier: 2 } + }), + new ProcessGoneDedupe() + ) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details).toEqual( + expect.objectContaining({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + ) + }) + + it('attaches the GPU identity snapshot to renderer crash details', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + + recordProcessGoneCrash({ record } as never, event(), new ProcessGoneDedupe()) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details).toEqual( + expect.objectContaining({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + ) + }) + + // Why: every other child type would just pad the report. + it('omits the GPU identity snapshot for non-GPU child crashes', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + + recordProcessGoneCrash( + { record } as never, + event({ source: 'child', processType: 'Utility', details: { type: 'Utility' } }), + new ProcessGoneDedupe() + ) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details).not.toHaveProperty('gpuVendorId') + }) + + // Why: the reworded crash dialog keys off a numeric gpuFallbackTier; if sanitization + // ever dropped or stringified it, isGpuFallbackCrashReport would silently never match. + it('preserves gpuFallbackTier as a number through sanitization', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + + recordProcessGoneCrash( + { record } as never, + event({ + source: 'child', + processType: 'GPU', + gpuFallbackActive: true, + details: { type: 'GPU', gpuFallbackTier: 2 } + }), + new ProcessGoneDedupe() + ) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details.gpuFallbackTier).toBe(2) + }) + it('allows the same renderer crash to retry after persistence fails', async () => { const record = vi .fn() diff --git a/src/main/index.ts b/src/main/index.ts index 0d450f31c33..e18cb6b13a0 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -101,7 +101,7 @@ import { NO_GPU_FALLBACK_TIER, getGpuFallbackTierSwitches, resolveGpuFallbackEscalation, - type GpuFallbackTier + type GpuFallbackTierOrNone } from './startup/gpu-fallback-tiers' import { purgeGpuCaches } from './startup/gpu-cache-purge' import { @@ -310,7 +310,7 @@ const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({ threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD }) // Why: 0 = hardware path. A non-zero tier is the rung of the fallback ladder this launch is running. -let gpuFallbackTierThisLaunch: GpuFallbackTier | typeof NO_GPU_FALLBACK_TIER = NO_GPU_FALLBACK_TIER +let gpuFallbackTierThisLaunch: GpuFallbackTierOrNone = NO_GPU_FALLBACK_TIER const GPU_INFO_CAPTURE_TIMEOUT_MS = 10_000 let localPtyStartupReady: Promise = Promise.resolve() let localPtyProviderStartupReady: Promise = Promise.resolve() @@ -1493,6 +1493,15 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void { ) } catch (error) { console.warn('[gpu-fallback] failed to persist marker:', error) + // Why: this abandons the tier record, the cache purge and the relaunch; console + // output reaches no support bundle, so the escalation would vanish without trace. + recordDurableCrashBreadcrumb('gpu_fallback_marker_write_failed', { + reason, + exitCode, + tier: nextTier, + previousTier: currentTier, + errorMessage: error instanceof Error ? error.message : String(error) + }) return } // Why: survives the update that invalidates the marker above. @@ -1509,11 +1518,17 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void { failed: purge.failed.join(',') }) isQuitting = true + // Why: the breadcrumbs above are in-memory only and this process exits next, so + // app_relaunch_requested is the sole durable record of what the escalation did. relaunchApp('gpu-fallback', { processReason: reason, exitCode, tier: nextTier, previousTier: currentTier, + resumedFromHistory, + recordedTier: recordedTier ?? null, + cachesPurged: purge.removed.join(','), + cachePurgeFailed: purge.failed.join(','), crashesInWindow: result.crashesInWindow }) app.exit(0) diff --git a/src/main/startup/gpu-cache-purge.test.ts b/src/main/startup/gpu-cache-purge.test.ts index c817e7c3656..789165abaca 100644 --- a/src/main/startup/gpu-cache-purge.test.ts +++ b/src/main/startup/gpu-cache-purge.test.ts @@ -1,17 +1,37 @@ +import type * as NodeFs from 'node:fs' import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import os from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { basename, join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { GPU_CACHE_DIRECTORY_NAMES, purgeGpuCaches } from './gpu-cache-purge' +// Why: a locked cache directory is a Windows-only failure in the wild; this fakes it +// so the `failed` bookkeeping is covered without a platform-specific (or root-skipped) test. +const { unremovableDirectory } = vi.hoisted(() => ({ unremovableDirectory: { name: '' } })) + +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + rmSync: (target: Parameters[0], options?: object) => { + if (unremovableDirectory.name && basename(String(target)) === unremovableDirectory.name) { + throw Object.assign(new Error('EBUSY: resource busy or locked'), { code: 'EBUSY' }) + } + return actual.rmSync(target, options) + } + } +}) + describe('purgeGpuCaches', () => { let userDataPath: string beforeEach(() => { userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-cache-purge-test-')) + unremovableDirectory.name = '' }) afterEach(() => { + unremovableDirectory.name = '' rmSync(userDataPath, { recursive: true, force: true }) }) @@ -47,4 +67,28 @@ describe('purgeGpuCaches', () => { it('does not throw for a missing userData directory', () => { expect(() => purgeGpuCaches(join(userDataPath, 'absent'))).not.toThrow() }) + + it('reports a locked cache directory as failed and still purges the rest', () => { + for (const name of GPU_CACHE_DIRECTORY_NAMES) { + mkdirSync(join(userDataPath, name), { recursive: true }) + } + unremovableDirectory.name = 'ShaderCache' + + const result = purgeGpuCaches(userDataPath) + + expect(result.failed).toEqual(['ShaderCache']) + expect(result.removed).toEqual( + GPU_CACHE_DIRECTORY_NAMES.filter((name) => name !== 'ShaderCache') + ) + expect(existsSync(join(userDataPath, 'ShaderCache'))).toBe(true) + expect(existsSync(join(userDataPath, 'GPUCache'))).toBe(false) + }) + + // Why: a locked directory must not abort the relaunch it precedes. + it('does not throw when a cache directory cannot be removed', () => { + mkdirSync(join(userDataPath, 'GPUCache'), { recursive: true }) + unremovableDirectory.name = 'GPUCache' + + expect(() => purgeGpuCaches(userDataPath)).not.toThrow() + }) }) diff --git a/src/main/startup/gpu-fallback-marker.test.ts b/src/main/startup/gpu-fallback-marker.test.ts index fcc4ff9bb80..74f79bfe154 100644 --- a/src/main/startup/gpu-fallback-marker.test.ts +++ b/src/main/startup/gpu-fallback-marker.test.ts @@ -49,6 +49,40 @@ describe('gpu-fallback-marker', () => { expect(existsSync(join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false) }) + // Why: the caller treats a throw as "escalation abandoned", so a failed write must + // surface rather than leave a half-written marker or a stray temp file behind. + it('throws and leaves no temp file behind when the atomic write fails', () => { + const missingUserDataPath = join(userDataPath, 'absent') + + expect(() => + writeGpuFallbackMarker( + missingUserDataPath, + { engagedAt: 1, crashesInWindow: 3, tier: 2 }, + environment + ) + ).toThrow() + + expect(existsSync(join(missingUserDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false) + expect(existsSync(join(missingUserDataPath, GPU_FALLBACK_MARKER_FILE))).toBe(false) + }) + + // Why: the rename is the step that actually fails in the wild (a locked or + // directory-shaped target), and that is the only path where a temp file can survive. + it('cleans up the temp file when the rename fails', () => { + mkdirSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE), { recursive: true }) + writeFileSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE, 'occupied'), 'x') + + expect(() => + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 3, tier: 2 }, + environment + ) + ).toThrow() + + expect(existsSync(join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false) + }) + it('overwrites an existing marker when escalating tiers', () => { writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3, tier: 1 }, environment) writeGpuFallbackMarker(userDataPath, { engagedAt: 2, crashesInWindow: 4, tier: 2 }, environment) diff --git a/src/main/startup/gpu-fallback-tiers.ts b/src/main/startup/gpu-fallback-tiers.ts index 62c8670f99a..e7ac46213bc 100644 --- a/src/main/startup/gpu-fallback-tiers.ts +++ b/src/main/startup/gpu-fallback-tiers.ts @@ -25,6 +25,9 @@ export const MAX_GPU_FALLBACK_TIER: GpuFallbackTier = 2 /** Tier 0 means "no fallback applied to this launch". */ export const NO_GPU_FALLBACK_TIER = 0 +/** A rung, or the hardware path. The only values a launch's current tier may hold. */ +export type GpuFallbackTierOrNone = GpuFallbackTier | typeof NO_GPU_FALLBACK_TIER + export type GpuFallbackSwitch = { name: string; value?: string } const TIER_SWITCHES: Record = {