diff --git a/src/main/crash-reporting/process-gone-recorder.test.ts b/src/main/crash-reporting/process-gone-recorder.test.ts index eec093c7ddc..ef85104d32c 100644 --- a/src/main/crash-reporting/process-gone-recorder.test.ts +++ b/src/main/crash-reporting/process-gone-recorder.test.ts @@ -15,6 +15,7 @@ import { type ProcessGoneCrashEvent } from './process-gone-recorder' import { _resetTracerForTests, setActiveSink, type TracerSink } from '../observability/tracer' +import { setGpuInfoSnapshotForTesting } from './gpu-info-snapshot' type CapturingSink = TracerSink & { records: unknown[]; flushMock: ReturnType } @@ -55,6 +56,7 @@ afterEach(() => { vi.restoreAllMocks() _resetTracerForTests() clearCrashBreadcrumbsForTest() + setGpuInfoSnapshotForTesting(null) }) describe('recordProcessGoneCrash', () => { @@ -334,6 +336,76 @@ describe('recordProcessGoneCrash', () => { await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(4)) }) + // Why: driver identity is the whole point of the GPU snapshot — if it stops reaching + // crash details, triage silently loses the vendor/device data it was added for. + it('attaches the GPU identity snapshot to GPU-child crash details', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + + recordProcessGoneCrash( + { record } as never, + event({ + source: 'child', + processType: 'GPU', + gpuFallbackActive: true, + details: { type: 'GPU', gpuFallbackTier: 2 } + }), + new ProcessGoneDedupe() + ) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details).toEqual( + expect.objectContaining({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + ) + }) + + it('attaches the GPU identity snapshot to renderer crash details', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + + recordProcessGoneCrash({ record } as never, event(), new ProcessGoneDedupe()) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details).toEqual( + expect.objectContaining({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + ) + }) + + // Why: every other child type would just pad the report. + it('omits the GPU identity snapshot for non-GPU child crashes', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' }) + + recordProcessGoneCrash( + { record } as never, + event({ source: 'child', processType: 'Utility', details: { type: 'Utility' } }), + new ProcessGoneDedupe() + ) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details).not.toHaveProperty('gpuVendorId') + }) + + // Why: the reworded crash dialog keys off a numeric gpuFallbackTier; if sanitization + // ever dropped or stringified it, isGpuFallbackCrashReport would silently never match. + it('preserves gpuFallbackTier as a number through sanitization', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report' }) + + recordProcessGoneCrash( + { record } as never, + event({ + source: 'child', + processType: 'GPU', + gpuFallbackActive: true, + details: { type: 'GPU', gpuFallbackTier: 2 } + }), + new ProcessGoneDedupe() + ) + + await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1)) + expect(record.mock.calls[0][0].details.gpuFallbackTier).toBe(2) + }) + it('allows the same renderer crash to retry after persistence fails', async () => { const record = vi .fn() diff --git a/src/main/index.ts b/src/main/index.ts index 0d450f31c33..e18cb6b13a0 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -101,7 +101,7 @@ import { NO_GPU_FALLBACK_TIER, getGpuFallbackTierSwitches, resolveGpuFallbackEscalation, - type GpuFallbackTier + type GpuFallbackTierOrNone } from './startup/gpu-fallback-tiers' import { purgeGpuCaches } from './startup/gpu-cache-purge' import { @@ -310,7 +310,7 @@ const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({ threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD }) // Why: 0 = hardware path. A non-zero tier is the rung of the fallback ladder this launch is running. -let gpuFallbackTierThisLaunch: GpuFallbackTier | typeof NO_GPU_FALLBACK_TIER = NO_GPU_FALLBACK_TIER +let gpuFallbackTierThisLaunch: GpuFallbackTierOrNone = NO_GPU_FALLBACK_TIER const GPU_INFO_CAPTURE_TIMEOUT_MS = 10_000 let localPtyStartupReady: Promise = Promise.resolve() let localPtyProviderStartupReady: Promise = Promise.resolve() @@ -1493,6 +1493,15 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void { ) } catch (error) { console.warn('[gpu-fallback] failed to persist marker:', error) + // Why: this abandons the tier record, the cache purge and the relaunch; console + // output reaches no support bundle, so the escalation would vanish without trace. + recordDurableCrashBreadcrumb('gpu_fallback_marker_write_failed', { + reason, + exitCode, + tier: nextTier, + previousTier: currentTier, + errorMessage: error instanceof Error ? error.message : String(error) + }) return } // Why: survives the update that invalidates the marker above. @@ -1509,11 +1518,17 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void { failed: purge.failed.join(',') }) isQuitting = true + // Why: the breadcrumbs above are in-memory only and this process exits next, so + // app_relaunch_requested is the sole durable record of what the escalation did. relaunchApp('gpu-fallback', { processReason: reason, exitCode, tier: nextTier, previousTier: currentTier, + resumedFromHistory, + recordedTier: recordedTier ?? null, + cachesPurged: purge.removed.join(','), + cachePurgeFailed: purge.failed.join(','), crashesInWindow: result.crashesInWindow }) app.exit(0) diff --git a/src/main/startup/gpu-cache-purge.test.ts b/src/main/startup/gpu-cache-purge.test.ts index c817e7c3656..789165abaca 100644 --- a/src/main/startup/gpu-cache-purge.test.ts +++ b/src/main/startup/gpu-cache-purge.test.ts @@ -1,17 +1,37 @@ +import type * as NodeFs from 'node:fs' import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import os from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { basename, join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { GPU_CACHE_DIRECTORY_NAMES, purgeGpuCaches } from './gpu-cache-purge' +// Why: a locked cache directory is a Windows-only failure in the wild; this fakes it +// so the `failed` bookkeeping is covered without a platform-specific (or root-skipped) test. +const { unremovableDirectory } = vi.hoisted(() => ({ unremovableDirectory: { name: '' } })) + +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + rmSync: (target: Parameters[0], options?: object) => { + if (unremovableDirectory.name && basename(String(target)) === unremovableDirectory.name) { + throw Object.assign(new Error('EBUSY: resource busy or locked'), { code: 'EBUSY' }) + } + return actual.rmSync(target, options) + } + } +}) + describe('purgeGpuCaches', () => { let userDataPath: string beforeEach(() => { userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-cache-purge-test-')) + unremovableDirectory.name = '' }) afterEach(() => { + unremovableDirectory.name = '' rmSync(userDataPath, { recursive: true, force: true }) }) @@ -47,4 +67,28 @@ describe('purgeGpuCaches', () => { it('does not throw for a missing userData directory', () => { expect(() => purgeGpuCaches(join(userDataPath, 'absent'))).not.toThrow() }) + + it('reports a locked cache directory as failed and still purges the rest', () => { + for (const name of GPU_CACHE_DIRECTORY_NAMES) { + mkdirSync(join(userDataPath, name), { recursive: true }) + } + unremovableDirectory.name = 'ShaderCache' + + const result = purgeGpuCaches(userDataPath) + + expect(result.failed).toEqual(['ShaderCache']) + expect(result.removed).toEqual( + GPU_CACHE_DIRECTORY_NAMES.filter((name) => name !== 'ShaderCache') + ) + expect(existsSync(join(userDataPath, 'ShaderCache'))).toBe(true) + expect(existsSync(join(userDataPath, 'GPUCache'))).toBe(false) + }) + + // Why: a locked directory must not abort the relaunch it precedes. + it('does not throw when a cache directory cannot be removed', () => { + mkdirSync(join(userDataPath, 'GPUCache'), { recursive: true }) + unremovableDirectory.name = 'GPUCache' + + expect(() => purgeGpuCaches(userDataPath)).not.toThrow() + }) }) diff --git a/src/main/startup/gpu-fallback-marker.test.ts b/src/main/startup/gpu-fallback-marker.test.ts index fcc4ff9bb80..74f79bfe154 100644 --- a/src/main/startup/gpu-fallback-marker.test.ts +++ b/src/main/startup/gpu-fallback-marker.test.ts @@ -49,6 +49,40 @@ describe('gpu-fallback-marker', () => { expect(existsSync(join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false) }) + // Why: the caller treats a throw as "escalation abandoned", so a failed write must + // surface rather than leave a half-written marker or a stray temp file behind. + it('throws and leaves no temp file behind when the atomic write fails', () => { + const missingUserDataPath = join(userDataPath, 'absent') + + expect(() => + writeGpuFallbackMarker( + missingUserDataPath, + { engagedAt: 1, crashesInWindow: 3, tier: 2 }, + environment + ) + ).toThrow() + + expect(existsSync(join(missingUserDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false) + expect(existsSync(join(missingUserDataPath, GPU_FALLBACK_MARKER_FILE))).toBe(false) + }) + + // Why: the rename is the step that actually fails in the wild (a locked or + // directory-shaped target), and that is the only path where a temp file can survive. + it('cleans up the temp file when the rename fails', () => { + mkdirSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE), { recursive: true }) + writeFileSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE, 'occupied'), 'x') + + expect(() => + writeGpuFallbackMarker( + userDataPath, + { engagedAt: 1, crashesInWindow: 3, tier: 2 }, + environment + ) + ).toThrow() + + expect(existsSync(join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false) + }) + it('overwrites an existing marker when escalating tiers', () => { writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3, tier: 1 }, environment) writeGpuFallbackMarker(userDataPath, { engagedAt: 2, crashesInWindow: 4, tier: 2 }, environment) diff --git a/src/main/startup/gpu-fallback-tiers.ts b/src/main/startup/gpu-fallback-tiers.ts index 62c8670f99a..e7ac46213bc 100644 --- a/src/main/startup/gpu-fallback-tiers.ts +++ b/src/main/startup/gpu-fallback-tiers.ts @@ -25,6 +25,9 @@ export const MAX_GPU_FALLBACK_TIER: GpuFallbackTier = 2 /** Tier 0 means "no fallback applied to this launch". */ export const NO_GPU_FALLBACK_TIER = 0 +/** A rung, or the hardware path. The only values a launch's current tier may hold. */ +export type GpuFallbackTierOrNone = GpuFallbackTier | typeof NO_GPU_FALLBACK_TIER + export type GpuFallbackSwitch = { name: string; value?: string } const TIER_SWITCHES: Record = {