mirror of
https://github.com/stablyai/orca.git
synced 2026-10-20 08:03:28 +00:00
Record GPU fallback escalation context in crash reports
Add durable logging when GPU marker writes fail and enhance relaunch telemetry with escalation state. Ensure GPU identity snapshots reach crash details for triage, and gracefully handle locked cache directories.
This commit is contained in:
@@ -15,6 +15,7 @@ import {
|
||||
type ProcessGoneCrashEvent
|
||||
} from './process-gone-recorder'
|
||||
import { _resetTracerForTests, setActiveSink, type TracerSink } from '../observability/tracer'
|
||||
import { setGpuInfoSnapshotForTesting } from './gpu-info-snapshot'
|
||||
|
||||
type CapturingSink = TracerSink & { records: unknown[]; flushMock: ReturnType<typeof vi.fn> }
|
||||
|
||||
@@ -55,6 +56,7 @@ afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
_resetTracerForTests()
|
||||
clearCrashBreadcrumbsForTest()
|
||||
setGpuInfoSnapshotForTesting(null)
|
||||
})
|
||||
|
||||
describe('recordProcessGoneCrash', () => {
|
||||
@@ -334,6 +336,76 @@ describe('recordProcessGoneCrash', () => {
|
||||
await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(4))
|
||||
})
|
||||
|
||||
// Why: driver identity is the whole point of the GPU snapshot — if it stops reaching
|
||||
// crash details, triage silently loses the vendor/device data it was added for.
|
||||
it('attaches the GPU identity snapshot to GPU-child crash details', async () => {
|
||||
const record = vi.fn().mockResolvedValue({ id: 'report' })
|
||||
setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' })
|
||||
|
||||
recordProcessGoneCrash(
|
||||
{ record } as never,
|
||||
event({
|
||||
source: 'child',
|
||||
processType: 'GPU',
|
||||
gpuFallbackActive: true,
|
||||
details: { type: 'GPU', gpuFallbackTier: 2 }
|
||||
}),
|
||||
new ProcessGoneDedupe()
|
||||
)
|
||||
|
||||
await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1))
|
||||
expect(record.mock.calls[0][0].details).toEqual(
|
||||
expect.objectContaining({ gpuInfoAvailable: true, gpuVendorId: '0x10de' })
|
||||
)
|
||||
})
|
||||
|
||||
it('attaches the GPU identity snapshot to renderer crash details', async () => {
|
||||
const record = vi.fn().mockResolvedValue({ id: 'report' })
|
||||
setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' })
|
||||
|
||||
recordProcessGoneCrash({ record } as never, event(), new ProcessGoneDedupe())
|
||||
|
||||
await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1))
|
||||
expect(record.mock.calls[0][0].details).toEqual(
|
||||
expect.objectContaining({ gpuInfoAvailable: true, gpuVendorId: '0x10de' })
|
||||
)
|
||||
})
|
||||
|
||||
// Why: every other child type would just pad the report.
|
||||
it('omits the GPU identity snapshot for non-GPU child crashes', async () => {
|
||||
const record = vi.fn().mockResolvedValue({ id: 'report' })
|
||||
setGpuInfoSnapshotForTesting({ gpuInfoAvailable: true, gpuVendorId: '0x10de' })
|
||||
|
||||
recordProcessGoneCrash(
|
||||
{ record } as never,
|
||||
event({ source: 'child', processType: 'Utility', details: { type: 'Utility' } }),
|
||||
new ProcessGoneDedupe()
|
||||
)
|
||||
|
||||
await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1))
|
||||
expect(record.mock.calls[0][0].details).not.toHaveProperty('gpuVendorId')
|
||||
})
|
||||
|
||||
// Why: the reworded crash dialog keys off a numeric gpuFallbackTier; if sanitization
|
||||
// ever dropped or stringified it, isGpuFallbackCrashReport would silently never match.
|
||||
it('preserves gpuFallbackTier as a number through sanitization', async () => {
|
||||
const record = vi.fn().mockResolvedValue({ id: 'report' })
|
||||
|
||||
recordProcessGoneCrash(
|
||||
{ record } as never,
|
||||
event({
|
||||
source: 'child',
|
||||
processType: 'GPU',
|
||||
gpuFallbackActive: true,
|
||||
details: { type: 'GPU', gpuFallbackTier: 2 }
|
||||
}),
|
||||
new ProcessGoneDedupe()
|
||||
)
|
||||
|
||||
await vi.waitFor(() => expect(record).toHaveBeenCalledTimes(1))
|
||||
expect(record.mock.calls[0][0].details.gpuFallbackTier).toBe(2)
|
||||
})
|
||||
|
||||
it('allows the same renderer crash to retry after persistence fails', async () => {
|
||||
const record = vi
|
||||
.fn()
|
||||
|
||||
+17
-2
@@ -101,7 +101,7 @@ import {
|
||||
NO_GPU_FALLBACK_TIER,
|
||||
getGpuFallbackTierSwitches,
|
||||
resolveGpuFallbackEscalation,
|
||||
type GpuFallbackTier
|
||||
type GpuFallbackTierOrNone
|
||||
} from './startup/gpu-fallback-tiers'
|
||||
import { purgeGpuCaches } from './startup/gpu-cache-purge'
|
||||
import {
|
||||
@@ -310,7 +310,7 @@ const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({
|
||||
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD
|
||||
})
|
||||
// Why: 0 = hardware path. A non-zero tier is the rung of the fallback ladder this launch is running.
|
||||
let gpuFallbackTierThisLaunch: GpuFallbackTier | typeof NO_GPU_FALLBACK_TIER = NO_GPU_FALLBACK_TIER
|
||||
let gpuFallbackTierThisLaunch: GpuFallbackTierOrNone = NO_GPU_FALLBACK_TIER
|
||||
const GPU_INFO_CAPTURE_TIMEOUT_MS = 10_000
|
||||
let localPtyStartupReady: Promise<void> = Promise.resolve()
|
||||
let localPtyProviderStartupReady: Promise<void> = Promise.resolve()
|
||||
@@ -1493,6 +1493,15 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void {
|
||||
)
|
||||
} catch (error) {
|
||||
console.warn('[gpu-fallback] failed to persist marker:', error)
|
||||
// Why: this abandons the tier record, the cache purge and the relaunch; console
|
||||
// output reaches no support bundle, so the escalation would vanish without trace.
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_marker_write_failed', {
|
||||
reason,
|
||||
exitCode,
|
||||
tier: nextTier,
|
||||
previousTier: currentTier,
|
||||
errorMessage: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
return
|
||||
}
|
||||
// Why: survives the update that invalidates the marker above.
|
||||
@@ -1509,11 +1518,17 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void {
|
||||
failed: purge.failed.join(',')
|
||||
})
|
||||
isQuitting = true
|
||||
// Why: the breadcrumbs above are in-memory only and this process exits next, so
|
||||
// app_relaunch_requested is the sole durable record of what the escalation did.
|
||||
relaunchApp('gpu-fallback', {
|
||||
processReason: reason,
|
||||
exitCode,
|
||||
tier: nextTier,
|
||||
previousTier: currentTier,
|
||||
resumedFromHistory,
|
||||
recordedTier: recordedTier ?? null,
|
||||
cachesPurged: purge.removed.join(','),
|
||||
cachePurgeFailed: purge.failed.join(','),
|
||||
crashesInWindow: result.crashesInWindow
|
||||
})
|
||||
app.exit(0)
|
||||
|
||||
@@ -1,17 +1,37 @@
|
||||
import type * as NodeFs from 'node:fs'
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { basename, join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { GPU_CACHE_DIRECTORY_NAMES, purgeGpuCaches } from './gpu-cache-purge'
|
||||
|
||||
// Why: a locked cache directory is a Windows-only failure in the wild; this fakes it
|
||||
// so the `failed` bookkeeping is covered without a platform-specific (or root-skipped) test.
|
||||
const { unremovableDirectory } = vi.hoisted(() => ({ unremovableDirectory: { name: '' } }))
|
||||
|
||||
vi.mock('node:fs', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof NodeFs>()
|
||||
return {
|
||||
...actual,
|
||||
rmSync: (target: Parameters<typeof actual.rmSync>[0], options?: object) => {
|
||||
if (unremovableDirectory.name && basename(String(target)) === unremovableDirectory.name) {
|
||||
throw Object.assign(new Error('EBUSY: resource busy or locked'), { code: 'EBUSY' })
|
||||
}
|
||||
return actual.rmSync(target, options)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe('purgeGpuCaches', () => {
|
||||
let userDataPath: string
|
||||
|
||||
beforeEach(() => {
|
||||
userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-cache-purge-test-'))
|
||||
unremovableDirectory.name = ''
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
unremovableDirectory.name = ''
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
@@ -47,4 +67,28 @@ describe('purgeGpuCaches', () => {
|
||||
it('does not throw for a missing userData directory', () => {
|
||||
expect(() => purgeGpuCaches(join(userDataPath, 'absent'))).not.toThrow()
|
||||
})
|
||||
|
||||
it('reports a locked cache directory as failed and still purges the rest', () => {
|
||||
for (const name of GPU_CACHE_DIRECTORY_NAMES) {
|
||||
mkdirSync(join(userDataPath, name), { recursive: true })
|
||||
}
|
||||
unremovableDirectory.name = 'ShaderCache'
|
||||
|
||||
const result = purgeGpuCaches(userDataPath)
|
||||
|
||||
expect(result.failed).toEqual(['ShaderCache'])
|
||||
expect(result.removed).toEqual(
|
||||
GPU_CACHE_DIRECTORY_NAMES.filter((name) => name !== 'ShaderCache')
|
||||
)
|
||||
expect(existsSync(join(userDataPath, 'ShaderCache'))).toBe(true)
|
||||
expect(existsSync(join(userDataPath, 'GPUCache'))).toBe(false)
|
||||
})
|
||||
|
||||
// Why: a locked directory must not abort the relaunch it precedes.
|
||||
it('does not throw when a cache directory cannot be removed', () => {
|
||||
mkdirSync(join(userDataPath, 'GPUCache'), { recursive: true })
|
||||
unremovableDirectory.name = 'GPUCache'
|
||||
|
||||
expect(() => purgeGpuCaches(userDataPath)).not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -49,6 +49,40 @@ describe('gpu-fallback-marker', () => {
|
||||
expect(existsSync(join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false)
|
||||
})
|
||||
|
||||
// Why: the caller treats a throw as "escalation abandoned", so a failed write must
|
||||
// surface rather than leave a half-written marker or a stray temp file behind.
|
||||
it('throws and leaves no temp file behind when the atomic write fails', () => {
|
||||
const missingUserDataPath = join(userDataPath, 'absent')
|
||||
|
||||
expect(() =>
|
||||
writeGpuFallbackMarker(
|
||||
missingUserDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 3, tier: 2 },
|
||||
environment
|
||||
)
|
||||
).toThrow()
|
||||
|
||||
expect(existsSync(join(missingUserDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false)
|
||||
expect(existsSync(join(missingUserDataPath, GPU_FALLBACK_MARKER_FILE))).toBe(false)
|
||||
})
|
||||
|
||||
// Why: the rename is the step that actually fails in the wild (a locked or
|
||||
// directory-shaped target), and that is the only path where a temp file can survive.
|
||||
it('cleans up the temp file when the rename fails', () => {
|
||||
mkdirSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE), { recursive: true })
|
||||
writeFileSync(join(userDataPath, GPU_FALLBACK_MARKER_FILE, 'occupied'), 'x')
|
||||
|
||||
expect(() =>
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: 1, crashesInWindow: 3, tier: 2 },
|
||||
environment
|
||||
)
|
||||
).toThrow()
|
||||
|
||||
expect(existsSync(join(userDataPath, `${GPU_FALLBACK_MARKER_FILE}.tmp`))).toBe(false)
|
||||
})
|
||||
|
||||
it('overwrites an existing marker when escalating tiers', () => {
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 1, crashesInWindow: 3, tier: 1 }, environment)
|
||||
writeGpuFallbackMarker(userDataPath, { engagedAt: 2, crashesInWindow: 4, tier: 2 }, environment)
|
||||
|
||||
@@ -25,6 +25,9 @@ export const MAX_GPU_FALLBACK_TIER: GpuFallbackTier = 2
|
||||
/** Tier 0 means "no fallback applied to this launch". */
|
||||
export const NO_GPU_FALLBACK_TIER = 0
|
||||
|
||||
/** A rung, or the hardware path. The only values a launch's current tier may hold. */
|
||||
export type GpuFallbackTierOrNone = GpuFallbackTier | typeof NO_GPU_FALLBACK_TIER
|
||||
|
||||
export type GpuFallbackSwitch = { name: string; value?: string }
|
||||
|
||||
const TIER_SWITCHES: Record<GpuFallbackTier, readonly GpuFallbackSwitch[]> = {
|
||||
|
||||
Reference in New Issue
Block a user