mirror of
https://github.com/stablyai/orca.git
synced 2026-10-19 08:03:13 +00:00
fix(gpu): stop replaying the whole crash loop after every update
The fallback marker is scoped to appVersion+electronVersion so a new build gets a fresh hardware attempt. That scoping also discarded what we knew about the machine: bundle 3GyOWOzGskzdXqlEG8z4UQ shows an update invalidating the marker, then the full crash burst and relaunch again. With a ladder that is worse -- each update re-walks it from the bottom. Record the strongest tier a machine has needed in a version-independent file. The post-update hardware probe still happens; when it fails the app resumes at the known tier instead of starting over. Monotonic, so a shallower later burst cannot talk a known-bad machine back down a rung, and a hint only -- any read failure degrades to normal escalation.
This commit is contained in:
+18
-2
@@ -104,6 +104,10 @@ import {
|
||||
type GpuFallbackTier
|
||||
} from './startup/gpu-fallback-tiers'
|
||||
import { purgeGpuCaches } from './startup/gpu-cache-purge'
|
||||
import {
|
||||
getResumeGpuFallbackTier,
|
||||
recordGpuFallbackRequiredTier
|
||||
} from './startup/gpu-fallback-required-tier'
|
||||
import { captureGpuInfoSnapshot } from './crash-reporting/gpu-info-snapshot'
|
||||
import {
|
||||
DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
|
||||
@@ -1434,8 +1438,14 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void {
|
||||
if (!result.shouldEngageFallback) {
|
||||
return
|
||||
}
|
||||
const userDataPath = getGpuFallbackUserDataPath()
|
||||
const currentTier = gpuFallbackTierThisLaunch
|
||||
const nextTier = getNextGpuFallbackTier(currentTier)
|
||||
// Why: after an update the build-scoped marker is gone but the machine is not.
|
||||
// Resuming at the tier it already needed is the difference between one relaunch
|
||||
// per update and re-walking the whole ladder every time.
|
||||
const resumeTier =
|
||||
currentTier === NO_GPU_FALLBACK_TIER ? getResumeGpuFallbackTier(userDataPath) : null
|
||||
const nextTier = resumeTier ?? getNextGpuFallbackTier(currentTier)
|
||||
if (currentTier !== NO_GPU_FALLBACK_TIER) {
|
||||
// Why: 20 of 21 crashed launches on the reported machine already carried
|
||||
// gpu_fallback_applied. Without this breadcrumb the failure looked like a fix.
|
||||
@@ -1463,6 +1473,7 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void {
|
||||
exitCode,
|
||||
tier: nextTier,
|
||||
previousTier: currentTier,
|
||||
resumedFromHistory: resumeTier !== null,
|
||||
crashesInWindow: result.crashesInWindow
|
||||
})
|
||||
const engagedAt = Date.now()
|
||||
@@ -1470,7 +1481,6 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void {
|
||||
if (!environment) {
|
||||
return
|
||||
}
|
||||
const userDataPath = getGpuFallbackUserDataPath()
|
||||
try {
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
@@ -1485,6 +1495,12 @@ function handleGpuChildCrash(reason: string, exitCode: number | null): void {
|
||||
console.warn('[gpu-fallback] failed to persist marker:', error)
|
||||
return
|
||||
}
|
||||
// Why: survives the update that invalidates the marker above.
|
||||
const recordedTier = recordGpuFallbackRequiredTier(userDataPath, nextTier, environment, engagedAt)
|
||||
recordCrashBreadcrumb('gpu_fallback_required_tier_recorded', {
|
||||
tier: nextTier,
|
||||
recordedTier: recordedTier ?? null
|
||||
})
|
||||
// Why: a shader cache written by the driver that just crashed replays the same
|
||||
// crash on the next GPU init, defeating the tier we just escalated to.
|
||||
const purge = purgeGpuCaches(userDataPath)
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
GPU_FALLBACK_REQUIRED_TIER_FILE,
|
||||
getResumeGpuFallbackTier,
|
||||
readGpuFallbackRequiredTier,
|
||||
recordGpuFallbackRequiredTier
|
||||
} from './gpu-fallback-required-tier'
|
||||
|
||||
describe('gpu-fallback-required-tier', () => {
|
||||
let userDataPath: string
|
||||
const environment = { appVersion: '1.4.155', electronVersion: '43.1.0' }
|
||||
|
||||
beforeEach(() => {
|
||||
userDataPath = mkdtempSync(join(os.tmpdir(), 'orca-gpu-required-tier-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('round-trips a recorded tier', () => {
|
||||
expect(recordGpuFallbackRequiredTier(userDataPath, 2, environment, 1_000)).toBe(2)
|
||||
expect(readGpuFallbackRequiredTier(userDataPath)).toEqual({
|
||||
schemeVersion: 1,
|
||||
requiredTier: 2,
|
||||
recordedAt: 1_000,
|
||||
appVersion: '1.4.155',
|
||||
electronVersion: '43.1.0'
|
||||
})
|
||||
})
|
||||
|
||||
it('reports no resume tier for a machine with no history', () => {
|
||||
expect(getResumeGpuFallbackTier(userDataPath)).toBeNull()
|
||||
expect(readGpuFallbackRequiredTier(userDataPath)).toBeNull()
|
||||
})
|
||||
|
||||
// Why: this is the whole point — the marker is version-scoped, this is not.
|
||||
it('survives an app and Electron version change', () => {
|
||||
recordGpuFallbackRequiredTier(userDataPath, 2, environment, 1_000)
|
||||
expect(getResumeGpuFallbackTier(userDataPath)).toBe(2)
|
||||
expect(readGpuFallbackRequiredTier(userDataPath)?.appVersion).toBe('1.4.155')
|
||||
})
|
||||
|
||||
// Why: a shallower later burst must not talk a known-bad machine back down a rung.
|
||||
it('is monotonic', () => {
|
||||
recordGpuFallbackRequiredTier(userDataPath, 2, environment, 1_000)
|
||||
expect(recordGpuFallbackRequiredTier(userDataPath, 1, environment, 2_000)).toBe(2)
|
||||
expect(readGpuFallbackRequiredTier(userDataPath)?.requiredTier).toBe(2)
|
||||
expect(readGpuFallbackRequiredTier(userDataPath)?.recordedAt).toBe(1_000)
|
||||
})
|
||||
|
||||
it('raises the recorded tier when escalating', () => {
|
||||
recordGpuFallbackRequiredTier(userDataPath, 1, environment, 1_000)
|
||||
expect(recordGpuFallbackRequiredTier(userDataPath, 2, environment, 2_000)).toBe(2)
|
||||
expect(readGpuFallbackRequiredTier(userDataPath)?.requiredTier).toBe(2)
|
||||
})
|
||||
|
||||
it('leaves no temp file behind', () => {
|
||||
recordGpuFallbackRequiredTier(userDataPath, 2, environment, 1_000)
|
||||
expect(existsSync(join(userDataPath, `${GPU_FALLBACK_REQUIRED_TIER_FILE}.tmp`))).toBe(false)
|
||||
})
|
||||
|
||||
// Why: a hint, not an authority — bad input degrades to normal ladder escalation.
|
||||
it('degrades to no history on corrupt, off-ladder, or wrong-version records', () => {
|
||||
const file = join(userDataPath, GPU_FALLBACK_REQUIRED_TIER_FILE)
|
||||
|
||||
writeFileSync(file, '{ not json')
|
||||
expect(getResumeGpuFallbackTier(userDataPath)).toBeNull()
|
||||
|
||||
writeFileSync(file, JSON.stringify({ schemeVersion: 1, requiredTier: 99, recordedAt: 1 }))
|
||||
expect(getResumeGpuFallbackTier(userDataPath)).toBeNull()
|
||||
|
||||
writeFileSync(file, JSON.stringify({ schemeVersion: 99, requiredTier: 2, recordedAt: 1 }))
|
||||
expect(getResumeGpuFallbackTier(userDataPath)).toBeNull()
|
||||
|
||||
writeFileSync(file, JSON.stringify({ schemeVersion: 1, requiredTier: 2 }))
|
||||
expect(getResumeGpuFallbackTier(userDataPath)).toBeNull()
|
||||
})
|
||||
|
||||
it('reports failure rather than throwing when the record cannot be written', () => {
|
||||
expect(
|
||||
recordGpuFallbackRequiredTier(join(userDataPath, 'missing'), 2, environment, 1_000)
|
||||
).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,112 @@
|
||||
import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { clampGpuFallbackTier, isGpuFallbackTier, type GpuFallbackTier } from './gpu-fallback-tiers'
|
||||
|
||||
/**
|
||||
* Version-independent record of the strongest fallback tier this machine has needed.
|
||||
*
|
||||
* The crash marker is scoped to appVersion+electronVersion so a new build gets a
|
||||
* fresh hardware attempt — deliberate, since a driver or Electron update may fix
|
||||
* the machine. But that scoping also discarded everything we had learned: every
|
||||
* update replayed the entire crash loop from the bottom of the ladder, which is
|
||||
* what made the app repeatedly unusable rather than once-unusable.
|
||||
*
|
||||
* This record survives updates. The post-update hardware probe still happens, but
|
||||
* when it fails the app resumes at the tier it already needed instead of walking
|
||||
* the ladder again.
|
||||
*
|
||||
* It is a hint, not an authority: any read failure degrades to normal escalation,
|
||||
* so a corrupt or unreadable file costs extra relaunches, never correctness.
|
||||
*/
|
||||
|
||||
export const GPU_FALLBACK_REQUIRED_TIER_FILE = 'gpu-fallback-required-tier.json'
|
||||
const GPU_FALLBACK_REQUIRED_TIER_TEMP_FILE = `${GPU_FALLBACK_REQUIRED_TIER_FILE}.tmp`
|
||||
export const GPU_FALLBACK_REQUIRED_TIER_SCHEME_VERSION = 1
|
||||
|
||||
export type GpuFallbackRequiredTierRecord = {
|
||||
schemeVersion: number
|
||||
requiredTier: GpuFallbackTier
|
||||
recordedAt: number
|
||||
/** Build that last needed this tier. Diagnostic only — it never scopes validity. */
|
||||
appVersion: string
|
||||
electronVersion: string
|
||||
}
|
||||
|
||||
function recordPath(userDataPath: string): string {
|
||||
return join(userDataPath, GPU_FALLBACK_REQUIRED_TIER_FILE)
|
||||
}
|
||||
|
||||
export function readGpuFallbackRequiredTier(
|
||||
userDataPath: string
|
||||
): GpuFallbackRequiredTierRecord | null {
|
||||
let parsed: Partial<Record<keyof GpuFallbackRequiredTierRecord, unknown>>
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(recordPath(userDataPath), 'utf-8')) as Partial<
|
||||
Record<keyof GpuFallbackRequiredTierRecord, unknown>
|
||||
>
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (
|
||||
parsed === null ||
|
||||
typeof parsed !== 'object' ||
|
||||
parsed.schemeVersion !== GPU_FALLBACK_REQUIRED_TIER_SCHEME_VERSION ||
|
||||
!isGpuFallbackTier(parsed.requiredTier) ||
|
||||
typeof parsed.recordedAt !== 'number' ||
|
||||
!Number.isFinite(parsed.recordedAt)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
schemeVersion: GPU_FALLBACK_REQUIRED_TIER_SCHEME_VERSION,
|
||||
requiredTier: clampGpuFallbackTier(parsed.requiredTier),
|
||||
recordedAt: parsed.recordedAt,
|
||||
appVersion: typeof parsed.appVersion === 'string' ? parsed.appVersion : '',
|
||||
electronVersion: typeof parsed.electronVersion === 'string' ? parsed.electronVersion : ''
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Raises the recorded tier to `tier`. Monotonic: a machine that once needed tier 2
|
||||
* must not be talked back down to tier 1 by a later, shallower crash burst.
|
||||
* Returns the tier now on disk, or null if it could not be persisted.
|
||||
*/
|
||||
export function recordGpuFallbackRequiredTier(
|
||||
userDataPath: string,
|
||||
tier: GpuFallbackTier,
|
||||
environment: { appVersion: string; electronVersion: string },
|
||||
now: number
|
||||
): GpuFallbackTier | null {
|
||||
const existing = readGpuFallbackRequiredTier(userDataPath)
|
||||
if (existing && existing.requiredTier >= tier) {
|
||||
return existing.requiredTier
|
||||
}
|
||||
const record: GpuFallbackRequiredTierRecord = {
|
||||
schemeVersion: GPU_FALLBACK_REQUIRED_TIER_SCHEME_VERSION,
|
||||
requiredTier: tier,
|
||||
recordedAt: now,
|
||||
appVersion: environment.appVersion,
|
||||
electronVersion: environment.electronVersion
|
||||
}
|
||||
const tempPath = join(userDataPath, GPU_FALLBACK_REQUIRED_TIER_TEMP_FILE)
|
||||
try {
|
||||
writeFileSync(tempPath, JSON.stringify(record))
|
||||
renameSync(tempPath, recordPath(userDataPath))
|
||||
return tier
|
||||
} catch {
|
||||
try {
|
||||
rmSync(tempPath, { force: true })
|
||||
} catch {
|
||||
// best effort; a stray temp file is inert
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tier to resume at when a hardware launch fails and no build-scoped marker exists,
|
||||
* or null when this machine has no history and should start at the bottom.
|
||||
*/
|
||||
export function getResumeGpuFallbackTier(userDataPath: string): GpuFallbackTier | null {
|
||||
return readGpuFallbackRequiredTier(userDataPath)?.requiredTier ?? null
|
||||
}
|
||||
Reference in New Issue
Block a user