mirror of
https://github.com/stablyai/orca.git
synced 2026-10-19 08:03:13 +00:00
fix(crash-reporting): stop suppressing GPU crashes that the fallback failed to fix
GPU child deaths are classified as recoverable Chromium churn, so a machine whose GPU crashes 4-5 times per launch produced only process_gone_suppressed breadcrumbs and never a user-visible anything. On the reported machine 20 of 21 crashed launches already carried gpu_fallback_applied and still died. A GPU crash while a fallback tier is already applied is not churn - the remediation failed - so it now records a real crash report and reaches the existing startup crash dialog. App-shutdown teardown stays suppressed, since quit tears the GPU child down noisily by design. GPU and renderer reports also carry the driver identity snapshot, and the dialog says the app is running with reduced graphics acceleration rather than claiming Orca closed unexpectedly - it did not, it relaunched itself degraded. Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
@@ -161,6 +161,16 @@ describe('shouldRecordProcessGoneCrash', () => {
|
||||
expectedTeardown: 'none'
|
||||
})
|
||||
).toBe(false)
|
||||
expect(
|
||||
shouldRecordProcessGoneCrash({
|
||||
source: 'child',
|
||||
processType: 'gpu',
|
||||
reason: 'crashed',
|
||||
exitCode: -2147483645,
|
||||
expectedTeardown: 'none',
|
||||
gpuFallbackActive: false
|
||||
})
|
||||
).toBe(false)
|
||||
expect(
|
||||
shouldRecordProcessGoneCrash({
|
||||
source: 'child',
|
||||
@@ -336,6 +346,60 @@ describe('shouldRecordProcessGoneCrash', () => {
|
||||
})
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
// Why: suppressing these is why an ineffective GPU fallback stayed invisible —
|
||||
// 20 of 21 crashed launches on the reported machine already ran the fallback.
|
||||
it('records GPU child crashes once the GPU fallback is already applied', () => {
|
||||
for (const reason of ['crashed', 'abnormal-exit'] as const) {
|
||||
expect(
|
||||
shouldRecordProcessGoneCrash({
|
||||
source: 'child',
|
||||
processType: 'GPU',
|
||||
reason,
|
||||
exitCode: -2147483645,
|
||||
expectedTeardown: 'none',
|
||||
gpuFallbackActive: true
|
||||
})
|
||||
).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps suppressing GPU exits shaped like teardown even under the fallback', () => {
|
||||
expect(
|
||||
shouldRecordProcessGoneCrash({
|
||||
source: 'child',
|
||||
processType: 'gpu',
|
||||
reason: 'killed',
|
||||
exitCode: 15,
|
||||
expectedTeardown: 'none',
|
||||
gpuFallbackActive: true
|
||||
})
|
||||
).toBe(false)
|
||||
expect(
|
||||
shouldRecordProcessGoneCrash({
|
||||
source: 'child',
|
||||
processType: 'gpu',
|
||||
reason: 'crashed',
|
||||
exitCode: -2147483645,
|
||||
expectedTeardown: 'app-shutdown',
|
||||
gpuFallbackActive: true
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('does not widen non-GPU child suppression when the fallback is active', () => {
|
||||
expect(
|
||||
shouldRecordProcessGoneCrash({
|
||||
source: 'child',
|
||||
processType: 'Utility',
|
||||
serviceName: 'network.mojom.NetworkService',
|
||||
reason: 'crashed',
|
||||
exitCode: -1,
|
||||
expectedTeardown: 'none',
|
||||
gpuFallbackActive: true
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('shouldRecoverRendererAfterProcessGone', () => {
|
||||
|
||||
@@ -24,12 +24,16 @@ function isRecoverableChromiumChildProcess({
|
||||
source,
|
||||
processType,
|
||||
serviceName,
|
||||
reason
|
||||
reason,
|
||||
expectedTeardown,
|
||||
gpuFallbackActive
|
||||
}: {
|
||||
source: ProcessGoneSource
|
||||
processType?: string
|
||||
serviceName?: string
|
||||
reason: string
|
||||
expectedTeardown: ExpectedTeardownScope
|
||||
gpuFallbackActive?: boolean
|
||||
}): boolean {
|
||||
if (source !== 'child') {
|
||||
return false
|
||||
@@ -39,7 +43,14 @@ function isRecoverableChromiumChildProcess({
|
||||
}
|
||||
const normalizedProcessType = processType?.toLowerCase()
|
||||
if (normalizedProcessType && RECOVERABLE_CHILD_PROCESS_TYPES.has(normalizedProcessType)) {
|
||||
return true
|
||||
// Why: a GPU crash while the GPU fallback is already applied is not churn —
|
||||
// the remediation failed, and suppressing it is why this was invisible. Quit
|
||||
// still tears the GPU child down noisily, so app-shutdown stays suppressed.
|
||||
return !(
|
||||
gpuFallbackActive === true &&
|
||||
normalizedProcessType === 'gpu' &&
|
||||
expectedTeardown !== 'app-shutdown'
|
||||
)
|
||||
}
|
||||
return (
|
||||
normalizedProcessType === 'utility' &&
|
||||
@@ -54,7 +65,8 @@ export function shouldRecordProcessGoneCrash({
|
||||
serviceName,
|
||||
reason,
|
||||
exitCode,
|
||||
expectedTeardown
|
||||
expectedTeardown,
|
||||
gpuFallbackActive
|
||||
}: {
|
||||
source: ProcessGoneSource
|
||||
processType?: string
|
||||
@@ -62,10 +74,20 @@ export function shouldRecordProcessGoneCrash({
|
||||
reason: string
|
||||
exitCode: number | null
|
||||
expectedTeardown: ExpectedTeardownScope
|
||||
gpuFallbackActive?: boolean
|
||||
}): boolean {
|
||||
// Why: GPU, Network Service, and Audio Service exits are recoverable Chromium
|
||||
// child-process churn; treating them as app crashes creates noisy user prompts.
|
||||
if (isRecoverableChromiumChildProcess({ source, processType, serviceName, reason })) {
|
||||
if (
|
||||
isRecoverableChromiumChildProcess({
|
||||
source,
|
||||
processType,
|
||||
serviceName,
|
||||
reason,
|
||||
expectedTeardown,
|
||||
gpuFallbackActive
|
||||
})
|
||||
) {
|
||||
return false
|
||||
}
|
||||
// Why: Electron reports intentional reload/update/quit teardown as `killed`.
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
processGoneDedupe,
|
||||
type ProcessGoneDedupe
|
||||
} from './process-gone-dedupe'
|
||||
import { getGpuInfoSnapshot } from './gpu-info-snapshot'
|
||||
import { getMainProcessLifecycleIdentity } from './main-process-lifecycle-identity'
|
||||
import { flushActiveSink, startSpan } from '../observability/tracer'
|
||||
|
||||
@@ -28,6 +29,8 @@ export type ProcessGoneCrashEvent = {
|
||||
exitCode: number | null
|
||||
expectedTeardown: ExpectedTeardownScope
|
||||
details: Record<string, unknown>
|
||||
/** True when this launch already runs a GPU fallback tier, so GPU deaths stop counting as churn. */
|
||||
gpuFallbackActive?: boolean
|
||||
}
|
||||
|
||||
type CrashReportRecorderStore = Pick<CrashReportStore, 'record'>
|
||||
@@ -65,7 +68,8 @@ export function recordProcessGoneCrash(
|
||||
typeof event.details.serviceName === 'string' ? event.details.serviceName : undefined,
|
||||
reason: event.reason,
|
||||
exitCode: event.exitCode,
|
||||
expectedTeardown: event.expectedTeardown
|
||||
expectedTeardown: event.expectedTeardown,
|
||||
gpuFallbackActive: event.gpuFallbackActive
|
||||
})
|
||||
) {
|
||||
recordDurableCrashBreadcrumb('process_gone_suppressed', processGoneBreadcrumbData(event))
|
||||
@@ -86,8 +90,15 @@ export function recordProcessGoneCrash(
|
||||
return
|
||||
}
|
||||
const mainProcessLifecycle = getMainProcessLifecycleIdentity()
|
||||
// Why: GPU and renderer deaths are the ones triage needs driver identity for;
|
||||
// every other child type would just pad the report.
|
||||
const gpuIdentity =
|
||||
event.source === 'renderer' || event.processType.toLowerCase() === 'gpu'
|
||||
? (getGpuInfoSnapshot() ?? {})
|
||||
: {}
|
||||
const crashDetails = buildProcessGoneCrashDetails({
|
||||
...event.details,
|
||||
...gpuIdentity,
|
||||
...mainProcessLifecycle
|
||||
})
|
||||
const breadcrumbs = getCrashBreadcrumbSnapshot()
|
||||
|
||||
@@ -15,6 +15,7 @@ import { Label } from '@/components/ui/label'
|
||||
import { useMountedRef } from '@/hooks/useMountedRef'
|
||||
import {
|
||||
formatCrashReportText,
|
||||
isGpuFallbackCrashReport,
|
||||
isReactErrorBoundaryReport,
|
||||
type CrashReportDiagnosticBundle,
|
||||
type CrashReportRecord
|
||||
@@ -34,17 +35,23 @@ function formatSummary(report: CrashReportRecord): string {
|
||||
const surface = typeof report.details.surface === 'string' ? report.details.surface : null
|
||||
return surface ? `React render error in ${surface}` : 'React render error'
|
||||
}
|
||||
return `${report.processType} ${report.reason}${
|
||||
const summary = `${report.processType} ${report.reason}${
|
||||
report.exitCode === null ? '' : ` (exit ${report.exitCode})`
|
||||
}`
|
||||
return isGpuFallbackCrashReport(report)
|
||||
? `${summary} — graphics fallback tier ${String(report.details.gpuFallbackTier)}`
|
||||
: summary
|
||||
}
|
||||
|
||||
function getDialogTitle(report: CrashReportRecord | null): string {
|
||||
if (!report) {
|
||||
return 'Report a crash'
|
||||
}
|
||||
return report && isReactErrorBoundaryReport(report)
|
||||
? 'Orca hit a recoverable UI error'
|
||||
if (isReactErrorBoundaryReport(report)) {
|
||||
return 'Orca hit a recoverable UI error'
|
||||
}
|
||||
return isGpuFallbackCrashReport(report)
|
||||
? 'Orca is running with reduced graphics acceleration'
|
||||
: 'Orca closed unexpectedly'
|
||||
}
|
||||
|
||||
@@ -52,8 +59,11 @@ function getDialogDescription(report: CrashReportRecord | null): string {
|
||||
if (!report) {
|
||||
return 'Send a privacy-safe crash report. Recent redacted diagnostic logs are included when available.'
|
||||
}
|
||||
return report && isReactErrorBoundaryReport(report)
|
||||
? 'Send a privacy-safe diagnostic report to help us understand the failed UI surface.'
|
||||
if (isReactErrorBoundaryReport(report)) {
|
||||
return 'Send a privacy-safe diagnostic report to help us understand the failed UI surface.'
|
||||
}
|
||||
return isGpuFallbackCrashReport(report)
|
||||
? "Your graphics driver keeps crashing Orca's GPU process, so Orca disabled hardware acceleration and restarted. Sending this privacy-safe report tells us which driver is affected."
|
||||
: 'Send a privacy-safe diagnostic report to help us understand what happened.'
|
||||
}
|
||||
|
||||
@@ -61,8 +71,11 @@ function getNotesPlaceholder(report: CrashReportRecord | null): string {
|
||||
if (!report) {
|
||||
return 'Optional: what happened?'
|
||||
}
|
||||
return report && isReactErrorBoundaryReport(report)
|
||||
? 'Optional: what were you doing before this UI error?'
|
||||
if (isReactErrorBoundaryReport(report)) {
|
||||
return 'Optional: what were you doing before this UI error?'
|
||||
}
|
||||
return isGpuFallbackCrashReport(report)
|
||||
? 'Optional: does anything still look wrong after the restart?'
|
||||
: 'Optional: what were you doing before Orca closed?'
|
||||
}
|
||||
|
||||
|
||||
@@ -163,6 +163,21 @@ export function isReactErrorBoundaryReport(report: CrashReportRecord): boolean {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* A GPU child crash that happened while a GPU fallback tier was already applied.
|
||||
*
|
||||
* Orca is still running here — it relaunched itself into a degraded GPU path —
|
||||
* so the dialog must not claim the app closed unexpectedly.
|
||||
*/
|
||||
export function isGpuFallbackCrashReport(report: CrashReportRecord): boolean {
|
||||
return (
|
||||
report.source === 'child' &&
|
||||
report.processType.toLowerCase() === 'gpu' &&
|
||||
typeof report.details.gpuFallbackTier === 'number' &&
|
||||
report.details.gpuFallbackTier > 0
|
||||
)
|
||||
}
|
||||
|
||||
export function sanitizeCrashReportString(
|
||||
value: string,
|
||||
maxLength = MAX_STRING_DETAIL_LENGTH
|
||||
|
||||
Reference in New Issue
Block a user