fix(crash-reporting): Windows native crash reports say what the fault touched and which code ran (#30691)

* fix(crash-reporting): record what a native access violation touched and the bytes that ran

Windows renderer AV reports carried only the faulting module offset. Two
recent ones (Orca.exe+0x23d3f88, +0x21674c2) map in official Electron
symbols to stores into memory the preceding instructions had just written,
which an intact binary on a healthy CPU cannot fault on. Without the access
kind, fault address and executed bytes, an engine bug, a corrupted code page
and unstable hardware look identical.

Read them from the Crashpad dump: AV access kind and target, the target's
distance from the faulting thread's stack pointer (x64), and 16 bytes at
the instruction pointer from the captured memory list.

* fix(crash-reporting): capture code bytes only inside a loaded module

They exist to diff against the shipped binary; bytes from JIT or other
anonymous memory add nothing and could carry page contents.

* fix(crash-reporting): capture code bytes only from Windows dumps

On macOS and Linux Crashpad records the touched data address as the exception
address, so bytes read there would be module data, not the faulting code.

---------

Co-authored-by: m4air <m4air@Mac.localdomain>

src/main/crash-reporting/minidump-crash-signature.ts

@@ -14,9 +14,14 @@
 import { findStream, isMinidump, MinidumpView, type MinidumpSource } from './minidump-stream-reader'
 import { readCrashpadAnnotations } from './minidump-crashpad-annotations'
 import { findEmbeddedCheckMessage } from './minidump-embedded-check'
+import { readMinidumpFaultContext, type MinidumpFaultContext } from './minidump-fault-context'
 
 const STREAM_TYPE_MODULE_LIST = 4
 const STREAM_TYPE_EXCEPTION = 6
+const STREAM_TYPE_SYSTEM_INFO = 7
+// MINIDUMP_SYSTEM_INFO.PlatformId; Crashpad writes its own ids for macOS and Linux.
+const SYSTEM_INFO_PLATFORM_ID_OFFSET = 20
+const VER_PLATFORM_WIN32_NT = 2
 
 const MODULE_RECORD_SIZE = 108
 // 8x headroom over a measured 1042-image macOS renderer, whose whole list a
@@ -45,6 +50,7 @@ export type MinidumpCrashSignature = {
   /** Module whose image range contains `exceptionAddress`. */
   readonly faultingModule?: string
   readonly faultingModuleOffset?: string
+  readonly fault?: MinidumpFaultContext
   /** Allowlisted Crashpad annotations, verbatim. */
   readonly annotations: Readonly<Record<string, string>>
 }
@@ -79,6 +85,14 @@ async function readModules(view: MinidumpView): Promise<ModuleRecord[]> {
   return modules
 }
 
+async function isWindowsDump(view: MinidumpView): Promise<boolean> {
+  const stream = await findStream(view, STREAM_TYPE_SYSTEM_INFO)
+  return (
+    stream !== null &&
+    (await view.u32(stream.rva + SYSTEM_INFO_PLATFORM_ID_OFFSET)) === VER_PLATFORM_WIN32_NT
+  )
+}
+
 function moduleBasename(modulePath: string): string {
   const separator = Math.max(modulePath.lastIndexOf('/'), modulePath.lastIndexOf('\\'))
   return separator >= 0 ? modulePath.slice(separator + 1) : modulePath
@@ -187,6 +201,12 @@ export async function parseMinidumpCrashSignature(
         signature.faultingModule = faulting.name
         signature.faultingModuleOffset = faulting.offset
       }
+      // Code bytes only from a loaded module in a Windows dump: they exist to diff against the
+      // shipped binary. JIT memory adds nothing, and on macOS/Linux Crashpad records the touched
+      // data address here, not the instruction pointer.
+      signature.fault = await readMinidumpFaultContext(view, exception.rva, code, address, {
+        captureCodeBytes: faulting !== undefined && (await isWindowsDump(view))
+      })
     }
   }
 
@@ -222,6 +242,19 @@ export function minidumpSignatureDetails(
   if (signature.faultingModuleOffset) {
     details.minidumpFaultingModuleOffset = signature.faultingModuleOffset
   }
+  const fault = signature.fault
+  if (fault?.access) {
+    details.minidumpExceptionAccess = fault.access
+  }
+  if (fault?.faultAddress) {
+    details.minidumpFaultAddress = fault.faultAddress
+  }
+  if (fault?.faultStackPointerDelta !== undefined) {
+    details.minidumpFaultStackPointerDelta = fault.faultStackPointerDelta
+  }
+  if (fault?.codeBytes) {
+    details.minidumpCodeBytes = fault.codeBytes
+  }
   for (const [key, value] of Object.entries(signature.annotations)) {
     if (key === 'LOG_FATAL' || key === 'abort-message' || key === 'ptype') {
       continue

src/main/crash-reporting/minidump-fault-context.test.ts

@@ -0,0 +1,202 @@
+import { describe, expect, it } from 'vitest'
+import { minidumpSignatureDetails, parseMinidumpCrashSignature } from './minidump-crash-signature'
+
+const STREAM_TYPE_MODULE_LIST = 4
+const STREAM_TYPE_MEMORY_LIST = 5
+const STREAM_TYPE_SYSTEM_INFO = 7
+const VER_PLATFORM_WIN32_NT = 2
+const CRASHPAD_PLATFORM_MACOS = 0x8101
+const STREAM_TYPE_EXCEPTION = 6
+const CONTEXT_SIZE = 0x4d0
+// Bytes Electron 43.7.8 x64 ships at electron.exe+0x23d3f88: movb %dil, 0x88(%r12) twice.
+const CODE = Buffer.from('4188bc24880000004188bc2490000000', 'hex')
+
+type FaultDump = {
+  code: number
+  rip: bigint
+  params?: bigint[]
+  rsp?: bigint
+  contextFlags?: number
+  memory?: { start: bigint; bytes: Buffer }[]
+  modules?: { base: bigint; size: number; name: string }[]
+  platformId?: number
+}
+
+/** Exception stream, x64 CONTEXT, MemoryList and ModuleList laid out as Crashpad writes them. */
+function buildFaultDump(options: FaultDump): Buffer {
+  const streamCount =
+    1 +
+    (options.memory ? 1 : 0) +
+    (options.modules ? 1 : 0) +
+    (options.platformId === undefined ? 0 : 1)
+  let cursor = 32 + streamCount * 12
+  const regions: Buffer[] = []
+  const append = (buf: Buffer): number => {
+    const rva = cursor
+    regions.push(buf)
+    cursor += buf.length
+    return rva
+  }
+
+  const context = Buffer.alloc(CONTEXT_SIZE)
+  context.writeUInt32LE(options.contextFlags ?? 0x0010001f, 0x30)
+  context.writeBigUInt64LE(options.rsp ?? 0n, 0x98)
+  const contextRva = append(context)
+
+  const exception = Buffer.alloc(168)
+  exception.writeUInt32LE(7, 0)
+  exception.writeUInt32LE(options.code, 8)
+  exception.writeBigUInt64LE(options.rip, 24)
+  const params = options.params ?? []
+  exception.writeUInt32LE(params.length, 32)
+  params.forEach((value, index) => exception.writeBigUInt64LE(value, 40 + index * 8))
+  exception.writeUInt32LE(CONTEXT_SIZE, 160)
+  exception.writeUInt32LE(contextRva, 164)
+  const streams = [{ type: STREAM_TYPE_EXCEPTION, size: 168, rva: append(exception) }]
+
+  if (options.memory) {
+    const dataRvas = options.memory.map((range) => append(range.bytes))
+    const list = Buffer.alloc(4 + options.memory.length * 16)
+    list.writeUInt32LE(options.memory.length, 0)
+    options.memory.forEach((range, index) => {
+      list.writeBigUInt64LE(range.start, 4 + index * 16)
+      list.writeUInt32LE(range.bytes.length, 12 + index * 16)
+      list.writeUInt32LE(dataRvas[index], 16 + index * 16)
+    })
+    streams.push({ type: STREAM_TYPE_MEMORY_LIST, size: list.length, rva: append(list) })
+  }
+
+  if (options.modules) {
+    const nameRvas = options.modules.map((module) => {
+      const name = Buffer.from(module.name, 'utf16le')
+      const length = Buffer.alloc(4)
+      length.writeUInt32LE(name.length, 0)
+      return append(Buffer.concat([length, name]))
+    })
+    const list = Buffer.alloc(4 + options.modules.length * 108)
+    list.writeUInt32LE(options.modules.length, 0)
+    options.modules.forEach((module, index) => {
+      list.writeBigUInt64LE(module.base, 4 + index * 108)
+      list.writeUInt32LE(module.size, 12 + index * 108)
+      list.writeUInt32LE(nameRvas[index], 24 + index * 108)
+    })
+    streams.push({ type: STREAM_TYPE_MODULE_LIST, size: list.length, rva: append(list) })
+  }
+
+  if (options.platformId !== undefined) {
+    const systemInfo = Buffer.alloc(56)
+    systemInfo.writeUInt32LE(options.platformId, 20)
+    streams.push({ type: STREAM_TYPE_SYSTEM_INFO, size: 56, rva: append(systemInfo) })
+  }
+
+  const header = Buffer.alloc(32 + streamCount * 12)
+  header.writeUInt32LE(0x504d444d, 0)
+  header.writeUInt32LE(streamCount, 8)
+  header.writeUInt32LE(32, 12)
+  streams.forEach((stream, index) => {
+    header.writeUInt32LE(stream.type, 32 + index * 12)
+    header.writeUInt32LE(stream.size, 36 + index * 12)
+    header.writeUInt32LE(stream.rva, 40 + index * 12)
+  })
+  return Buffer.concat([header, ...regions])
+}
+
+describe('minidump fault context', () => {
+  it('reports a write AV into the faulting frame and the executed bytes', async () => {
+    const rip = 0x7ff6_3941_3f88n
+    const rsp = 0x8a_3a03_d000n
+    const dump = buildFaultDump({
+      code: 0xc0000005,
+      rip,
+      rsp,
+      params: [1n, rsp + 0x3c8n],
+      memory: [
+        { start: rip - 1n, bytes: Buffer.concat([Buffer.from([0x80]), CODE, Buffer.from([0x4c])]) }
+      ],
+      platformId: VER_PLATFORM_WIN32_NT,
+      modules: [{ base: rip - 0x23d3f88n, size: 0x9000000, name: 'C:\\Orca\\Orca.exe' }]
+    })
+
+    const signature = await parseMinidumpCrashSignature(dump)
+    const details = minidumpSignatureDetails(signature!)
+
+    expect(details).toMatchObject({
+      minidumpExceptionCode: '0xc0000005',
+      minidumpExceptionAccess: 'write',
+      minidumpFaultAddress: '0x8a3a03d3c8',
+      minidumpFaultStackPointerDelta: 0x3c8,
+      minidumpCodeBytes: CODE.toString('hex')
+    })
+  })
+
+  it('omits the stack delta for a fault far from the stack and for non-x64 contexts', async () => {
+    const far = await parseMinidumpCrashSignature(
+      buildFaultDump({ code: 0xc0000005, rip: 0x1000n, rsp: 0x10_0000_0000n, params: [0n, 0x20n] })
+    )
+    expect(far?.fault).toEqual({ access: 'read', faultAddress: '0x20' })
+
+    const arm = await parseMinidumpCrashSignature(
+      buildFaultDump({
+        code: 0xc0000005,
+        rip: 0x1000n,
+        rsp: 0x30n,
+        contextFlags: 0x00400000,
+        params: [8n, 0x20n]
+      })
+    )
+    expect(arm?.fault).toEqual({ access: 'execute', faultAddress: '0x20' })
+  })
+
+  it('keeps only code bytes for non-AV exceptions and truncates at the captured range', async () => {
+    const signature = await parseMinidumpCrashSignature(
+      buildFaultDump({
+        code: 0x80000003,
+        rip: 0x5000n,
+        params: [1n, 0x20n],
+        memory: [
+          { start: 0x100n, bytes: Buffer.from('aa', 'hex') },
+          { start: 0x4ffen, bytes: Buffer.from('9090cc0f0b', 'hex') }
+        ],
+        platformId: VER_PLATFORM_WIN32_NT,
+        modules: [{ base: 0x4000n, size: 0x2000, name: 'Orca.exe' }]
+      })
+    )
+    expect(signature?.fault).toEqual({ codeBytes: 'cc0f0b' })
+  })
+
+  it('captures no code bytes outside a loaded module, such as JIT code', async () => {
+    const signature = await parseMinidumpCrashSignature(
+      buildFaultDump({
+        code: 0xc0000005,
+        rip: 0x9000n,
+        rsp: 0x10_0000_0000n,
+        params: [0n, 0x20n],
+        memory: [{ start: 0x9000n, bytes: CODE }],
+        platformId: VER_PLATFORM_WIN32_NT,
+        modules: [{ base: 0x4000n, size: 0x2000, name: 'Orca.exe' }]
+      })
+    )
+    expect(signature?.fault).toEqual({ access: 'read', faultAddress: '0x20' })
+  })
+
+  it('captures no code bytes from a macOS dump, where the address is the touched data', async () => {
+    const signature = await parseMinidumpCrashSignature(
+      buildFaultDump({
+        code: 0x1,
+        rip: 0x5000n,
+        memory: [{ start: 0x5000n, bytes: CODE }],
+        platformId: CRASHPAD_PLATFORM_MACOS,
+        modules: [{ base: 0x4000n, size: 0x2000, name: 'Electron Framework' }]
+      })
+    )
+    expect(signature?.fault).toEqual({})
+  })
+
+  it('degrades to nothing when the AV parameters or memory list are missing', async () => {
+    const signature = await parseMinidumpCrashSignature(
+      buildFaultDump({ code: 0xc0000005, rip: 0x5000n })
+    )
+    expect(signature?.exceptionAddress).toBe('0x5000')
+    expect(signature?.fault).toEqual({})
+  })
+})

src/main/crash-reporting/minidump-fault-context.ts

@@ -0,0 +1,128 @@
+// What a native fault touched and which bytes were executing.
+//
+// Why: the faulting RVA alone cannot separate an engine bug from a corrupted
+// code page or unstable CPU. Two Windows renderer AVs mapped to official
+// symbols as writes into memory the preceding instructions had just written;
+// the code bytes Crashpad captured at the instruction pointer settle that by
+// comparison with the shipped binary.
+
+import { findStream, type MinidumpView } from './minidump-stream-reader'
+
+const STREAM_TYPE_MEMORY_LIST = 5
+const EXCEPTION_ACCESS_VIOLATION = 0xc0000005
+
+// MINIDUMP_EXCEPTION_STREAM: ThreadId, alignment, MINIDUMP_EXCEPTION (152 bytes), ThreadContext.
+const NUMBER_PARAMETERS_OFFSET = 8 + 24
+const EXCEPTION_INFORMATION_OFFSET = 8 + 32
+const THREAD_CONTEXT_OFFSET = 8 + 152
+
+// x64 CONTEXT; other architectures report no stack-pointer delta.
+const AMD64_CONTEXT_SIZE = 0x4d0
+const AMD64_CONTEXT_FLAGS_OFFSET = 0x30
+const CONTEXT_AMD64 = 0x00100000
+const AMD64_RSP_OFFSET = 0x98
+
+const MEMORY_DESCRIPTOR_SIZE = 16
+const MAX_MEMORY_RANGES = 16_384
+const CODE_BYTES = 16
+// Beyond this the fault is not a slot in the faulting frame, so the delta adds nothing.
+const MAX_STACK_POINTER_DELTA = 1024 * 1024
+
+const ACCESS_KINDS: Readonly<Record<number, MinidumpFaultContext['access']>> = {
+  0: 'read',
+  1: 'write',
+  8: 'execute'
+}
+
+export type MinidumpFaultContext = {
+  readonly access?: 'read' | 'write' | 'execute'
+  readonly faultAddress?: string
+  /** Fault address minus the faulting thread's stack pointer, when within 1 MiB. */
+  readonly faultStackPointerDelta?: number
+  /** Hex of the bytes at the instruction pointer, as captured in the dump. */
+  readonly codeBytes?: string
+}
+
+async function readAmd64StackPointer(
+  view: MinidumpView,
+  exceptionRva: number
+): Promise<bigint | null> {
+  const context = await view.location(exceptionRva + THREAD_CONTEXT_OFFSET)
+  if (!context || context.size < AMD64_CONTEXT_SIZE) {
+    return null
+  }
+  const flags = await view.u32(context.rva + AMD64_CONTEXT_FLAGS_OFFSET)
+  if (flags === null || (flags & CONTEXT_AMD64) === 0) {
+    return null
+  }
+  return view.u64(context.rva + AMD64_RSP_OFFSET)
+}
+
+async function readCapturedBytes(view: MinidumpView, address: bigint): Promise<string | null> {
+  const stream = await findStream(view, STREAM_TYPE_MEMORY_LIST)
+  if (!stream) {
+    return null
+  }
+  const count = await view.u32(stream.rva)
+  if (count === null || count > MAX_MEMORY_RANGES) {
+    return null
+  }
+  for (let index = 0; index < count; index += 1) {
+    const descriptor = stream.rva + 4 + index * MEMORY_DESCRIPTOR_SIZE
+    const start = await view.u64(descriptor)
+    const range = await view.location(descriptor + 8)
+    if (start === null) {
+      return null
+    }
+    if (!range || address < start || address >= start + BigInt(range.size)) {
+      continue
+    }
+    const skip = Number(address - start)
+    const size = Math.min(CODE_BYTES, range.size - skip)
+    const bytes = await view.bytes({ size, rva: range.rva + skip }, CODE_BYTES)
+    return bytes ? bytes.toString('hex') : null
+  }
+  return null
+}
+
+export async function readMinidumpFaultContext(
+  view: MinidumpView,
+  exceptionRva: number,
+  exceptionCode: number | null,
+  exceptionAddress: bigint,
+  options: { readonly captureCodeBytes: boolean }
+): Promise<MinidumpFaultContext> {
+  const result: { -readonly [K in keyof MinidumpFaultContext]: MinidumpFaultContext[K] } = {}
+  const codeBytes = options.captureCodeBytes
+    ? await readCapturedBytes(view, exceptionAddress)
+    : null
+  if (codeBytes) {
+    result.codeBytes = codeBytes
+  }
+  if (exceptionCode !== EXCEPTION_ACCESS_VIOLATION) {
+    return result
+  }
+  const parameterCount = await view.u32(exceptionRva + NUMBER_PARAMETERS_OFFSET)
+  if (parameterCount === null || parameterCount < 2) {
+    return result
+  }
+  const kind = await view.u64(exceptionRva + EXCEPTION_INFORMATION_OFFSET)
+  const faultAddress = await view.u64(exceptionRva + EXCEPTION_INFORMATION_OFFSET + 8)
+  const access = kind === null ? undefined : ACCESS_KINDS[Number(kind)]
+  if (access) {
+    result.access = access
+  }
+  if (faultAddress === null) {
+    return result
+  }
+  result.faultAddress = `0x${faultAddress.toString(16)}`
+  const stackPointer = await readAmd64StackPointer(view, exceptionRva)
+  if (stackPointer !== null) {
+    const delta = faultAddress - stackPointer
+    const magnitude = delta < 0n ? -delta : delta
+    if (magnitude <= BigInt(MAX_STACK_POINTER_DELTA)) {
+      result.faultStackPointerDelta = Number(delta)
+    }
+  }
+  return result
+}