diff --git a/config/main-sync-fs-baseline.txt b/config/main-sync-fs-baseline.txt index b950b4ec34e..aaa2d1ee155 100644 --- a/config/main-sync-fs-baseline.txt +++ b/config/main-sync-fs-baseline.txt @@ -265,6 +265,7 @@ 4 src/main/hang-watchdog/hang-incident-snapshot.ts 1 src/main/hang-watchdog/hang-watchdog-worker-path.ts 1 src/main/hang-watchdog/previous-hang-detection.ts +9 src/main/hang-watchdog/renderer-hang-records.ts 1 src/main/hermes/hermes-default-home.ts 10 src/main/hermes/hermes-home-filesystem.ts 2 src/main/hermes/hook-service.ts diff --git a/src/main/hang-watchdog/hang-profile-reducer.ts b/src/main/hang-watchdog/hang-profile-reducer.ts index 716f9c0729b..9e9cf625aa1 100644 --- a/src/main/hang-watchdog/hang-profile-reducer.ts +++ b/src/main/hang-watchdog/hang-profile-reducer.ts @@ -39,11 +39,17 @@ export function reduceHangProfile(profile: Profiler.Profile): HangStacks { counts.set(id, (counts.get(id) ?? 0) + 1) validSamples++ } - const top: HangStack[] = [] + const selected: { id: number; count: number }[] = [] for (const [id, count] of counts) { - if (top.length === 3 && count <= top[2].count) { + if (selected.length === 3 && count <= selected[2].count) { continue } + selected.push({ id, count }) + selected.sort((a, b) => b.count - a.count) + selected.splice(3) + } + const top: HangStack[] = [] + for (const { id, count } of selected) { const frames: HangFrame[] = [] const seen = new Set() let current: number | undefined = id diff --git a/src/main/hang-watchdog/main-js-hang-capture.ts b/src/main/hang-watchdog/main-js-hang-capture.ts index ed788b7c791..243144e6630 100644 --- a/src/main/hang-watchdog/main-js-hang-capture.ts +++ b/src/main/hang-watchdog/main-js-hang-capture.ts @@ -2,10 +2,11 @@ import { reduceHangProfile } from './hang-profile-reducer' import type { Profiler, Session } from 'node:inspector' import { emptyHangStacks, type MainJsCapture } from './hang-stacks' +type ProfilerCommandParameters = { interval?: number } type CaptureSession = Pick & { post: ( method: string, - params: object, + params: ProfilerCommandParameters, callback: (error: Error | null, result?: unknown) => void ) => void } @@ -66,7 +67,10 @@ export function captureMainJsHang(options: { } const deadline = setTimeout(() => stop('deadline'), Math.max(0, 5_000 - elapsed())) publish({}) - const post = (method: string, params: object = {}): Promise<{ ok: boolean; result?: unknown }> => + const post = ( + method: string, + params: ProfilerCommandParameters = {} + ): Promise<{ ok: boolean; result?: unknown }> => new Promise((resolve) => { // Exactly one caller awaits each response before the next command or disconnect. try { diff --git a/src/main/hang-watchdog/main-thread-hang-capture.test.ts b/src/main/hang-watchdog/main-thread-hang-capture.test.ts new file mode 100644 index 00000000000..16c65ef996e --- /dev/null +++ b/src/main/hang-watchdog/main-thread-hang-capture.test.ts @@ -0,0 +1,135 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { MainToHangWatchdogWorkerMessage } from './hang-watchdog-worker-protocol' + +const state = vi.hoisted(() => ({ + clock: 0, + delayedWrite: false, + commands: new Array(), + terminations: new Array<() => void>(), + records: new Array() +})) +vi.mock('node:worker_threads', () => ({ isMainThread: false, parentPort: null, workerData: null })) +vi.mock('node:inspector', () => ({ + Session: class { + connectToMainThread() {} + disconnect() {} + post(method: string, _params: unknown, response: (error: null, result?: unknown) => void) { + state.commands.push(method) + response( + null, + method === 'Profiler.stop' + ? { profile: { nodes: [], samples: [], startTime: 0, endTime: 1 } } + : undefined + ) + } + } +})) +vi.mock('@orca/process-host', () => ({ + runProcess: async (options: { onChildTerminated: () => void }) => { + state.terminations.push(options.onChildTerminated) + return { stdout: '', stderr: '', code: null, timedOut: true } + } +})) +vi.mock('./hang-detection-marker', () => ({ + readAvailableHangDetectionMarker: () => null, + writeHangDetectionMarker: (_path: string, record: unknown) => { + state.records.push(structuredClone(record)) + if (state.delayedWrite) { + state.clock = 6000 + } + } +})) +vi.mock('./hang-incident-snapshot', () => ({ + readCompletedHangContext: () => null, + collectHangMachineSnapshot: async () => null +})) +import { runWatchdog } from './main-thread-hang-watchdog-entry' + +beforeEach(() => { + state.clock = 0 + state.delayedWrite = false + state.commands = [] + state.terminations = [] + state.records = [] + vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin') + vi.useFakeTimers() + vi.setSystemTime(0) + vi.spyOn(performance, 'now').mockImplementation(() => state.clock) +}) +afterEach(() => { + for (const terminate of state.terminations.splice(0)) { + terminate() + } + vi.clearAllTimers() + vi.useRealTimers() + vi.restoreAllMocks() +}) +function watchdog() { + let message: (value: MainToHangWatchdogWorkerMessage) => void = () => {} + runWatchdog( + { + parentPid: 123, + markerPath: '/qa/main-thread-hang.json', + timeoutMs: 100, + checkIntervalMs: 25 + }, + { + on: (_event, listener) => { + message = listener + }, + close: () => {}, + postMessage: (value) => message({ type: 'capture-drain-ack', token: value.token }) + } + ) + return { send: (value: MainToHangWatchdogWorkerMessage) => message(value) } +} +it('does not start a profiler after baseline publication has exhausted the detection budget', async () => { + state.delayedWrite = true + const owner = watchdog() + await vi.advanceTimersByTimeAsync(125) + await vi.dynamicImportSettled() + expect(state.commands).toEqual(['Profiler.enable', 'Profiler.disable']) + expect(state.records.at(-1)).toMatchObject({ + observations: [ + expect.objectContaining({ + mainJs: expect.objectContaining({ + state: 'not-started', + stopReason: 'deadline', + stopRequestedAfterDetectionMs: 6000 + }) + }) + ] + }) + owner.send({ type: 'shutdown' }) +}) +it('admits a later incident after a late native termination receipt releases the physical owner', async () => { + const owner = watchdog() + await vi.advanceTimersByTimeAsync(125) + await vi.dynamicImportSettled() + owner.send({ type: 'heartbeat' }) + await vi.dynamicImportSettled() + expect(state.records.at(-1)).toMatchObject({ + observations: [ + expect.objectContaining({ native: expect.objectContaining({ cleanup: 'unverified' }) }) + ] + }) + state.terminations.shift()?.() + await vi.dynamicImportSettled() + expect(state.records.at(-1)).toMatchObject({ + observations: [ + expect.objectContaining({ native: expect.objectContaining({ cleanup: 'terminated' }) }) + ] + }) + await vi.advanceTimersByTimeAsync(125) + await vi.dynamicImportSettled() + expect(state.records.at(-1)).toMatchObject({ + observations: [ + expect.anything(), + expect.objectContaining({ + mainJs: expect.objectContaining({ state: 'collecting' }), + native: expect.objectContaining({ state: 'completed', cleanup: 'unverified' }) + }) + ] + }) + owner.send({ type: 'shutdown' }) +}) diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts index a0c7e663223..94cc8e897db 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts @@ -80,11 +80,10 @@ export function runWatchdog( observations = retainHangObservation(observations, { ...record, ...change }) save() } - const capture = (record: HangObservation): void => { + const capture = (record: HangObservation, detectedAt: number): void => { if (isMainThread || captureOwner || stopped || record.kind !== 'heartbeat-stall') { return } - const detectedAt = performance.now() const owner: CaptureOwner = { id: record.id } captureOwner = owner owner.done = (async () => { @@ -166,6 +165,7 @@ export function runWatchdog( }) } const observe = (kind: HangObservation['kind'], durationMs: number, startedAt: number): void => { + const detectedAt = performance.now() const snapshot = captureHangSnapshot() const now = snapshot.wallMs ?? Date.now() const suspend = sleepPeriod?.suspend @@ -204,7 +204,7 @@ export function runWatchdog( } openReceiptWindow(record.id) save() - capture(record) + capture(record, detectedAt) enrich(record) } const loop = createHangWatchdogDetectionLoop({ diff --git a/src/main/hang-watchdog/native-hang-capture.ts b/src/main/hang-watchdog/native-hang-capture.ts index cf6810c1a56..7a493f6f0b2 100644 --- a/src/main/hang-watchdog/native-hang-capture.ts +++ b/src/main/hang-watchdog/native-hang-capture.ts @@ -119,6 +119,10 @@ export function captureNativeHang(options: { }): { stop: () => void; done: Promise } { const abort = new AbortController() let terminated = false + let release: () => void = () => {} + const termination = new Promise((resolve) => { + release = resolve + }) const baseline: NativeHangCapture = { ...emptyHangStacks(), state: 'collecting', @@ -130,6 +134,11 @@ export function captureNativeHang(options: { cleanup: 'pending' } options.publish(baseline) + let latest = baseline + const publish = (capture: NativeHangCapture): void => { + latest = capture + options.publish(capture) + } const done = (async (): Promise => { try { const { runProcess } = await import('@orca/process-host') @@ -142,9 +151,10 @@ export function captureNativeHang(options: { signal: abort.signal, onChildTerminated: () => { terminated = true + release() } }) - options.publish({ + publish({ ...baseline, ...reduceNativeHangSample(result.stdout), state: 'completed', @@ -155,7 +165,7 @@ export function captureNativeHang(options: { ...(result.code !== null ? { exitCode: result.code } : {}) }) } catch { - options.publish({ + publish({ ...baseline, state: 'failed', completedAt: Date.now(), @@ -163,7 +173,8 @@ export function captureNativeHang(options: { }) } if (!terminated) { - await new Promise(() => {}) + await termination + publish({ ...latest, cleanup: 'terminated' }) } })() return { stop: () => abort.abort(), done } diff --git a/src/main/hang-watchdog/previous-hang-detection.test.ts b/src/main/hang-watchdog/previous-hang-detection.test.ts index 06c7abfe9c1..bb277618cad 100644 --- a/src/main/hang-watchdog/previous-hang-detection.test.ts +++ b/src/main/hang-watchdog/previous-hang-detection.test.ts @@ -44,6 +44,7 @@ beforeEach(() => { afterEach(() => { setActiveSink(null) vi.unstubAllEnvs() + vi.useRealTimers() rmSync(dir, { recursive: true, force: true }) }) @@ -224,9 +225,11 @@ describe('previous hang detection', () => { }) }) -it('replays an old renderer capture into the current trace before consuming it so the default bundle includes it', () => { - const detectedAt = Date.now() - 2 * 60 * 60 * 1000 - publishRendererHang(dir, { +it('replays an old renderer capture into the current trace before consuming it so the default bundle includes it', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-11-11T12:00:00Z')) + const detectedAt = 1794391200000 + await publishRendererHang(dir, { id: `456:${detectedAt}:3`, parentPid: 456, windowId: 3, @@ -279,3 +282,56 @@ it('replays an old renderer capture into the current trace before consuming it s sink.close() } }) + +it('puts renderer completion into the current bundle while retaining next-launch evidence', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-11-11T12:00:00Z')) + const traceFilePath = join(dir, 'logs', 'main.trace.ndjson') + const sink = createLocalFileSink({ filePath: traceFilePath, maxFiles: 1 }) + try { + setActiveSink(sink) + await publishRendererHang(dir, { + id: '123:1000:3', + parentPid: 123, + windowId: 3, + detectedAt: 1000, + completedAt: 2000, + status: 'captured', + samples: 1, + validSamples: 1, + truncated: false, + stacks: [ + { + count: 1, + hot: true, + frames: [{ name: 'rendererBusy', module: 'app.js', line: 9, column: 2 }] + } + ] + }) + const bundle = collectBundle({ + traceFilePath, + maxFiles: 1, + appVersion: 'test', + platform: 'darwin', + arch: 'arm64', + osRelease: 'test', + orcaChannel: 'dev' + }) + const capture = bundle.payload + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .find((record) => record.attributes?.['breadcrumb.name'] === 'renderer_hang_capture') + expect(capture?.attributes['breadcrumb.data']).toMatchObject({ + detectedAt: 1000, + completedAt: 2000, + rendererJs0_frame0: 'rendererBusy [app.js:9:2]' + }) + expect(JSON.parse(readFileSync(join(dir, 'renderer-hang.json'), 'utf8'))).toMatchObject({ + records: [expect.objectContaining({ completedAt: 2000 })] + }) + } finally { + setActiveSink(null) + sink.close() + } +}) diff --git a/src/main/hang-watchdog/renderer-hang-capture.test.ts b/src/main/hang-watchdog/renderer-hang-capture.test.ts index 9c6fb2db065..5ac56aafafe 100644 --- a/src/main/hang-watchdog/renderer-hang-capture.test.ts +++ b/src/main/hang-watchdog/renderer-hang-capture.test.ts @@ -126,3 +126,12 @@ it('drops a cut input field before redaction while preserving complete frames', truncated: true }) }) + +it('records an unanswered renderer request explicitly while retaining its physical owner', () => { + const blocked = window(6) + blocked.start()?.stop() + expect(state.records.at(-1)).toMatchObject({ windowId: 6, status: 'request-pending', samples: 0 }) + const shared = window(7) + shared.start() + expect(state.records.at(-1)).toMatchObject({ windowId: 7, status: 'process-capture-pending' }) +}) diff --git a/src/main/hang-watchdog/renderer-hang-capture.ts b/src/main/hang-watchdog/renderer-hang-capture.ts index 451d451264c..04f9721911d 100644 --- a/src/main/hang-watchdog/renderer-hang-capture.ts +++ b/src/main/hang-watchdog/renderer-hang-capture.ts @@ -73,9 +73,11 @@ function stopEpisode(contentsId: number, id: string): void { episode, episode.record.stacks.length ? 'captured' - : episode.record.status === 'collecting' - ? 'stopped' - : episode.record.status + : [...pending.values()].some((owner) => owner.episodeId === id) + ? 'request-pending' + : episode.record.status === 'collecting' + ? 'stopped' + : episode.record.status ) episodes.delete(contentsId) } diff --git a/src/main/hang-watchdog/renderer-hang-records.ts b/src/main/hang-watchdog/renderer-hang-records.ts index 7c974c559c5..1ed3f7449a0 100644 --- a/src/main/hang-watchdog/renderer-hang-records.ts +++ b/src/main/hang-watchdog/renderer-hang-records.ts @@ -8,6 +8,7 @@ import { statSync, writeFileSync } from 'node:fs' +import { open, readFile, rename, stat } from 'node:fs/promises' import { join } from 'node:path' import { parseHangStacks, type HangStacks } from './hang-stacks' import { resolveObservabilityConsent } from '../observability' @@ -28,6 +29,7 @@ export type RendererHangRecord = HangStacks & { | 'policy-missing' | 'failed' | 'process-capture-pending' + | 'request-pending' | 'stopped' } const validStatus = [ @@ -37,9 +39,17 @@ const validStatus = [ 'policy-missing', 'failed', 'process-capture-pending', + 'request-pending', 'stopped' ] -let current: RendererHangRecord[] = [] +type RendererHangWriter = { + path: string + records: RendererHangRecord[] + initialized: boolean + revision: number + running?: Promise +} +let writer: RendererHangWriter | undefined function pathFor(userDataPath: string, previous = false): string { return join(userDataPath, `renderer-hang.json${previous ? '.previous' : ''}`) @@ -83,24 +93,27 @@ function parseRecord(value: unknown): RendererHangRecord | null { : {}) } } +function parseRecords(data: unknown): RendererHangRecord[] { + if ( + !data || + typeof data !== 'object' || + !('version' in data) || + data.version !== 1 || + !('records' in data) || + !Array.isArray(data.records) || + data.records.length > 4 + ) { + return [] + } + return data.records.map(parseRecord).filter((item): item is RendererHangRecord => item !== null) +} function read(path: string): RendererHangRecord[] | undefined { try { if (statSync(path).size > 16 * 1024) { return [] } const data: unknown = JSON.parse(readFileSync(path, 'utf8')) - if ( - !data || - typeof data !== 'object' || - !('version' in data) || - data.version !== 1 || - !('records' in data) || - !Array.isArray(data.records) || - data.records.length > 4 - ) { - return [] - } - return data.records.map(parseRecord).filter((item): item is RendererHangRecord => item !== null) + return parseRecords(data) } catch (error) { return error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT' ? [] @@ -153,25 +166,83 @@ function write(path: string, records: RendererHangRecord[]): void { renameSync(`${path}.tmp`, path) } -export function publishRendererHang(userDataPath: string, record: RendererHangRecord): void { - if (!resolveObservabilityConsent().localFileEnabled) { - return - } +async function persist(owner: RendererHangWriter): Promise { try { - const normalized = parseRecord(record) - if (!normalized) { - return + if (!owner.initialized) { + let disk: RendererHangRecord[] = [] + try { + if ((await stat(owner.path)).size <= 16 * 1024) { + disk = parseRecords(JSON.parse(await readFile(owner.path, 'utf8'))) + } + } catch (error) { + if (!error || typeof error !== 'object' || !('code' in error) || error.code !== 'ENOENT') { + return + } + } + owner.records = merge(disk, owner.records) + owner.initialized = true } - const disk = read(pathFor(userDataPath)) - if (!disk) { - return + let committed = -1 + while (committed !== owner.revision) { + const revision = owner.revision + const text = JSON.stringify({ version: 1, records: owner.records }) + if (Buffer.byteLength(text) > 16 * 1024) { + return + } + const file = await open(`${owner.path}.tmp`, 'w', 0o600) + try { + await file.chmod(0o600) + await file.writeFile(text) + } finally { + await file.close() + } + await rename(`${owner.path}.tmp`, owner.path) + committed = revision } - current = merge(merge(disk, current), [normalized]) - write(pathFor(userDataPath), current) } catch { - /* Optional diagnostics never block a window lifecycle event. */ + // Keep the admitted snapshot for a later incident publication to retry. + } finally { + owner.running = undefined } } +function recordCapture(record: RendererHangRecord): void { + recordDurableCrashBreadcrumb('renderer_hang_capture', { + previousPid: record.parentPid, + windowId: record.windowId, + detectedAt: record.detectedAt, + completedAt: record.completedAt ?? null, + ...hangCaptureDetails('rendererJs', record) + }) +} +export function publishRendererHang( + userDataPath: string, + record: RendererHangRecord +): Promise { + if (!resolveObservabilityConsent().localFileEnabled) { + return Promise.resolve() + } + const normalized = parseRecord(record) + if (!normalized) { + return Promise.resolve() + } + const path = pathFor(userDataPath) + if (!writer || writer.path !== path) { + writer = { path, records: [], initialized: false, revision: 0 } + } + const admitted = admit(normalized) + writer.records = merge(writer.records, [admitted]) + writer.revision++ + if (admitted.completedAt !== undefined) { + try { + recordCapture(admitted) + } catch { + // A failed trace flush must not prevent the independent pending-file write. + } + } + const owner = writer + owner.running ??= Promise.resolve().then(() => persist(owner)) + return owner.running +} export function preservePreviousRendererHangs(userDataPath: string): void { if (!resolveObservabilityConsent().localFileEnabled) { @@ -188,7 +259,7 @@ export function preservePreviousRendererHangs(userDataPath: string): void { } write(pathFor(userDataPath, true), merge(previous, records)) rmSync(pathFor(userDataPath), { force: true }) - current = [] + writer = undefined } catch { /* Keep the original pending file if preservation failed. */ } @@ -204,13 +275,7 @@ export function reportPreviousRendererHangs(userDataPath: string): void { return } for (const record of records) { - recordDurableCrashBreadcrumb('renderer_hang_capture', { - previousPid: record.parentPid, - windowId: record.windowId, - detectedAt: record.detectedAt, - completedAt: record.completedAt ?? null, - ...hangCaptureDetails('rendererJs', record) - }) + recordCapture(record) } // Each breadcrumb synchronously requests a trace flush before pending data is consumed. rmSync(pathFor(userDataPath, true), { force: true }) diff --git a/src/main/hang-watchdog/renderer-hang-stack-policy.test.ts b/src/main/hang-watchdog/renderer-hang-stack-policy.test.ts index 1a109878f9a..4f4429db22e 100644 --- a/src/main/hang-watchdog/renderer-hang-stack-policy.test.ts +++ b/src/main/hang-watchdog/renderer-hang-stack-policy.test.ts @@ -42,3 +42,34 @@ it('opts in only the owned main document and preserves unrelated response policy listener({ url: 'file:///app/index.html', resourceType: 'mainFrame', webContentsId: 4 }, answer) expect(answer).toHaveBeenLastCalledWith({}) }) + +it('normalizes the development document URL for both the filter and response identity', () => { + let listener: (details: unknown, callback: (value: unknown) => void) => void = () => {} + const onHeadersReceived = vi.fn((_filter, callback) => { + listener = callback + }) + const contents = Object.assign(new EventEmitter(), { + id: 8, + session: { webRequest: { onHeadersReceived } } + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this Electron test double supplies the members used by the policy; unused native members are never accessed. + installRendererHangStackPolicy(contents as unknown as WebContents, 'http://localhost:5173') + expect(onHeadersReceived).toHaveBeenCalledWith( + { urls: ['http://localhost:5173/'], types: ['mainFrame'] }, + expect.any(Function) + ) + const answer = vi.fn() + listener( + { + url: 'http://localhost:5173/', + resourceType: 'mainFrame', + webContentsId: 8, + responseHeaders: {} + }, + answer + ) + expect(answer).toHaveBeenLastCalledWith({ + responseHeaders: { 'Document-Policy': ['include-js-call-stacks-in-crash-reports'] } + }) + contents.emit('destroyed') +}) diff --git a/src/main/hang-watchdog/renderer-hang-stack-policy.ts b/src/main/hang-watchdog/renderer-hang-stack-policy.ts index c189ff0b71e..24ab7e846f3 100644 --- a/src/main/hang-watchdog/renderer-hang-stack-policy.ts +++ b/src/main/hang-watchdog/renderer-hang-stack-policy.ts @@ -5,7 +5,7 @@ const owners = new WeakMap>() export function installRendererHangStackPolicy(contents: WebContents, documentUrl: string): void { try { - installPolicy(contents, documentUrl) + installPolicy(contents, new URL(documentUrl).href) } catch { // A rejected diagnostic policy must not prevent the main document from loading. }