From 92b6ffd17d015e41784b06fc0c7b7e6f97422bd1 Mon Sep 17 00:00:00 2001 From: Langning Zhang Date: Fri, 14 Aug 2026 16:21:00 +0800 Subject: [PATCH] Terminate renderer graph reload generations and contain disposed-frame notifications (#14070) * fix(runtime): terminate renderer graph reload generations * fix(runtime): harden renderer reload teardown * fix(runtime): fence renderer graph publication ownership * test(runtime): register renderer graph reload gate * test(runtime): record live reload validation * fix(runtime): ignore cancelled renderer navigations * chore: preserve main formatting during branch sync * chore: satisfy changed-code quality gate * fix(runtime): restore cancelled renderer reloads * fix(runtime): preserve committed reload fencing * test(runtime): prove cancelled reload timeout * docs(reliability): record reload cancellation oracle --------- Co-authored-by: Jinwoo-H --- config/reliability-gates.jsonc | 175 ++++++++++++++ src/main/ipc/runtime.test.ts | 42 +++- src/main/ipc/runtime.ts | 12 +- src/main/runtime/orca-runtime.test.ts | 215 ++++++++++++++++++ src/main/runtime/orca-runtime.ts | 200 +++++++++++++++- .../runtime-graph-reload-lifecycle.test.ts | 104 +++++++++ .../runtime/runtime-graph-reload-lifecycle.ts | 87 +++++++ .../attach-main-window-services.test.ts | 21 ++ .../window/attach-main-window-services.ts | 39 +++- .../renderer-document-navigation.test.ts | 137 +++++++++++ .../window/renderer-document-navigation.ts | 74 ++++++ ...ntime-renderer-notification-sender.test.ts | 108 +++++++++ .../runtime-renderer-notification-sender.ts | 79 +++++++ src/preload/api/runtime-api.ts | 6 +- src/preload/index.ts | 6 +- .../src/runtime/sync-runtime-graph.ts | 5 +- src/shared/runtime-types.ts | 5 + ...blocked-navigation-graph-authority.spec.ts | 106 +++++++++ 18 files changed, 1393 insertions(+), 28 deletions(-) create mode 100644 src/main/runtime/runtime-graph-reload-lifecycle.test.ts create mode 100644 src/main/runtime/runtime-graph-reload-lifecycle.ts create mode 100644 src/main/window/renderer-document-navigation.test.ts create mode 100644 src/main/window/renderer-document-navigation.ts create mode 100644 src/main/window/runtime-renderer-notification-sender.test.ts create mode 100644 src/main/window/runtime-renderer-notification-sender.ts create mode 100644 tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 45c7aed8242..a076c324708 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1452,6 +1452,181 @@ ], "demotionRule": "Quarantine the Electron journey only with a linked product or harness defect; demote if activation changes the owner/runtime/daemon/PTY identity, loses prior output, opens before provider readiness, or fails to honor a committed quit." }, + { + "id": "runtime.renderer-graph-reload-termination", + "title": "Renderer graph reloads terminate under exact document and window authority", + "maturity": "experimental", + "protection": "partial", + "owner": "runtime-platform", + "layer": "electron-runtime-contract", + "surfaces": [ + "desktop renderer reload and crash recovery", + "headless-to-desktop promotion", + "CLI and paired-runtime graph availability" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "paired-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "paired-runtime"], + "coverageNotes": "Deterministic lifecycle, IPC, window-notification, and runtime ownership tests cover generation replacement, stale callbacks, same-frame old-document publication, blocked external navigation, cancelled renderer navigation, concurrent provisional starts, process loss, timeout cleanup, headless fallback, and competing windows. Isolated macOS Electron validation covers cancelled navigation, forced renderer recovery, and headless serve promotion with live daemon-backed PTY identity and I/O. Linux, Windows, WSL, SSH promotion, and mixed-version desktop/server live runs remain uncollected; the renderer generation stays local to Electron IPC and does not change the remote wire.", + "motivatingLinks": [ + "https://github.com/stablyai/orca/issues/14066", + "https://linear.app/stably/issue/STA-4016", + "https://github.com/stablyai/orca/pull/14070" + ], + "invariant": "Every genuine renderer graph reload revision settles exactly once, while a blocked external navigation that preserves the current document never starts a revision. A cancelled renderer-document navigation restores authority only when that same renderer survived; cancellation after an earlier document committed remains fenced until publication or timeout. Replacing a reload cancels only the prior revision; its timer and callbacks cannot settle a newer revision. A desktop reload succeeds only from a publication carrying a new renderer-document generation, while a failed headless promotion may recover only from its originating window. Timeout, renderer loss, and close reach a terminal graph state, and successful desktop publication permanently retires headless sentinel fallback until a new headless runtime is established.", + "oracle": "Start a blocked external main-frame navigation and require the current renderer graph and notification path to remain authoritative. Start a same-renderer navigation, cancel it before commit, and require the renderer canary and ready authority to survive; coalesce overlapping provisional starts into one fence, and keep an earlier committed reload fenced if a later navigation is cancelled. Drive two genuine reloads half a timeout apart and require the first deadline to leave the second revision reloading while the second deadline alone makes it unavailable. During the second revision publish from the prior renderer document through the still-current main frame and require rejection without graph mutation or timeout cancellation. Restore headless authority after failed promotion, publish from a competing window, and require rejection while the sentinel remains authoritative. Then force a real renderer crash and separately promote an isolated headless serve owner; require runtime graph readiness, stable runtime/main/daemon/PTY identities, recovered visible terminal output, and post-recovery input/output.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/runtime-graph-reload-lifecycle.test.ts src/main/window/runtime-renderer-notification-sender.test.ts src/main/window/renderer-document-navigation.test.ts src/main/ipc/runtime.test.ts src/main/window/attach-main-window-services.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/runtime/sync-runtime-graph-payload-partition.test.ts", + "pnpm exec electron-vite build --mode e2e", + "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "SKIP_BUILD=1 pnpm exec playwright test e2e/renderer-crash-recovery-terminal-input.spec.ts tests/e2e/renderer-crash-recovery-terminal-input.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "SKIP_BUILD=1 pnpm exec playwright test e2e/headless-serve-desktop-activation.spec.ts tests/e2e/headless-serve-desktop-activation.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" + ], + "testFiles": [ + "src/main/runtime/runtime-graph-reload-lifecycle.test.ts", + "src/main/window/runtime-renderer-notification-sender.test.ts", + "src/main/window/renderer-document-navigation.test.ts", + "src/main/ipc/runtime.test.ts", + "src/main/window/attach-main-window-services.test.ts", + "src/main/runtime/orca-runtime.test.ts", + "src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts", + "src/renderer/src/runtime/sync-runtime-graph-payload-partition.test.ts", + "tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts", + "tests/e2e/renderer-crash-recovery-terminal-input.spec.ts", + "tests/e2e/headless-serve-desktop-activation.spec.ts" + ], + "assertionRefs": [ + { + "file": "src/main/runtime/runtime-graph-reload-lifecycle.test.ts", + "assertions": [ + "each revision settles exactly once", + "replacement cancels the old revision and fences its timeout", + "settlement and timeout release their timers" + ] + }, + { + "file": "src/main/runtime/orca-runtime.test.ts", + "assertions": [ + "only the newest reload owns the terminal timeout", + "a superseded renderer-document generation cannot settle a reload", + "cancelled navigation restores only the renderer or headless authority that actually survived", + "cancellation after an earlier document commit stays fenced until its restarted timeout", + "failed headless promotion remains pinned to its originating window", + "successful desktop publication retires headless fallback" + ] + }, + { + "file": "src/main/ipc/runtime.test.ts", + "assertions": [ + "graph publication requires a non-empty renderer generation", + "only the current BrowserWindow main frame may publish" + ] + }, + { + "file": "src/main/window/runtime-renderer-notification-sender.test.ts", + "assertions": [ + "renderer failure is contained once per load generation", + "late events cannot revive a closed sender" + ] + }, + { + "file": "src/main/window/renderer-document-navigation.test.ts", + "assertions": [ + "blocked external navigation does not start a renderer generation", + "only the packaged renderer file or development origin starts a renderer generation", + "concurrent provisional starts share one reload fence and cancel only after every attempt fails" + ] + }, + { + "file": "tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts", + "assertions": [ + "blocked external navigation starts and stops loading without replacing the renderer document", + "runtime graph epoch, readiness, and authoritative window remain unchanged", + "a later listener can cancel a renderer navigation while its canary and authoritative graph recover" + ] + }, + { + "file": "tests/e2e/renderer-crash-recovery-terminal-input.spec.ts", + "assertions": [ + "main observes renderer process loss and a completed recovery load", + "the same daemon-backed PTY remains reachable through transport and direct-write probes" + ] + }, + { + "file": "tests/e2e/headless-serve-desktop-activation.spec.ts", + "assertions": [ + "promotion preserves main owner, runtime, daemon, and PTY identity", + "terminal output survives promotion and post-promotion input/output remains live" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-08-14", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/runtime-graph-reload-lifecycle.test.ts src/main/window/runtime-renderer-notification-sender.test.ts src/main/window/renderer-document-navigation.test.ts src/main/ipc/runtime.test.ts src/main/window/attach-main-window-services.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/runtime/sync-runtime-graph-payload-partition.test.ts", + "result": "passed", + "durationSeconds": 12, + "summary": "Eight focused files passed with 1,175 tests and one pre-existing skip after the merge from latest main. The suite covers exact reload settlement, timeout replacement, cancelled-navigation recovery, concurrent provisional starts, document and window authority, notification containment, preload IPC, and renderer publication scheduling." + }, + { + "date": "2026-08-14", + "runner": "local", + "platform": "macos", + "command": "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 8, + "summary": "Two isolated Electron flows passed: blocked external navigation left epoch and authority unchanged, and a same-renderer navigation cancelled by a later listener preserved its canary while returning the incremented graph generation to ready under the same authoritative window." + }, + { + "date": "2026-08-13", + "runner": "local", + "platform": "macos", + "command": "SKIP_BUILD=1 pnpm exec playwright test e2e/renderer-crash-recovery-terminal-input.spec.ts tests/e2e/renderer-crash-recovery-terminal-input.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 18, + "summary": "Two forced renderer process deaths each produced a main-observed recovery load; the same daemon-backed PTY remained live through renderer transport and direct-write probes." + }, + { + "date": "2026-08-13", + "runner": "local", + "platform": "macos", + "command": "SKIP_BUILD=1 pnpm exec playwright test e2e/headless-serve-desktop-activation.spec.ts tests/e2e/headless-serve-desktop-activation.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 10, + "summary": "The isolated headless serve owner promoted to a real renderer while preserving main owner, runtime, daemon, and PTY identity. Pre-promotion output remained visible and post-promotion terminal input/output succeeded; a full-window proof image records both markers." + } + ], + "runtimeBudget": { + "p95Seconds": 420, + "scope": "focused deterministic contracts plus isolated forced-crash and headless-promotion Electron journeys" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "Fresh deterministic and two isolated Electron journeys are green; CI soak history is pending." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "A byte-identical three-case oracle was rerun on 2026-08-13. Latest main ee8dd4796e failed all three: reload remained reloading after 22.5 seconds, the prior renderer document settled the replacement, and a competing window stole failed-promotion authority. Submitted PR head feeca6c22f passed the bounded replacement timeout but failed both authority cases. The locally rewritten candidate passed all three. Disabling the local document-generation and pending-window fences reproduces the submitted-head failures; disabling the cancelled-navigation classifier leaves the graph reloading and increments its epoch in the new live oracle." + }, + "performanceBudget": { + "required": true, + "evidence": "Fresh delegated audit passed with no blocking, major, or actionable minor findings. The candidate retains at most one unref'd timer, reuses the existing renderer document UUID, adds about 67 serialized bytes to each already-coalesced local graph IPC, and performs constant-time generation/window checks. It adds no polling, subprocess, network request, remote frame, provider scan, or steady-state fanout; a dedicated benchmark would measure existing graph/structured-clone noise rather than meaningful incremental work." + }, + "promotionCriteria": [ + "Collect CI and soak history without unexplained flakes.", + "Collect live Linux and Windows desktop reload evidence.", + "Add live SSH or WSL promotion evidence if provider-specific graph restoration diverges." + ], + "knownGaps": [ + "The live journeys cover macOS and daemon-backed local terminals, not Linux desktop, Windows, WSL, or SSH promotion.", + "The same-frame stale-document race is deterministic fault injection because Playwright cannot schedule an invoke precisely between did-start-loading and frame replacement.", + "No mixed-version live run is required because rendererGeneration is confined to same-version preload/main Electron IPC and the remote runtime wire is unchanged." + ], + "demotionRule": "Demote if a reload revision settles more than once, a stale document or competing window can publish, a timer survives settlement, desktop failure stays indefinitely reloading, headless fallback returns after successful promotion, or either isolated Electron journey loses runtime or PTY identity." + }, { "id": "runtime.websocket-heartbeat-cadence", "title": "Runtime WebSockets enter an owned shared heartbeat cadence", diff --git a/src/main/ipc/runtime.test.ts b/src/main/ipc/runtime.test.ts index 0a884aedfba..a05dc315bec 100644 --- a/src/main/ipc/runtime.test.ts +++ b/src/main/ipc/runtime.test.ts @@ -42,13 +42,51 @@ describe('registerRuntimeHandlers', () => { fromWebContentsMock.mockReturnValue({ id: 17 }) + const currentMainFrame = {} + const sender = { mainFrame: currentMainFrame } const handler = syncRegistration![1] - const result = handler({ sender: {} }, { tabs: [], leaves: [] }) + const graph = { tabs: [], leaves: [], rendererGeneration: 'renderer-1' } + const result = handler({ sender, senderFrame: currentMainFrame }, graph) - expect(runtime.syncWindowGraph).toHaveBeenCalledWith(17, { tabs: [], leaves: [] }) + expect(runtime.syncWindowGraph).toHaveBeenCalledWith(17, graph) expect(result).toEqual({ graphStatus: 'ready' }) }) + it('rejects a graph publication queued by a superseded main frame', () => { + const runtime = { + syncWindowGraph: vi.fn(), + getStatus: vi.fn(), + getRuntimeId: vi.fn() + } + registerRuntimeHandlers(runtime as never) + const handler = handleMock.mock.calls.find( + ([channel]) => channel === 'runtime:syncWindowGraph' + )![1] + const sender = { mainFrame: { generation: 2 } } + fromWebContentsMock.mockReturnValue({ id: 17 }) + + expect(() => + handler({ sender, senderFrame: { generation: 1 } }, { tabs: [], leaves: [] }) + ).toThrow('Runtime graph sync must originate from the current main frame') + expect(runtime.syncWindowGraph).not.toHaveBeenCalled() + }) + + it('rejects graph publications without a renderer generation', () => { + const runtime = { syncWindowGraph: vi.fn() } + registerRuntimeHandlers(runtime as never) + const handler = handleMock.mock.calls.find( + ([channel]) => channel === 'runtime:syncWindowGraph' + )![1] + const currentMainFrame = {} + const sender = { mainFrame: currentMainFrame } + fromWebContentsMock.mockReturnValue({ id: 17 }) + + expect(() => + handler({ sender, senderFrame: currentMainFrame }, { tabs: [], leaves: [] }) + ).toThrow('Runtime graph sync requires a renderer generation') + expect(runtime.syncWindowGraph).not.toHaveBeenCalled() + }) + it('routes generic local runtime RPC calls through the dispatcher', async () => { const runtime = { syncWindowGraph: vi.fn(), diff --git a/src/main/ipc/runtime.ts b/src/main/ipc/runtime.ts index fbd3e7630d9..dfa4dc7eba4 100644 --- a/src/main/ipc/runtime.ts +++ b/src/main/ipc/runtime.ts @@ -2,9 +2,9 @@ import { BrowserWindow, ipcMain } from 'electron' import type { OrcaRuntimeService } from '../runtime/orca-runtime' import type { RuntimeBrowserDriverState, + RuntimeRendererSyncWindowGraph, RuntimeStatus, RuntimeSyncWindowGraphResult, - RuntimeSyncWindowGraph, RuntimeTerminalDriverState } from '../../shared/runtime-types' import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' @@ -28,11 +28,19 @@ export function registerRuntimeHandlers(runtime: OrcaRuntimeService): void { ipcMain.handle( 'runtime:syncWindowGraph', - (event, graph: RuntimeSyncWindowGraph): RuntimeSyncWindowGraphResult => { + (event, graph: RuntimeRendererSyncWindowGraph): RuntimeSyncWindowGraphResult => { const window = BrowserWindow.fromWebContents(event.sender) if (!window) { throw new Error('Runtime graph sync must originate from a BrowserWindow') } + if (event.senderFrame !== event.sender.mainFrame) { + // Why: a disposed main frame can leave an invoke queued after its + // replacement starts. It must not settle the replacement generation. + throw new Error('Runtime graph sync must originate from the current main frame') + } + if (typeof graph.rendererGeneration !== 'string' || graph.rendererGeneration.length === 0) { + throw new Error('Runtime graph sync requires a renderer generation') + } return runtime.syncWindowGraph(window.id, graph) } ) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index f651bb7e3fa..c6afbb3280e 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -63,6 +63,7 @@ import { resolveWorktreeScanCacheTtlMs, type RuntimeTerminalAgentStatusEvent } from './orca-runtime' +import { RUNTIME_GRAPH_RELOAD_TIMEOUT_MS } from './runtime-graph-reload-lifecycle' import { appendRecentPtyPathCandidates, recentTerminalPathCandidatesIncludePath, @@ -2489,6 +2490,62 @@ describe('OrcaRuntimeService', () => { expect(runtime.getStatus().graphStatus).toBe('ready') }) + it('restores a surviving renderer when its reload is cancelled', () => { + const runtime = createRuntime() + runtime.attachWindow(TEST_WINDOW_ID) + runtime.markGraphReady(TEST_WINDOW_ID) + const fence = runtime.markRendererReloading(TEST_WINDOW_ID) + if (fence === null) { + throw new Error('expected active renderer reload fence') + } + + expect(fence.recovery).toBe('renderer') + expect(runtime.getStatus().graphStatus).toBe('reloading') + expect(runtime.markRendererReloadCancelled(TEST_WINDOW_ID, fence)).toBe(true) + expect(runtime.getStatus().graphStatus).toBe('ready') + }) + + it('keeps an earlier committed reload fenced when a later reload is cancelled', async () => { + vi.useFakeTimers() + try { + const runtime = createRuntime() + runtime.attachWindow(TEST_WINDOW_ID) + runtime.markGraphReady(TEST_WINDOW_ID) + const committedFence = runtime.markRendererReloading(TEST_WINDOW_ID) + const cancelledFence = runtime.markRendererReloading(TEST_WINDOW_ID) + if (committedFence === null || cancelledFence === null) { + throw new Error('expected active renderer reload fences') + } + + expect(cancelledFence.recovery).toBe('reloading') + expect(runtime.markRendererReloadCancelled(TEST_WINDOW_ID, committedFence)).toBe(false) + expect(runtime.markRendererReloadCancelled(TEST_WINDOW_ID, cancelledFence)).toBe(false) + await vi.advanceTimersByTimeAsync(RUNTIME_GRAPH_RELOAD_TIMEOUT_MS - 1) + expect(runtime.getStatus().graphStatus).toBe('reloading') + await vi.advanceTimersByTimeAsync(1) + expect(runtime.getStatus().graphStatus).toBe('unavailable') + } finally { + vi.useRealTimers() + } + }) + + it('restores headless authority when desktop promotion navigation is cancelled', () => { + const runtime = createRuntime() + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) + runtime.attachWindow(TEST_WINDOW_ID) + const fence = runtime.markRendererReloading(TEST_WINDOW_ID) + if (fence === null) { + throw new Error('expected active promotion reload fence') + } + + expect(fence.recovery).toBe('headless') + expect(runtime.markRendererReloadCancelled(TEST_WINDOW_ID, fence)).toBe(false) + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: HEADLESS_RUNTIME_WINDOW_ID, + graphStatus: 'ready' + }) + }) + it('drops back to unavailable and clears authority when the window disappears', () => { const runtime = createRuntime() @@ -2504,6 +2561,164 @@ describe('OrcaRuntimeService', () => { }) }) + it('restores headless graph authority after a promoted renderer reload times out', async () => { + vi.useFakeTimers() + try { + const runtime = createRuntime() + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) + runtime.registerPty('persisted-pty', TEST_WORKTREE_ID, null, { + tabId: 'host-tab', + leafId: HEADLESS_LEAF_ID + }) + runtime.attachWindow(TEST_WINDOW_ID) + + await vi.advanceTimersByTimeAsync(RUNTIME_GRAPH_RELOAD_TIMEOUT_MS) + + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: HEADLESS_RUNTIME_WINDOW_ID, + graphStatus: 'ready' + }) + expect((await runtime.listTerminals()).terminals).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ptyId: 'persisted-pty', connected: true, writable: true }) + ]) + ) + } finally { + vi.useRealTimers() + } + }) + + it('moves a desktop graph to unavailable when its reload times out', async () => { + vi.useFakeTimers() + try { + const runtime = createRuntime() + runtime.attachWindow(TEST_WINDOW_ID) + runtime.markGraphReady(TEST_WINDOW_ID) + runtime.markRendererReloading(TEST_WINDOW_ID) + + await vi.advanceTimersByTimeAsync(RUNTIME_GRAPH_RELOAD_TIMEOUT_MS) + + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: TEST_WINDOW_ID, + graphStatus: 'unavailable', + rendererGraphEpoch: 1 + }) + } finally { + vi.useRealTimers() + } + }) + + it('recovers a failed headless promotion and accepts a later renderer generation', () => { + const runtime = createRuntime() + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) + runtime.attachWindow(TEST_WINDOW_ID) + + runtime.markGraphReloadFailed(TEST_WINDOW_ID, 'renderer-process-gone') + + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: HEADLESS_RUNTIME_WINDOW_ID, + graphStatus: 'ready' + }) + + runtime.syncWindowGraph(TEST_WINDOW_ID, { tabs: [], leaves: [] }) + + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: TEST_WINDOW_ID, + graphStatus: 'ready' + }) + }) + + it('retires the headless fallback after renderer promotion succeeds', () => { + const runtime = createRuntime() + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) + runtime.attachWindow(TEST_WINDOW_ID) + runtime.syncWindowGraph(TEST_WINDOW_ID, { tabs: [], leaves: [] }) + + runtime.markRendererReloading(TEST_WINDOW_ID) + runtime.markGraphReloadFailed(TEST_WINDOW_ID, 'renderer-process-gone') + + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: TEST_WINDOW_ID, + graphStatus: 'unavailable' + }) + + runtime.syncWindowGraph(TEST_WINDOW_ID, { tabs: [], leaves: [] }) + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: TEST_WINDOW_ID, + graphStatus: 'ready' + }) + }) + + it('does not let a superseded reload timeout overwrite a newer renderer graph', async () => { + vi.useFakeTimers() + try { + const runtime = createRuntime() + runtime.attachWindow(TEST_WINDOW_ID) + runtime.markGraphReady(TEST_WINDOW_ID) + runtime.markRendererReloading(TEST_WINDOW_ID) + await vi.advanceTimersByTimeAsync(RUNTIME_GRAPH_RELOAD_TIMEOUT_MS / 2) + runtime.markRendererReloading(TEST_WINDOW_ID) + + await vi.advanceTimersByTimeAsync(RUNTIME_GRAPH_RELOAD_TIMEOUT_MS / 2) + + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: TEST_WINDOW_ID, + graphStatus: 'reloading' + }) + + await vi.advanceTimersByTimeAsync(RUNTIME_GRAPH_RELOAD_TIMEOUT_MS / 2) + + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: TEST_WINDOW_ID, + graphStatus: 'unavailable' + }) + } finally { + vi.useRealTimers() + } + }) + + it('rejects a same-frame graph publication from the superseded renderer generation', () => { + const runtime = createRuntime() + runtime.attachWindow(TEST_WINDOW_ID) + runtime.syncWindowGraph(TEST_WINDOW_ID, { + tabs: [], + leaves: [], + rendererGeneration: 'renderer-a' + }) + runtime.markRendererReloading(TEST_WINDOW_ID) + + expect(() => + runtime.syncWindowGraph(TEST_WINDOW_ID, { + tabs: [], + leaves: [], + rendererGeneration: 'renderer-a' + }) + ).toThrow('Runtime graph publisher belongs to a superseded renderer generation') + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: TEST_WINDOW_ID, + graphStatus: 'reloading' + }) + }) + + it('keeps a restored headless graph pinned to the failed promotion window', () => { + const runtime = createRuntime() + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) + runtime.attachWindow(TEST_WINDOW_ID) + runtime.markGraphReloadFailed(TEST_WINDOW_ID, 'renderer-process-gone') + + expect(() => + runtime.syncWindowGraph(2, { + tabs: [], + leaves: [], + rendererGeneration: 'renderer-b' + }) + ).toThrow('Runtime graph publisher does not match the pending desktop promotion') + expect(runtime.getStatus()).toMatchObject({ + authoritativeWindowId: HEADLESS_RUNTIME_WINDOW_ID, + graphStatus: 'ready' + }) + }) + it('stays unavailable during initial loads before a graph is published', () => { const runtime = createRuntime() diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index e5e614eba93..a0707815ab3 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -388,11 +388,16 @@ import { type RuntimeSessionTabCloseReason, type RuntimeBrowserDriverState, type RuntimeTerminalDriverState, + type RuntimeRendererSyncWindowGraph, type RuntimeSyncWindowGraph, type RuntimeWorktreeListResult, type BrowserTabInfo, type BrowserScreencastResult } from '../../shared/runtime-types' +import { + RUNTIME_GRAPH_RELOAD_TIMEOUT_MS, + RuntimeGraphReloadLifecycle +} from './runtime-graph-reload-lifecycle' import { LINEAR_SEARCH_MAX_LIMIT, LINEAR_WRITE_BODY_CAP, @@ -2754,6 +2759,11 @@ function getSetupRunnerCommandPlatformForLaunch( return getSetupRunnerCommandPlatformForPath(setup?.runnerScriptPath ?? '', fallbackPlatform) } +export type RuntimeRendererReloadFence = Readonly<{ + revision: number + recovery: 'renderer' | 'headless' | 'reloading' +}> + export class OrcaRuntimeService { private readonly runtimeId = randomUUID() private readonly startedAt = Date.now() @@ -2774,6 +2784,18 @@ export class OrcaRuntimeService { private rendererGraphEpoch = 0 private graphStatus: RuntimeGraphStatus = 'unavailable' private authoritativeWindowId: number | null = null + private headlessGraphFallbackAvailable = false + private pendingHeadlessPromotionWindowId: number | null = null + private rendererGeneration: string | null = null + private readonly graphReloadLifecycle = new RuntimeGraphReloadLifecycle({ + timeoutMs: RUNTIME_GRAPH_RELOAD_TIMEOUT_MS, + onSettled: ({ revision, windowId, outcome, durationMs }) => { + console.info( + `[runtime-graph] reload revision=${revision} window=${windowId} outcome=${outcome} durationMs=${durationMs}` + ) + }, + onTimeout: (_revision, windowId) => this.handleGraphReloadTimeout(windowId) + }) // Why: paired graph transactions need foreground timer cadence only until their publication settles. private readonly rendererPublicationThrottle = new RendererPublicationThrottle() private tabs = new Map() @@ -5531,11 +5553,18 @@ export class OrcaRuntimeService { attachWindow(windowId: number): void { if (this.authoritativeWindowId === HEADLESS_RUNTIME_WINDOW_ID) { + if ( + this.pendingHeadlessPromotionWindowId !== null && + windowId !== this.pendingHeadlessPromotionWindowId + ) { + return + } // Why: promotion is a renderer reload of the same graph owner, not a new // runtime; stale handles must transition before the real window publishes. this.persistWindowlessPtyBindingsForDesktopAttach() - this.markRendererReloading(HEADLESS_RUNTIME_WINDOW_ID) + this.pendingHeadlessPromotionWindowId = windowId this.authoritativeWindowId = windowId + this.beginGraphReload(windowId) return } if (this.authoritativeWindowId === null) { @@ -5612,13 +5641,45 @@ export class OrcaRuntimeService { } } - syncWindowGraph(windowId: number, graph: RuntimeSyncWindowGraph): RuntimeSyncWindowGraphResult { + syncWindowGraph( + windowId: number, + graph: RuntimeSyncWindowGraph | RuntimeRendererSyncWindowGraph + ): RuntimeSyncWindowGraphResult { + if ( + windowId !== HEADLESS_RUNTIME_WINDOW_ID && + this.authoritativeWindowId === HEADLESS_RUNTIME_WINDOW_ID && + this.headlessGraphFallbackAvailable + ) { + if (windowId !== this.pendingHeadlessPromotionWindowId) { + throw new Error('Runtime graph publisher does not match the pending desktop promotion') + } + // Why: a renderer may publish after a failed promotion was restored to + // headless authority; accepting that late healthy graph is self-healing. + this.attachWindow(windowId) + } if (this.authoritativeWindowId === null) { this.authoritativeWindowId = windowId } if (windowId !== this.authoritativeWindowId) { throw new Error('Runtime graph publisher does not match the authoritative window') } + const rendererGeneration = + windowId === HEADLESS_RUNTIME_WINDOW_ID + ? null + : 'rendererGeneration' in graph && typeof graph.rendererGeneration === 'string' + ? graph.rendererGeneration + : undefined + if ( + typeof rendererGeneration === 'string' && + rendererGeneration === this.rendererGeneration && + this.graphStatus !== 'ready' + ) { + throw new Error('Runtime graph publisher belongs to a superseded renderer generation') + } + if (windowId === HEADLESS_RUNTIME_WINDOW_ID) { + this.headlessGraphFallbackAvailable = true + this.rendererGeneration = null + } const graphWasReady = this.graphStatus === 'ready' const previousTabs = this.tabs @@ -5786,9 +5847,10 @@ export class OrcaRuntimeService { this.mobileSessionTabsNotifyCoalescer.schedule(worktreeId) } } - this.graphStatus = 'ready' - this.setTerminalSideEffectConsumerAvailable(windowId !== HEADLESS_RUNTIME_WINDOW_ID) - this.refreshWritableFlags() + this.markGraphReady(windowId) + if (rendererGeneration !== undefined) { + this.rendererGeneration = rendererGeneration + } for (const leaf of this.leaves.values()) { this.adoptPreAllocatedHandle(leaf) const previousLeaf = previousLeaves.get(this.getLeafKey(leaf.tabId, leaf.leafId)) @@ -28602,16 +28664,38 @@ export class OrcaRuntimeService { return false } - markRendererReloading(windowId: number): void { + markRendererReloading(windowId: number): RuntimeRendererReloadFence | null { + if ( + windowId !== HEADLESS_RUNTIME_WINDOW_ID && + this.authoritativeWindowId === HEADLESS_RUNTIME_WINDOW_ID && + this.headlessGraphFallbackAvailable + ) { + this.attachWindow(windowId) + const revision = this.graphReloadLifecycle.getActiveRevision() + return this.authoritativeWindowId === windowId && revision !== null + ? { revision, recovery: 'headless' } + : null + } if (windowId !== this.authoritativeWindowId) { - return + return null + } + if (this.graphStatus === 'reloading') { + return { + revision: this.graphReloadLifecycle.begin(windowId), + recovery: this.shouldRestoreHeadlessGraph(windowId) ? 'headless' : 'reloading' + } } if (this.graphStatus !== 'ready') { - return + return null } + return { revision: this.beginGraphReload(windowId), recovery: 'renderer' } + } + + private beginGraphReload(windowId: number): number { // Why: a renderer reload tears down the live graph, so live handles must go stale immediately, not be reused against the rebuild. this.rendererGraphEpoch += 1 this.graphStatus = 'reloading' + const revision = this.graphReloadLifecycle.begin(windowId) this.setTerminalSideEffectConsumerAvailable(false) this.rememberDetachedPreAllocatedLeaves() this.handles.clear() @@ -28619,21 +28703,76 @@ export class OrcaRuntimeService { // Why: handleByPtyId (pre-allocated CLI handles) survives reloads so CLI agents keep control; adoptPreAllocatedHandle re-links on the new graph. this.rejectAllWaiters('terminal_handle_stale') this.refreshWritableFlags() + return revision + } + + markRendererReloadCancelled(windowId: number, fence: RuntimeRendererReloadFence): boolean { + if ( + windowId !== this.authoritativeWindowId || + this.graphStatus !== 'reloading' || + !this.graphReloadLifecycle.settle(fence.revision, 'cancelled') + ) { + return false + } + if (fence.recovery === 'headless' && this.shouldRestoreHeadlessGraph(windowId)) { + this.restoreHeadlessGraphAuthority() + return false + } + if (fence.recovery === 'renderer') { + this.graphStatus = 'ready' + this.setTerminalSideEffectConsumerAvailable(true) + this.refreshWritableFlags() + return true + } + this.graphReloadLifecycle.begin(windowId) + return false } markGraphReady(windowId: number): void { if (windowId !== this.authoritativeWindowId) { return } + this.graphReloadLifecycle.settleActive('success') + if (windowId !== HEADLESS_RUNTIME_WINDOW_ID) { + this.headlessGraphFallbackAvailable = false + this.pendingHeadlessPromotionWindowId = null + } this.graphStatus = 'ready' this.setTerminalSideEffectConsumerAvailable(windowId !== HEADLESS_RUNTIME_WINDOW_ID) this.refreshWritableFlags() } - markGraphUnavailable(windowId: number): void { + markGraphReloadFailed( + windowId: number, + _reason: 'renderer-frame-unavailable' | 'renderer-process-gone' + ): void { if (windowId !== this.authoritativeWindowId) { return } + if (this.graphStatus === 'ready') { + this.beginGraphReload(windowId) + } + this.graphReloadLifecycle.settleActive('failure') + this.transitionGraphReloadToTerminalState(windowId) + } + + markGraphUnavailable(windowId: number): void { + if ( + this.authoritativeWindowId === HEADLESS_RUNTIME_WINDOW_ID && + windowId === this.pendingHeadlessPromotionWindowId + ) { + this.pendingHeadlessPromotionWindowId = null + return + } + if (windowId !== this.authoritativeWindowId) { + return + } + this.graphReloadLifecycle.settleActive('cancelled') + if (this.shouldRestoreHeadlessGraph(windowId)) { + this.pendingHeadlessPromotionWindowId = null + this.restoreHeadlessGraphAuthority() + return + } // Why: once the authoritative renderer graph disappears, fail closed for live-terminal ops instead of guessing from old state. if (this.graphStatus !== 'unavailable') { this.rendererGraphEpoch += 1 @@ -28651,6 +28790,49 @@ export class OrcaRuntimeService { this.rejectAllWaiters('terminal_handle_stale') } + private handleGraphReloadTimeout(windowId: number): void { + if (windowId !== this.authoritativeWindowId || this.graphStatus !== 'reloading') { + return + } + this.transitionGraphReloadToTerminalState(windowId) + } + + private transitionGraphReloadToTerminalState(windowId: number): void { + if (this.shouldRestoreHeadlessGraph(windowId)) { + this.restoreHeadlessGraphAuthority() + return + } + this.graphStatus = 'unavailable' + this.setTerminalSideEffectConsumerAvailable(false) + this.rememberDetachedPreAllocatedLeaves() + this.tabs.clear() + this.leaves.clear() + this.leavesByPtyId.clear() + this.handles.clear() + this.handleByLeafKey.clear() + this.rejectAllWaiters('terminal_handle_stale') + this.refreshWritableFlags() + } + + private shouldRestoreHeadlessGraph(windowId: number): boolean { + return windowId !== HEADLESS_RUNTIME_WINDOW_ID && this.headlessGraphFallbackAvailable + } + + private restoreHeadlessGraphAuthority(): void { + this.rendererGraphEpoch += 1 + this.authoritativeWindowId = HEADLESS_RUNTIME_WINDOW_ID + this.graphStatus = 'ready' + this.rendererGeneration = null + this.setTerminalSideEffectConsumerAvailable(false) + this.tabs.clear() + this.leaves.clear() + this.leavesByPtyId.clear() + this.handles.clear() + this.handleByLeafKey.clear() + this.rejectAllWaiters('terminal_handle_stale') + this.refreshWritableFlags() + } + private assertGraphReady(): void { if (this.graphStatus !== 'ready') { throw new Error('runtime_unavailable') diff --git a/src/main/runtime/runtime-graph-reload-lifecycle.test.ts b/src/main/runtime/runtime-graph-reload-lifecycle.test.ts new file mode 100644 index 00000000000..cdd64fd6493 --- /dev/null +++ b/src/main/runtime/runtime-graph-reload-lifecycle.test.ts @@ -0,0 +1,104 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + RuntimeGraphReloadLifecycle, + type RuntimeGraphReloadSettlement +} from './runtime-graph-reload-lifecycle' + +describe('RuntimeGraphReloadLifecycle', () => { + afterEach(() => { + vi.useRealTimers() + }) + + it('records success, failure, cancellation, and timeout as terminal outcomes', async () => { + vi.useFakeTimers() + const settlements: RuntimeGraphReloadSettlement[] = [] + const timeouts: number[] = [] + const lifecycle = new RuntimeGraphReloadLifecycle({ + timeoutMs: 100, + onSettled: (settlement) => settlements.push(settlement), + onTimeout: (revision) => timeouts.push(revision) + }) + + const success = lifecycle.begin(1) + expect(lifecycle.settle(success, 'success')).toBe(true) + const failure = lifecycle.begin(1) + expect(lifecycle.settle(failure, 'failure')).toBe(true) + const cancelled = lifecycle.begin(1) + expect(lifecycle.settle(cancelled, 'cancelled')).toBe(true) + const timeout = lifecycle.begin(1) + await vi.advanceTimersByTimeAsync(100) + + expect(settlements.map(({ revision, outcome }) => ({ revision, outcome }))).toEqual([ + { revision: success, outcome: 'success' }, + { revision: failure, outcome: 'failure' }, + { revision: cancelled, outcome: 'cancelled' }, + { revision: timeout, outcome: 'timeout' } + ]) + expect(timeouts).toEqual([timeout]) + expect(lifecycle.getActiveRevision()).toBeNull() + }) + + it('cancels a superseded generation and ignores its stale completion and timeout', async () => { + vi.useFakeTimers() + const settlements: RuntimeGraphReloadSettlement[] = [] + const timeouts: number[] = [] + const lifecycle = new RuntimeGraphReloadLifecycle({ + timeoutMs: 100, + onSettled: (settlement) => settlements.push(settlement), + onTimeout: (revision) => timeouts.push(revision) + }) + + const first = lifecycle.begin(1) + await vi.advanceTimersByTimeAsync(60) + const second = lifecycle.begin(1) + + expect(lifecycle.settle(first, 'success')).toBe(false) + await vi.advanceTimersByTimeAsync(40) + expect(timeouts).toEqual([]) + expect(lifecycle.getActiveRevision()).toBe(second) + + await vi.advanceTimersByTimeAsync(60) + expect(timeouts).toEqual([second]) + expect(settlements.map(({ revision, outcome }) => ({ revision, outcome }))).toEqual([ + { revision: first, outcome: 'cancelled' }, + { revision: second, outcome: 'timeout' } + ]) + }) + + it('keeps a generation started by a cancellation observer active', async () => { + vi.useFakeTimers() + const settlements: RuntimeGraphReloadSettlement[] = [] + const timeouts: number[] = [] + let nestedRevision: number | null = null + let didReenter = false + let lifecycle!: RuntimeGraphReloadLifecycle + lifecycle = new RuntimeGraphReloadLifecycle({ + timeoutMs: 100, + onSettled: (settlement) => { + settlements.push(settlement) + if (settlement.outcome === 'cancelled' && !didReenter) { + didReenter = true + nestedRevision = lifecycle.begin(3) + } + }, + onTimeout: (revision) => timeouts.push(revision) + }) + + const first = lifecycle.begin(1) + const replacement = lifecycle.begin(2) + + expect(replacement).toBe(2) + expect(nestedRevision).toBe(3) + expect(lifecycle.getActiveRevision()).toBe(nestedRevision) + expect(lifecycle.settle(replacement, 'success')).toBe(false) + + await vi.advanceTimersByTimeAsync(100) + + expect(timeouts).toEqual([nestedRevision]) + expect(settlements.map(({ revision, outcome }) => ({ revision, outcome }))).toEqual([ + { revision: first, outcome: 'cancelled' }, + { revision: replacement, outcome: 'cancelled' }, + { revision: nestedRevision, outcome: 'timeout' } + ]) + }) +}) diff --git a/src/main/runtime/runtime-graph-reload-lifecycle.ts b/src/main/runtime/runtime-graph-reload-lifecycle.ts new file mode 100644 index 00000000000..36253d544ea --- /dev/null +++ b/src/main/runtime/runtime-graph-reload-lifecycle.ts @@ -0,0 +1,87 @@ +export const RUNTIME_GRAPH_RELOAD_TIMEOUT_MS = 15_000 + +export type RuntimeGraphReloadOutcome = 'success' | 'failure' | 'cancelled' | 'timeout' + +export type RuntimeGraphReloadSettlement = Readonly<{ + revision: number + windowId: number + outcome: RuntimeGraphReloadOutcome + durationMs: number +}> + +type ActiveRuntimeGraphReload = Readonly<{ + revision: number + windowId: number + startedAt: number + timer: ReturnType +}> + +export class RuntimeGraphReloadLifecycle { + private revision = 0 + private active: ActiveRuntimeGraphReload | null = null + + constructor( + private readonly options: { + timeoutMs: number + onSettled?: (settlement: RuntimeGraphReloadSettlement) => void + onTimeout?: (revision: number, windowId: number) => void + } + ) {} + + begin(windowId: number): number { + const cancelled = this.active ? this.finish(this.active.revision, 'cancelled') : null + + const revision = ++this.revision + const startedAt = Date.now() + const timer = setTimeout(() => { + const settlement = this.finish(revision, 'timeout') + if (!settlement) { + return + } + this.options.onSettled?.(settlement) + this.options.onTimeout?.(revision, windowId) + }, this.options.timeoutMs) + timer.unref?.() + this.active = { revision, windowId, startedAt, timer } + if (cancelled) { + this.options.onSettled?.(cancelled) + } + return revision + } + + settle(revision: number, outcome: RuntimeGraphReloadOutcome): boolean { + const settlement = this.finish(revision, outcome) + if (!settlement) { + return false + } + this.options.onSettled?.(settlement) + return true + } + + private finish( + revision: number, + outcome: RuntimeGraphReloadOutcome + ): RuntimeGraphReloadSettlement | null { + const active = this.active + if (!active || active.revision !== revision) { + return null + } + + clearTimeout(active.timer) + this.active = null + return { + revision, + windowId: active.windowId, + outcome, + durationMs: Math.max(0, Date.now() - active.startedAt) + } + } + + settleActive(outcome: RuntimeGraphReloadOutcome): boolean { + return this.active ? this.settle(this.active.revision, outcome) : false + } + + getActiveRevision(): number | null { + return this.active?.revision ?? null + } +} diff --git a/src/main/window/attach-main-window-services.test.ts b/src/main/window/attach-main-window-services.test.ts index ac473733753..e18f8a33e39 100644 --- a/src/main/window/attach-main-window-services.test.ts +++ b/src/main/window/attach-main-window-services.test.ts @@ -121,6 +121,7 @@ type MainWindowStub = { once: MockFn webContents: { id?: number + getURL: MockFn isDestroyed?: MockFn isLoadingMainFrame: MockFn on: MockFn @@ -137,6 +138,8 @@ type RuntimeStub = { attachWindow: MockFn setNotifier: MockFn markRendererReloading: MockFn + markRendererReloadCancelled: MockFn + markGraphReloadFailed: MockFn markGraphUnavailable: MockFn } @@ -150,6 +153,7 @@ function createMainWindow( once: vi.fn(), webContents: { id: 1, + getURL: vi.fn(() => 'file:///opt/orca/renderer/index.html'), isDestroyed: vi.fn(() => false), isLoadingMainFrame: vi.fn(() => true), on: vi.fn(), @@ -175,6 +179,8 @@ function createRuntime(): RuntimeStub { attachWindow: vi.fn(), setNotifier: vi.fn(), markRendererReloading: vi.fn(), + markRendererReloadCancelled: vi.fn(), + markGraphReloadFailed: vi.fn(), markGraphUnavailable: vi.fn() } } @@ -798,6 +804,21 @@ describe('attachMainWindowServices', () => { ) }) + it('marks renderer process loss as a graph reload failure', () => { + const mainWindow = createMainWindow() + const runtime = createRuntime() + attachMainWindowServices(mainWindow as never, createStore(), runtime as never) + + const handlers = mainWindow.webContents.on.mock.calls + .filter(([event]) => event === 'render-process-gone') + .map(([, handler]) => handler as () => void) + for (const handler of handlers) { + handler() + } + + expect(runtime.markGraphReloadFailed).toHaveBeenCalledWith(1, 'renderer-process-gone') + }) + it('accepts terminal reveal replies only from the main window renderer', async () => { const sendMock = vi.fn() const mainWindow = createMainWindow({ send: sendMock }) diff --git a/src/main/window/attach-main-window-services.ts b/src/main/window/attach-main-window-services.ts index 0fbd850f06b..4ad095bef15 100644 --- a/src/main/window/attach-main-window-services.ts +++ b/src/main/window/attach-main-window-services.ts @@ -64,6 +64,8 @@ import { setWorktreeBaseDirectoryWatcherSyncContext } from '../ipc/worktree-base-directory-watcher' import { logStartupMilestone } from '../startup/startup-diagnostics' +import { createRuntimeRendererNotificationSender } from './runtime-renderer-notification-sender' +import { registerRendererDocumentNavigation } from './renderer-document-navigation' const UPDATER_SETUP_FALLBACK_MS = 15_000 @@ -318,11 +320,13 @@ function registerRuntimeWindowLifecycle( const notifierToken = ++runtimeNotifierTokenCounter activeRuntimeNotifierToken = notifierToken runtime.attachWindow(mainWindow.id) - const send = (channel: string, ...args: unknown[]): void => { - if (!mainWindow.isDestroyed()) { - mainWindow.webContents.send(channel, ...args) - } - } + const mainWebContents = mainWindow.webContents + const rendererNotifications = createRuntimeRendererNotificationSender({ + isWindowDestroyed: () => mainWindow.isDestroyed(), + webContents: mainWebContents, + onFailure: (reason) => runtime.markGraphReloadFailed(mainWindow.id, reason) + }) + const send = rendererNotifications.send runtime.setNotifier({ worktreesChanged: (repoId, renamed) => { // Why: clear scan caches before the renderer handles this event, so it can't read stale TTL entries after a mutation. @@ -401,7 +405,7 @@ function registerRuntimeWindowLifecycle( }) } ipcMain.on('terminal:tabCreateReply', handler) - send('ui:createTerminal', { + const sent = send('ui:createTerminal', { requestId, worktreeId, ptyId: opts.ptyId, @@ -424,6 +428,11 @@ function registerRuntimeWindowLifecycle( : {}), ...(opts.focus !== undefined ? { focus: opts.focus } : {}) }) + if (!sent) { + clearTimeout(timer) + ipcMain.removeListener('terminal:tabCreateReply', handler) + reject(new Error('runtime_unavailable')) + } }), resolveLegacyWorkerTerminalRecovery: (paneKey, resolution, ptyId) => send('agentStatus:legacyWorkerTerminalRecovery', { @@ -489,11 +498,23 @@ function registerRuntimeWindowLifecycle( browserDriverChanged: (browserPageId, driver) => send('runtime:browserDriverChanged', { browserPageId, driver }) }) - // Why: fail closed during renderer reload so CLI calls can't act on stale terminal mappings. - mainWindow.webContents.on('did-start-loading', () => { - runtime.markRendererReloading(mainWindow.id) + registerRendererDocumentNavigation(mainWebContents, () => { + rendererNotifications.onMainFrameReloadStarted() + const fence = runtime.markRendererReloading(mainWindow.id) + return () => { + if (fence && runtime.markRendererReloadCancelled(mainWindow.id, fence)) { + rendererNotifications.onMainFrameReloadCancelled() + } + } + }) + mainWebContents.on('did-finish-load', () => { + rendererNotifications.onMainFrameLoadFinished() + }) + mainWebContents.on('render-process-gone', () => { + rendererNotifications.onRendererProcessGone() }) mainWindow.on('closed', () => { + rendererNotifications.close() runtime.markGraphUnavailable(mainWindow.id) if (activeRuntimeNotifierToken === notifierToken) { // Why: the notifier closes over the window; clear it in the no-window gap so the runtime can't retain destroyed graphs. diff --git a/src/main/window/renderer-document-navigation.test.ts b/src/main/window/renderer-document-navigation.test.ts new file mode 100644 index 00000000000..62538b59715 --- /dev/null +++ b/src/main/window/renderer-document-navigation.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it, vi } from 'vitest' +import { registerRendererDocumentNavigation } from './renderer-document-navigation' + +describe('renderer document navigation', () => { + function createFixture(currentUrl: string, onStarted = vi.fn(() => vi.fn())) { + const handlers = new Map void>() + const on = vi.fn((event: string, handler: (...args: unknown[]) => void) => { + handlers.set(event, handler) + }) + registerRendererDocumentNavigation({ getURL: () => currentUrl, on } as never, onStarted) + return { + navigate: handlers.get('did-start-navigation'), + failProvisionalLoad: handlers.get('did-fail-provisional-load'), + willNavigate: handlers.get('will-navigate'), + commitNavigation: handlers.get('did-frame-navigate'), + onStarted + } + } + + it('accepts the packaged renderer document but not a blocked external load', () => { + const fixture = createFixture('file:///opt/orca/renderer/index.html') + + fixture.navigate?.({}, 'https://github.com/stablyai/orca/issues', false, true) + expect(fixture.onStarted).not.toHaveBeenCalled() + fixture.navigate?.({}, 'file:///opt/orca/renderer/index.html?reload=1', false, true) + expect(fixture.onStarted).toHaveBeenCalledOnce() + }) + + it('accepts same-origin development navigation only', () => { + const fixture = createFixture('http://localhost:5173/') + + fixture.navigate?.({}, 'https://example.com/', false, true) + fixture.navigate?.({}, 'http://localhost:5173/settings', false, true) + expect(fixture.onStarted).toHaveBeenCalledOnce() + }) + + it('rejects same-document, subframe, and missing renderer navigation', () => { + const fixture = createFixture('') + + fixture.navigate?.({}, 'http://localhost:5173/', false, true) + fixture.navigate?.({}, 'http://localhost:5173/', true, true) + fixture.navigate?.({}, 'http://localhost:5173/', false, false) + expect(fixture.onStarted).not.toHaveBeenCalled() + }) + + it('cancels only the matching main-frame provisional navigation', () => { + const cancel = vi.fn() + const fixture = createFixture('http://localhost:5173/', vi.fn(() => cancel)) + + fixture.navigate?.({}, 'http://localhost:5173/reload', false, true) + fixture.failProvisionalLoad?.({}, -3, 'aborted', 'other', true, 1, 1) + fixture.failProvisionalLoad?.({}, -3, 'aborted', 'http://localhost:5173/reload', false, 1, 1) + expect(cancel).not.toHaveBeenCalled() + + fixture.failProvisionalLoad?.( + {}, + -3, + 'aborted', + 'http://localhost:5173/reload', + true, + 1, + 1 + ) + expect(cancel).toHaveBeenCalledOnce() + }) + + it('shares one reload fence across concurrent provisional navigations', () => { + const cancel = vi.fn() + const onStarted = vi.fn(() => cancel) + const fixture = createFixture('http://localhost:5173/', onStarted) + + fixture.navigate?.({}, 'http://localhost:5173/reload-a', false, true) + fixture.navigate?.({}, 'http://localhost:5173/reload-b', false, true) + + expect(onStarted).toHaveBeenCalledOnce() + fixture.failProvisionalLoad?.( + {}, + -3, + 'aborted', + 'http://localhost:5173/reload-a', + true, + 1, + 1 + ) + expect(cancel).not.toHaveBeenCalled() + fixture.failProvisionalLoad?.( + {}, + -3, + 'aborted', + 'http://localhost:5173/reload-b', + true, + 1, + 1 + ) + expect(cancel).toHaveBeenCalledOnce() + }) + + it('does not cancel a navigation after its document commits', () => { + const cancel = vi.fn() + const fixture = createFixture('file:///opt/orca/renderer/index.html', vi.fn(() => cancel)) + + fixture.navigate?.({}, 'file:///opt/orca/renderer/index.html?reload=1', false, true) + fixture.commitNavigation?.( + {}, + 'file:///opt/orca/renderer/index.html?reload=1', + -1, + '', + true, + 1, + 1 + ) + fixture.failProvisionalLoad?.( + {}, + -3, + 'aborted', + 'file:///opt/orca/renderer/index.html?reload=1', + true, + 1, + 1 + ) + + expect(cancel).not.toHaveBeenCalled() + }) + + it('cancels when a later will-navigate listener blocks the navigation', async () => { + const cancel = vi.fn() + const fixture = createFixture('http://localhost:5173/', vi.fn(() => cancel)) + const event = { defaultPrevented: false } + + fixture.navigate?.({}, 'http://localhost:5173/reload', false, true) + fixture.willNavigate?.(event, 'http://localhost:5173/reload', false, true, 1, 1) + event.defaultPrevented = true + await Promise.resolve() + + expect(cancel).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/window/renderer-document-navigation.ts b/src/main/window/renderer-document-navigation.ts new file mode 100644 index 00000000000..1fc82a07cd8 --- /dev/null +++ b/src/main/window/renderer-document-navigation.ts @@ -0,0 +1,74 @@ +import type { WebContents } from 'electron' + +function isRendererDocumentNavigation(currentUrl: string, nextUrl: string): boolean { + try { + const current = new URL(currentUrl) + const next = new URL(nextUrl) + if (current.protocol === 'file:') { + return ( + next.protocol === 'file:' && + next.host === current.host && + next.pathname === current.pathname + ) + } + return ( + (current.protocol === 'http:' || current.protocol === 'https:') && + next.origin === current.origin + ) + } catch { + return false + } +} + +export function registerRendererDocumentNavigation( + webContents: Pick, + onStarted: () => (() => void) | void +): void { + const pendingUrls: string[] = [] + let cancelReload: (() => void) | null = null + const cancelPending = (url: string): void => { + const index = pendingUrls.indexOf(url) + if (index !== -1) { + pendingUrls.splice(index, 1) + } + if (pendingUrls.length === 0) { + const cancel = cancelReload + cancelReload = null + cancel?.() + } + } + // Why: did-start-loading also fires for blocked external links whose renderer document survives. + webContents.on('did-start-navigation', (_event, url, isSameDocument, isMainFrame) => { + if (isMainFrame && !isSameDocument && isRendererDocumentNavigation(webContents.getURL(), url)) { + if (pendingUrls.length === 0) { + cancelReload = onStarted() ?? null + } + pendingUrls.push(url) + } + }) + webContents.on( + 'did-fail-provisional-load', + (_event, _errorCode, _errorDescription, validatedUrl, isMainFrame) => { + if (!isMainFrame) { + return + } + cancelPending(validatedUrl) + } + ) + webContents.on('will-navigate', (event, url, _sameDocument, isMainFrame) => { + if (!isMainFrame) { + return + } + queueMicrotask(() => { + if (event.defaultPrevented) { + cancelPending(url) + } + }) + }) + webContents.on('did-frame-navigate', (_event, _url, _code, _status, isMainFrame) => { + if (isMainFrame) { + pendingUrls.length = 0 + cancelReload = null + } + }) +} diff --git a/src/main/window/runtime-renderer-notification-sender.test.ts b/src/main/window/runtime-renderer-notification-sender.test.ts new file mode 100644 index 00000000000..d482bb404fb --- /dev/null +++ b/src/main/window/runtime-renderer-notification-sender.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it, vi } from 'vitest' +import { createRuntimeRendererNotificationSender } from './runtime-renderer-notification-sender' + +type RendererSend = (channel: string, ...args: unknown[]) => void + +function createSender( + options: { + send?: RendererSend + windowDestroyed?: boolean + webContentsDestroyed?: boolean + } = {} +) { + const send = options.send ?? vi.fn() + const onFailure = vi.fn() + const warn = vi.fn() + const sender = createRuntimeRendererNotificationSender({ + isWindowDestroyed: () => options.windowDestroyed ?? false, + webContents: { + isDestroyed: () => options.webContentsDestroyed ?? false, + send + }, + onFailure, + warn + }) + return { sender, send, onFailure, warn } +} + +describe('runtime renderer notification sender', () => { + it('contains a disposed frame and suppresses repeated sends and warnings', () => { + const failure = new Error('Render frame was disposed before WebFrameMain could be accessed') + const fixture = createSender({ + send: vi.fn(() => { + throw failure + }) + }) + + expect(fixture.sender.send('repos:changed')).toBe(false) + expect(fixture.sender.send('worktrees:changed')).toBe(false) + expect(fixture.sender.send('repos:changed')).toBe(false) + + expect(fixture.send).toHaveBeenCalledOnce() + expect(fixture.warn).toHaveBeenCalledOnce() + expect(fixture.onFailure).toHaveBeenCalledExactlyOnceWith('renderer-frame-unavailable') + }) + + it('pauses during a main-frame reload and resumes only after load finishes', () => { + const fixture = createSender() + + fixture.sender.onMainFrameReloadStarted() + expect(fixture.sender.send('repos:changed')).toBe(false) + fixture.sender.onMainFrameLoadFinished() + + expect(fixture.sender.send('repos:changed')).toBe(true) + expect(fixture.send).toHaveBeenCalledOnce() + expect(fixture.onFailure).not.toHaveBeenCalled() + }) + + it('resumes when a provisional main-frame reload is cancelled', () => { + const fixture = createSender() + + fixture.sender.onMainFrameReloadStarted() + expect(fixture.sender.send('repos:changed')).toBe(false) + fixture.sender.onMainFrameReloadCancelled() + + expect(fixture.sender.send('repos:changed')).toBe(true) + expect(fixture.send).toHaveBeenCalledOnce() + }) + + it('treats an absent or destroyed renderer as unavailable without throwing', () => { + const missingWindow = createSender({ windowDestroyed: true }) + const missingWebContents = createSender({ webContentsDestroyed: true }) + + expect(missingWindow.sender.send('repos:changed')).toBe(false) + expect(missingWebContents.sender.send('repos:changed')).toBe(false) + expect(missingWindow.onFailure).not.toHaveBeenCalled() + expect(missingWebContents.onFailure).not.toHaveBeenCalled() + }) + + it('reports renderer process loss once per load generation', () => { + const fixture = createSender() + + fixture.sender.onRendererProcessGone() + fixture.sender.onRendererProcessGone() + fixture.sender.onMainFrameReloadStarted() + fixture.sender.onMainFrameLoadFinished() + fixture.sender.onRendererProcessGone() + fixture.sender.onRendererProcessGone() + + expect(fixture.warn).toHaveBeenCalledTimes(2) + expect(fixture.onFailure).toHaveBeenCalledTimes(2) + expect(fixture.onFailure).toHaveBeenNthCalledWith(1, 'renderer-process-gone') + expect(fixture.onFailure).toHaveBeenNthCalledWith(2, 'renderer-process-gone') + }) + + it('keeps close terminal when renderer lifecycle events arrive late', () => { + const fixture = createSender() + + fixture.sender.close() + fixture.sender.onMainFrameLoadFinished() + fixture.sender.onMainFrameReloadStarted() + fixture.sender.onRendererProcessGone() + + expect(fixture.sender.send('repos:changed')).toBe(false) + expect(fixture.send).not.toHaveBeenCalled() + expect(fixture.warn).not.toHaveBeenCalled() + expect(fixture.onFailure).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/window/runtime-renderer-notification-sender.ts b/src/main/window/runtime-renderer-notification-sender.ts new file mode 100644 index 00000000000..e761fa5bf1d --- /dev/null +++ b/src/main/window/runtime-renderer-notification-sender.ts @@ -0,0 +1,79 @@ +export type RuntimeRendererGraphFailureReason = + | 'renderer-frame-unavailable' + | 'renderer-process-gone' + +export function createRuntimeRendererNotificationSender(args: { + isWindowDestroyed: () => boolean + webContents: { + isDestroyed: () => boolean + send: (channel: string, ...args: unknown[]) => void + } + onFailure: (reason: RuntimeRendererGraphFailureReason) => void + warn?: (message: string) => void +}): { + send: (channel: string, ...values: unknown[]) => boolean + onMainFrameReloadStarted: () => void + onMainFrameReloadCancelled: () => void + onMainFrameLoadFinished: () => void + onRendererProcessGone: () => void + close: () => void +} { + let available = true + let warningEmitted = false + let closed = false + const warn = args.warn ?? ((message: string) => console.warn(message)) + const suspend = (reason: RuntimeRendererGraphFailureReason): void => { + if (closed || (!available && warningEmitted)) { + return + } + available = false + if (!warningEmitted) { + warningEmitted = true + warn(`[runtime-graph] Renderer notifications suspended: ${reason}`) + } + args.onFailure(reason) + } + + return { + send: (channel, ...values) => { + if (closed || args.isWindowDestroyed() || args.webContents.isDestroyed() || !available) { + return false + } + try { + args.webContents.send(channel, ...values) + return true + } catch { + // Why: renderer notification is a side effect; a disposed frame must not + // fail the persistence or runtime operation that produced the event. + suspend('renderer-frame-unavailable') + return false + } + }, + onMainFrameReloadStarted: () => { + if (closed) { + return + } + available = false + warningEmitted = false + }, + onMainFrameReloadCancelled: () => { + if (closed) { + return + } + available = true + warningEmitted = false + }, + onMainFrameLoadFinished: () => { + if (closed) { + return + } + available = true + warningEmitted = false + }, + onRendererProcessGone: () => suspend('renderer-process-gone'), + close: () => { + closed = true + available = false + } + } +} diff --git a/src/preload/api/runtime-api.ts b/src/preload/api/runtime-api.ts index 1acc43f5f2c..b796949b7f6 100644 --- a/src/preload/api/runtime-api.ts +++ b/src/preload/api/runtime-api.ts @@ -1,7 +1,7 @@ import type { RuntimeBrowserDriverState, + RuntimeRendererSyncWindowGraph, RuntimeStatus, - RuntimeSyncWindowGraph, RuntimeSyncWindowGraphResult, RuntimeTerminalDriverState } from '../../shared/runtime-types' @@ -16,7 +16,9 @@ export type RuntimeEnvironmentSubscriptionHandle = { export type RuntimeApi = { runtime: { - syncWindowGraph: (graph: RuntimeSyncWindowGraph) => Promise + syncWindowGraph: ( + graph: RuntimeRendererSyncWindowGraph + ) => Promise getStatus: () => Promise call: (args: { method: string; params?: unknown }) => Promise> getTerminalFitOverrides: () => Promise< diff --git a/src/preload/index.ts b/src/preload/index.ts index 46c5d5362d8..c5bb2959816 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -138,9 +138,9 @@ import type { import type { RuntimeBrowserDriverState, RuntimeMobileSessionTabMove, + RuntimeRendererSyncWindowGraph, RuntimeStatus, RuntimeSyncWindowGraphResult, - RuntimeSyncWindowGraph, RuntimeTerminalCreateRequestPayload, RuntimeTerminalDriverState, RuntimeTerminalPresentation @@ -4324,7 +4324,9 @@ const api = { }, runtime: { - syncWindowGraph: (graph: RuntimeSyncWindowGraph): Promise => + syncWindowGraph: ( + graph: RuntimeRendererSyncWindowGraph + ): Promise => ipcRenderer.invoke('runtime:syncWindowGraph', graph), getStatus: (): Promise => ipcRenderer.invoke('runtime:getStatus'), call: (args: { method: string; params?: unknown }): Promise> => diff --git a/src/renderer/src/runtime/sync-runtime-graph.ts b/src/renderer/src/runtime/sync-runtime-graph.ts index 2422a31b3c7..3b5f1f62066 100644 --- a/src/renderer/src/runtime/sync-runtime-graph.ts +++ b/src/renderer/src/runtime/sync-runtime-graph.ts @@ -20,7 +20,7 @@ import type { RuntimeMobileSessionSnapshotTab, RuntimeMobileTerminalTheme, RuntimeMobileSessionTabsSnapshot, - RuntimeSyncWindowGraph + RuntimeRendererSyncWindowGraph } from '../../../shared/runtime-types' import { isTerminalLeafId, makePaneKey, parsePaneKey } from '../../../shared/stable-pane-id' import { isWebTerminalSurfaceTabId } from '../../../shared/terminal-surface-id' @@ -728,9 +728,10 @@ async function syncRuntimeGraph(): Promise { const generatedTitlesEnabled = state.settings?.tabAutoGenerateTitle === true const mobileSessionTabs = buildMobileSessionTabSnapshots(state, systemPrefersDark) const publication = partitionMobileSessionPublication(mobileSessionTabs) - const graph: RuntimeSyncWindowGraph = { + const graph: RuntimeRendererSyncWindowGraph = { tabs: [], leaves: [], + rendererGeneration: mobileSessionPublicationEpoch, mobileSessionTabs: publication.changed, unchangedMobileSessionWorktrees: publication.unchangedWorktrees } diff --git a/src/shared/runtime-types.ts b/src/shared/runtime-types.ts index aa5ed57d939..d6b257ca85f 100644 --- a/src/shared/runtime-types.ts +++ b/src/shared/runtime-types.ts @@ -150,6 +150,11 @@ export type RuntimeSyncWindowGraph = { unchangedMobileSessionWorktrees?: string[] } +export type RuntimeRendererSyncWindowGraph = RuntimeSyncWindowGraph & { + /** Unique to one renderer document; a reload must publish from a new generation. */ + rendererGeneration: string +} + export type RuntimeNativeChatLaunchDraftResolution = { tabId: string text: string diff --git a/tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts b/tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts new file mode 100644 index 00000000000..d7238e3dc4c --- /dev/null +++ b/tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts @@ -0,0 +1,106 @@ +import { test, expect } from './helpers/orca-app' +import { RuntimeClient } from '../../src/cli/runtime-client' +import type { RuntimeStatus } from '../../src/shared/runtime-types' +import { waitForSessionReady } from './helpers/store' + +type NavigationProbe = { + startedLoading: number + finishedLoading: number + stoppedLoading: number +} + +declare global { + var __blockedNavigationProbe: NavigationProbe | undefined +} + +test('blocked navigation preserves the renderer document and graph authority', async ({ + electronApp, + orcaPage +}) => { + await waitForSessionReady(orcaPage) + const userDataDir = await electronApp.evaluate(({ app }) => app.getPath('userData')) + const client = new RuntimeClient(userDataDir, 30_000, null, null) + const before = (await client.call('status.get')).result + + await electronApp.evaluate(({ BrowserWindow, shell }) => { + shell.openExternal = async () => undefined + const probe = { startedLoading: 0, finishedLoading: 0, stoppedLoading: 0 } + globalThis.__blockedNavigationProbe = probe + const contents = BrowserWindow.getAllWindows()[0]!.webContents + contents.on('did-start-loading', () => probe.startedLoading++) + contents.on('did-finish-load', () => probe.finishedLoading++) + contents.on('did-stop-loading', () => probe.stoppedLoading++) + }) + await orcaPage.evaluate(() => { + ;(window as unknown as { __blockedNavigationCanary: string }).__blockedNavigationCanary = + 'alive' + const anchor = document.createElement('a') + anchor.href = 'https://example.invalid/blocked' + document.body.append(anchor) + anchor.click() + }) + + await expect + .poll(() => electronApp.evaluate(() => globalThis.__blockedNavigationProbe)) + .toMatchObject({ startedLoading: 1, finishedLoading: 0, stoppedLoading: 1 }) + expect( + await orcaPage.evaluate( + () => (window as unknown as { __blockedNavigationCanary?: string }).__blockedNavigationCanary + ) + ).toBe('alive') + await expect + .poll(async () => { + const status = (await client.call('status.get')).result + return { + graphStatus: status.graphStatus, + rendererGraphEpoch: status.rendererGraphEpoch, + authoritativeWindowId: status.authoritativeWindowId + } + }) + .toEqual({ + graphStatus: 'ready', + rendererGraphEpoch: before.rendererGraphEpoch, + authoritativeWindowId: before.authoritativeWindowId + }) +}) + +test('cancelled renderer reload restores the surviving graph authority', async ({ + electronApp, + orcaPage +}) => { + await waitForSessionReady(orcaPage) + const userDataDir = await electronApp.evaluate(({ app }) => app.getPath('userData')) + const client = new RuntimeClient(userDataDir, 30_000, null, null) + const before = (await client.call('status.get')).result + await orcaPage.evaluate(() => { + ;(window as unknown as { __cancelledReloadCanary: string }).__cancelledReloadCanary = 'alive' + }) + + await electronApp.evaluate(({ BrowserWindow }) => { + const contents = BrowserWindow.getAllWindows()[0]!.webContents + const url = new URL(contents.getURL()) + url.searchParams.set('cancelled-reload', '1') + contents.once('will-navigate', (event) => event.preventDefault()) + void contents.executeJavaScript(`window.location.assign(${JSON.stringify(url.href)})`) + }) + + await expect + .poll(async () => { + const status = (await client.call('status.get')).result + return { + graphStatus: status.graphStatus, + rendererGraphEpoch: status.rendererGraphEpoch, + authoritativeWindowId: status.authoritativeWindowId + } + }) + .toEqual({ + graphStatus: 'ready', + rendererGraphEpoch: before.rendererGraphEpoch + 1, + authoritativeWindowId: before.authoritativeWindowId + }) + expect( + await orcaPage.evaluate( + () => (window as unknown as { __cancelledReloadCanary?: string }).__cancelledReloadCanary + ) + ).toBe('alive') +})