From 3ef777b0b5c68189b22bf249770ff203ef65911c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:55:58 -0700 Subject: [PATCH] fix(runtime): resume notifications after graph recovery --- config/reliability-gates.jsonc | 36 +++++++++++++++---- src/main/runtime/orca-runtime.ts | 4 +++ ...runtime-graph-publication-recovery.test.ts | 4 +++ .../window/attach-main-window-services.ts | 5 +++ ...ntime-renderer-notification-sender.test.ts | 16 +++++++++ .../runtime-renderer-notification-sender.ts | 25 ++++++------- 6 files changed, 69 insertions(+), 21 deletions(-) diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 0826c5dea46..5fa23e0f6fe 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -2310,16 +2310,18 @@ "providers": ["local", "daemon", "ssh", "paired-runtime"], "coveredPlatforms": ["macos"], "coveredProviders": ["local", "daemon", "paired-runtime"], - "coverageNotes": "Deterministic lifecycle, IPC, window-notification, and runtime ownership tests cover generation replacement, stale callbacks, same-frame old-document publication, blocked external navigation, cancelled renderer navigation, concurrent provisional starts, process loss, timeout cleanup, headless fallback, and competing windows. Isolated macOS Electron validation covers cancelled navigation, forced renderer recovery, and headless serve promotion with live daemon-backed PTY identity and I/O. Linux, Windows, WSL, SSH promotion, and mixed-version desktop/server live runs remain uncollected; the renderer generation stays local to Electron IPC and does not change the remote wire.", + "coverageNotes": "Deterministic lifecycle, IPC, window-notification, and runtime ownership tests cover generation replacement, stale callbacks, same-frame old-document publication, blocked external navigation, cancelled renderer navigation, concurrent provisional starts, current-renderer recovery after a notification-frame failure, notification resumption after accepted publication, process loss, timeout cleanup, retired windows, headless fallback, and competing windows. Isolated macOS Electron validation covers cancelled navigation, forced renderer recovery, and headless serve promotion with live daemon-backed PTY identity and I/O. Linux, Windows, WSL, SSH promotion, and mixed-version desktop/server live runs remain uncollected; the renderer generation stays local to Electron IPC and does not change the remote wire.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/14066", "https://linear.app/stably/issue/STA-4016", - "https://github.com/stablyai/orca/pull/14070" + "https://github.com/stablyai/orca/pull/14070", + "https://github.com/stablyai/orca/issues/16521", + "https://linear.app/stably/issue/STA-5523" ], - "invariant": "Every genuine renderer graph reload revision settles exactly once, while a blocked external navigation that preserves the current document never starts a revision. A cancelled renderer-document navigation restores authority only when that same renderer survived; cancellation after an earlier document committed remains fenced until publication or timeout. Replacing a reload cancels only the prior revision; its timer and callbacks cannot settle a newer revision. A desktop reload succeeds only from a publication carrying a new renderer-document generation, while a failed headless promotion may recover only from its originating window. Timeout, renderer loss, and close reach a terminal graph state, and successful desktop publication permanently retires headless sentinel fallback until a new headless runtime is established.", - "oracle": "Start a blocked external main-frame navigation and require the current renderer graph and notification path to remain authoritative. Start a same-renderer navigation, cancel it before commit, and require the renderer canary and ready authority to survive; coalesce overlapping provisional starts into one fence, and keep an earlier committed reload fenced if a later navigation is cancelled. Drive two genuine reloads half a timeout apart and require the first deadline to leave the second revision reloading while the second deadline alone makes it unavailable. During the second revision publish from the prior renderer document through the still-current main frame and require rejection without graph mutation or timeout cancellation. Restore headless authority after failed promotion, publish from a competing window, and require rejection while the sentinel remains authoritative. Then force a real renderer crash and separately promote an isolated headless serve owner; require runtime graph readiness, stable runtime/main/daemon/PTY identities, recovered visible terminal output, and post-recovery input/output.", + "invariant": "Every genuine renderer graph reload revision settles exactly once, while a blocked external navigation that preserves the current document never starts a revision. A cancelled renderer-document navigation restores authority only when that same renderer survived; cancellation after an earlier document committed remains fenced until publication or timeout. Replacing a reload cancels only the prior revision; its timer and callbacks cannot settle a newer revision. A desktop reload succeeds only from a publication carrying a new renderer-document generation, while a failed headless promotion may recover only from its originating window. A live current renderer whose notification frame failed may restore an unavailable graph with its last accepted generation, and that accepted publication restores main-to-renderer notifications; timeout, renderer loss, and retired-window publications remain fenced. Timeout, renderer loss, and close reach a terminal graph state, and successful desktop publication permanently retires headless sentinel fallback until a new headless runtime is established.", + "oracle": "Start a blocked external main-frame navigation and require the current renderer graph and notification path to remain authoritative. Start a same-renderer navigation, cancel it before commit, and require the renderer canary and ready authority to survive; coalesce overlapping provisional starts into one fence, and keep an earlier committed reload fenced if a later navigation is cancelled. Drive two genuine reloads half a timeout apart and require the first deadline to leave the second revision reloading while the second deadline alone makes it unavailable. During the second revision publish from the prior renderer document through the still-current main frame and require rejection without graph mutation or timeout cancellation. Fail one notification send while the current renderer remains live, require its same-generation retry to restore graph readiness and notification delivery, then reject the same generation after timeout, renderer loss, or window retirement. Restore headless authority after failed promotion, publish from a competing window, and require rejection while the sentinel remains authoritative. Then force a real renderer crash and separately promote an isolated headless serve owner; require runtime graph readiness, stable runtime/main/daemon/PTY identities, recovered visible terminal output, and post-recovery input/output.", "commands": [ - "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/runtime-graph-reload-lifecycle.test.ts src/main/window/runtime-renderer-notification-sender.test.ts src/main/window/renderer-document-navigation.test.ts src/main/ipc/runtime.test.ts src/main/window/attach-main-window-services.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/runtime/sync-runtime-graph-payload-partition.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/runtime-graph-reload-lifecycle.test.ts src/main/runtime/runtime-graph-publication-recovery.test.ts src/main/window/runtime-renderer-notification-sender.test.ts src/main/window/renderer-document-navigation.test.ts src/main/ipc/runtime.test.ts src/main/window/attach-main-window-services.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/runtime/sync-runtime-graph-payload-partition.test.ts", "pnpm exec electron-vite build --mode e2e", "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/renderer-blocked-navigation-graph-authority.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "SKIP_BUILD=1 pnpm exec playwright test e2e/renderer-crash-recovery-terminal-input.spec.ts tests/e2e/renderer-crash-recovery-terminal-input.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", @@ -2327,6 +2329,7 @@ ], "testFiles": [ "src/main/runtime/runtime-graph-reload-lifecycle.test.ts", + "src/main/runtime/runtime-graph-publication-recovery.test.ts", "src/main/window/runtime-renderer-notification-sender.test.ts", "src/main/window/renderer-document-navigation.test.ts", "src/main/ipc/runtime.test.ts", @@ -2358,6 +2361,14 @@ "successful desktop publication retires headless fallback" ] }, + { + "file": "src/main/runtime/runtime-graph-publication-recovery.test.ts", + "assertions": [ + "a live current renderer heals a frame-unavailable graph with its accepted generation", + "an accepted renderer graph publication restores main-to-renderer notifications", + "timeout, process loss, and retired-window publications remain fenced" + ] + }, { "file": "src/main/ipc/runtime.test.ts", "assertions": [ @@ -2369,6 +2380,7 @@ "file": "src/main/window/runtime-renderer-notification-sender.test.ts", "assertions": [ "renderer failure is contained once per load generation", + "accepted current-renderer graph publication resumes notification delivery", "late events cannot revive a closed sender" ] }, @@ -2404,6 +2416,15 @@ } ], "evidenceRuns": [ + { + "date": "2026-08-27", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/runtime-graph-reload-lifecycle.test.ts src/main/runtime/runtime-graph-publication-recovery.test.ts src/main/window/runtime-renderer-notification-sender.test.ts src/main/window/renderer-document-navigation.test.ts src/main/ipc/runtime.test.ts src/main/window/attach-main-window-services.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/runtime/sync-runtime-graph-payload-partition.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 13, + "summary": "Nine focused files passed 1,284 tests with one pre-existing skip, including same-generation frame-failure recovery, timeout/process/retired-window fencing, notification resumption, bounded renderer retry, reload ownership, IPC authority, and payload acknowledgement." + }, { "date": "2026-08-14", "runner": "local", @@ -2451,11 +2472,11 @@ }, "redGreenEvidence": { "status": "complete", - "evidence": "A byte-identical three-case oracle was rerun on 2026-08-13. Latest main ee8dd4796e failed all three: reload remained reloading after 22.5 seconds, the prior renderer document settled the replacement, and a competing window stole failed-promotion authority. Submitted PR head feeca6c22f passed the bounded replacement timeout but failed both authority cases. The locally rewritten candidate passed all three. Disabling the local document-generation and pending-window fences reproduces the submitted-head failures; disabling the cancelled-navigation classifier leaves the graph reloading and increments its epoch in the new live oracle." + "evidence": "A byte-identical three-case oracle was rerun on 2026-08-13. Latest main ee8dd4796e failed all three: reload remained reloading after 22.5 seconds, the prior renderer document settled the replacement, and a competing window stole failed-promotion authority. Submitted PR head feeca6c22f passed the bounded replacement timeout but failed both authority cases. The locally rewritten candidate passed all three. Disabling the local document-generation and pending-window fences reproduces the submitted-head failures; disabling the cancelled-navigation classifier leaves the graph reloading and increments its epoch in the new live oracle. On 2026-08-27, the PR head restored graph readiness after notification-frame failure but failed two focused oracles because accepted publication did not signal notification recovery and the sender had no recovery transition; the final notifier acceptance path passes both tests." }, "performanceBudget": { "required": true, - "evidence": "Fresh delegated audit passed with no blocking, major, or actionable minor findings. The candidate retains at most one unref'd timer, reuses the existing renderer document UUID, adds about 67 serialized bytes to each already-coalesced local graph IPC, and performs constant-time generation/window checks. It adds no polling, subprocess, network request, remote frame, provider scan, or steady-state fanout; a dedicated benchmark would measure existing graph/structured-clone noise rather than meaningful incremental work." + "evidence": "The renderer retains at most one publication-retry timer after a rejected graph sync, backs off from 500 ms to a 30-second cap, and clears the timer on success or disable. The existing reload lifecycle retains its one bounded timer; publication acceptance adds only constant-time generation/window checks and one local notifier-state reset. The change adds no subprocess, network request, remote frame, provider scan, or steady-state fanout." }, "promotionCriteria": [ "Collect CI and soak history without unexplained flakes.", @@ -2465,6 +2486,7 @@ "knownGaps": [ "The live journeys cover macOS and daemon-backed local terminals, not Linux desktop, Windows, WSL, or SSH promotion.", "The same-frame stale-document race is deterministic fault injection because Playwright cannot schedule an invoke precisely between did-start-loading and frame replacement.", + "The paired-mobile empty-inventory failure state requires main-process fault injection that the mobile client and emulator cannot trigger; recovery is deterministic contract coverage rather than a live paired-device journey.", "No mixed-version live run is required because rendererGeneration is confined to same-version preload/main Electron IPC and the remote runtime wire is unchanged." ], "demotionRule": "Demote if a reload revision settles more than once, a stale document or competing window can publish, a timer survives settlement, desktop failure stays indefinitely reloading, headless fallback returns after successful promotion, or either isolated Electron journey loses runtime or PTY identity." diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index e29941a9120..3e207e4e5a3 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -2253,6 +2253,7 @@ function resolveTerminalPresentation(opts: { } type RuntimeNotifier = { + rendererGraphPublished?(windowId: number): void worktreesChanged(repoId: string, renamed?: { oldWorktreeId: string; newWorktreeId: string }): void worktreeBaseStatus?(event: WorktreeBaseStatusEvent): void worktreeRemoteBranchConflict?(event: WorktreeRemoteBranchConflictEvent): void @@ -7340,6 +7341,9 @@ export class OrcaRuntimeService { if (rendererGeneration !== undefined) { this.rendererGeneration = rendererGeneration } + if (windowId !== HEADLESS_RUNTIME_WINDOW_ID) { + this.notifier?.rendererGraphPublished?.(windowId) + } for (const leaf of this.leaves.values()) { this.adoptPreAllocatedHandle(leaf) const previousLeaf = previousLeaves.get(this.getLeafKey(leaf.tabId, leaf.leafId)) diff --git a/src/main/runtime/runtime-graph-publication-recovery.test.ts b/src/main/runtime/runtime-graph-publication-recovery.test.ts index fd3568bdee6..f21bde56d06 100644 --- a/src/main/runtime/runtime-graph-publication-recovery.test.ts +++ b/src/main/runtime/runtime-graph-publication-recovery.test.ts @@ -49,8 +49,11 @@ describe('runtime graph publication recovery', () => { it('lets the still-live renderer republish after a terminal reload failure', () => { const runtime = createRuntime() + const rendererGraphPublished = vi.fn() + runtime.setNotifier({ rendererGraphPublished } as never) runtime.attachWindow(WINDOW_ID) runtime.syncWindowGraph(WINDOW_ID, desktopGraph('renderer-a')) + rendererGraphPublished.mockClear() expect(runtime.getStatus()).toMatchObject({ graphStatus: 'ready' }) // A renderer notification send failure suspends the graph while the same @@ -68,6 +71,7 @@ describe('runtime graph publication recovery', () => { graphStatus: 'ready', liveTabCount: 1 }) + expect(rendererGraphPublished).toHaveBeenCalledExactlyOnceWith(WINDOW_ID) }) it('still rejects the pre-reload document while its replacement is loading', () => { diff --git a/src/main/window/attach-main-window-services.ts b/src/main/window/attach-main-window-services.ts index ce62a6d3da4..655cc7c3c20 100644 --- a/src/main/window/attach-main-window-services.ts +++ b/src/main/window/attach-main-window-services.ts @@ -336,6 +336,11 @@ function registerRuntimeWindowLifecycle( }) const send = rendererNotifications.send runtime.setNotifier({ + rendererGraphPublished: (windowId) => { + if (windowId === mainWindow.id) { + rendererNotifications.onRendererGraphPublished() + } + }, worktreesChanged: (repoId, renamed) => { // Why: clear scan caches before the renderer handles this event, so it can't read stale TTL entries after a mutation. runWorktreeChangeInvalidators(repoId) diff --git a/src/main/window/runtime-renderer-notification-sender.test.ts b/src/main/window/runtime-renderer-notification-sender.test.ts index d482bb404fb..c01fb559954 100644 --- a/src/main/window/runtime-renderer-notification-sender.test.ts +++ b/src/main/window/runtime-renderer-notification-sender.test.ts @@ -43,6 +43,22 @@ describe('runtime renderer notification sender', () => { expect(fixture.onFailure).toHaveBeenCalledExactlyOnceWith('renderer-frame-unavailable') }) + it('resumes after the current renderer publishes an accepted graph', () => { + const send = vi + .fn() + .mockImplementationOnce(() => { + throw new Error('Render frame was disposed') + }) + .mockImplementation(() => {}) + const fixture = createSender({ send }) + + expect(fixture.sender.send('repos:changed')).toBe(false) + fixture.sender.onRendererGraphPublished() + + expect(fixture.sender.send('repos:changed')).toBe(true) + expect(send).toHaveBeenCalledTimes(2) + }) + it('pauses during a main-frame reload and resumes only after load finishes', () => { const fixture = createSender() diff --git a/src/main/window/runtime-renderer-notification-sender.ts b/src/main/window/runtime-renderer-notification-sender.ts index e761fa5bf1d..a4e024b02af 100644 --- a/src/main/window/runtime-renderer-notification-sender.ts +++ b/src/main/window/runtime-renderer-notification-sender.ts @@ -15,6 +15,7 @@ export function createRuntimeRendererNotificationSender(args: { onMainFrameReloadStarted: () => void onMainFrameReloadCancelled: () => void onMainFrameLoadFinished: () => void + onRendererGraphPublished: () => void onRendererProcessGone: () => void close: () => void } { @@ -33,6 +34,13 @@ export function createRuntimeRendererNotificationSender(args: { } args.onFailure(reason) } + const resume = (): void => { + if (closed) { + return + } + available = true + warningEmitted = false + } return { send: (channel, ...values) => { @@ -56,20 +64,9 @@ export function createRuntimeRendererNotificationSender(args: { available = false warningEmitted = false }, - onMainFrameReloadCancelled: () => { - if (closed) { - return - } - available = true - warningEmitted = false - }, - onMainFrameLoadFinished: () => { - if (closed) { - return - } - available = true - warningEmitted = false - }, + onMainFrameReloadCancelled: resume, + onMainFrameLoadFinished: resume, + onRendererGraphPublished: resume, onRendererProcessGone: () => suspend('renderer-process-gone'), close: () => { closed = true