From 08dedd2cdbf64a2321acc8bb428dadf7ef88e030 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:32:23 -0800 Subject: [PATCH] fix(usage): retry failed backend startup on demand --- .../backend-desktop-capabilities.ts | 65 +++++++++ .../backend-supervisor.test.ts | 128 +++++++++++++++++ .../backend-desktop/backend-supervisor.ts | 134 ++++++++---------- 3 files changed, 256 insertions(+), 71 deletions(-) diff --git a/src/main/backend-desktop/backend-desktop-capabilities.ts b/src/main/backend-desktop/backend-desktop-capabilities.ts index baf515edd8d..0d8bef80826 100644 --- a/src/main/backend-desktop/backend-desktop-capabilities.ts +++ b/src/main/backend-desktop/backend-desktop-capabilities.ts @@ -1,8 +1,30 @@ +import { BrowserWindow, type WebContents, type WebFrameMain } from 'electron' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' import { agentTokenUsageSchema } from '../../shared/telemetry-agent-token-usage-schema' import { claudeProfileTranscriptDirs } from '../claude-usage/transcript-file-discovery' import { loadKnownUsageWorktreesByRepo } from '../usage/worktree-attribution/usage-worktree-metadata' import { isTelemetryEnabled, track } from '../telemetry/client' import type { UsageWorktreeSource } from '../usage/worktree-attribution/usage-worktree-metadata' +import { currentWorkerEntryLayout } from '../worker-threads/worker-thread-entry-path' +import { isRendererDocumentNavigation } from '../window/renderer-document-navigation' + +export function trustsBackendRenderer(contents: WebContents): boolean { + if ( + contents.isDestroyed() || + contents.getType() !== 'window' || + !BrowserWindow.fromWebContents(contents) + ) { + return false + } + const layout = currentWorkerEntryLayout(__dirname) + const roots = process.env.ELECTRON_RENDERER_URL + ? [process.env.ELECTRON_RENDERER_URL] + : ['index.html', 'space.html', 'popout.html'].map( + (name) => pathToFileURL(join(layout.moduleDir, '../renderer', name)).href + ) + return roots.some((root) => isRendererDocumentNavigation(root, contents.getURL())) +} export function createBackendDesktopCapabilities( store: UsageWorktreeSource @@ -28,3 +50,46 @@ export function createBackendDesktopCapabilities( throw new Error('This desktop capability is not supported.') } } + +export type BackendDocument = { + contents: WebContents + frame: WebFrameMain + token: string + retire(): void +} + +export function trackBackendDocument( + documents: Map, + contents: WebContents, + frame: WebFrameMain, + token: string, + retireToken: (token: string) => void +): BackendDocument { + documents.get(contents.id)?.retire() + const document: BackendDocument = { contents, frame, token, retire: () => {} } + const retire = (): void => { + if (documents.get(contents.id) !== document) { + return + } + documents.delete(contents.id) + retireToken(document.token) + contents.off('did-frame-navigate', navigation) + contents.off('destroyed', retire) + } + const navigation = ( + _event: unknown, + _url: string, + _httpCode: number, + _httpStatus: string, + mainFrame: boolean + ): void => { + if (mainFrame) { + retire() + } + } + document.retire = retire + documents.set(contents.id, document) + contents.on('did-frame-navigate', navigation) + contents.once('destroyed', retire) + return document +} diff --git a/src/main/backend-desktop/backend-supervisor.test.ts b/src/main/backend-desktop/backend-supervisor.test.ts index 0d3cde062ba..88f4c706171 100644 --- a/src/main/backend-desktop/backend-supervisor.test.ts +++ b/src/main/backend-desktop/backend-supervisor.test.ts @@ -232,3 +232,131 @@ it('retires only the exited renderer document and rejects its pending port deliv child.emit('exit', 0) await stopped }) + +function startupFixture() { + const children: (EventEmitter & { + postMessage: ReturnType + kill: ReturnType + })[] = [] + rig.fork.mockImplementation(() => { + const child = Object.assign(new EventEmitter(), { + postMessage: vi.fn(), + kill: vi.fn(() => false) + }) + children.push(child) + return child + }) + const supervisor = new BackendSupervisor({ + getRepos: () => [], + getAllWorktreeMeta: () => ({}), + getRememberedWorktreeDisplayNames: () => new Map(), + whenLoaded: async () => {} + }) + const frame = { postMessage: vi.fn() } + const sender = Object.assign(new EventEmitter(), { + id: 4, + mainFrame: frame, + isDestroyed: () => false, + getType: () => 'window', + getURL: () => 'file:///index.html' + }) + const request = rig.handle.mock.calls.find(([channel]) => channel === BACKEND_PORT_REQUEST)?.[1] + const token = '44444444-4444-4444-4444-444444444444' + const read = () => + supervisor.automationClient('claudeUsage').getAutomationRunUsage({ + worktreeId: null, + terminalSessionId: null, + startedAt: null, + completedAt: null + }) + return { + children, + supervisor, + frame, + read, + request: (purpose = 'demand') => request({ sender, senderFrame: frame }, token, purpose) + } +} + +it('retries a rejected signal-free launch on new demand without replay or false retirement', async () => { + vi.useFakeTimers() + const { children, supervisor, frame, read, request } = startupFixture() + const ready = { type: 'ready', protocol: 1, capabilities: ['usage'] } + for (let attempt = 0; attempt < 4; attempt++) { + const demand = expect(read()).rejects.toThrow('Usage is still unavailable') + const joined = expect(request('recovery')).rejects.toThrow('Usage is still unavailable') + await vi.advanceTimersByTimeAsync(30_000) + await demand + await joined + children[attempt].emit('message', ready) + expect(frame.postMessage).not.toHaveBeenCalledWith(BACKEND_CONNECTION_CHANGED, { + connected: true, + unavailable: false + }) + await expect(request('recovery')).rejects.toThrow('Usage is still unavailable') + expect(children).toHaveLength(attempt + 1) + expect(rig.call).not.toHaveBeenCalled() + } + const retry = read() + const port = request() + children[4].emit('message', ready) + rig.call.mockResolvedValueOnce({ inputTokens: 42 }) + await expect(retry).resolves.toEqual({ inputTokens: 42 }) + await port + expect(rig.call).toHaveBeenCalledTimes(1) + const notifications = frame.postMessage.mock.calls.slice() + for (const child of children.slice(0, 4)) { + child.emit('spawn') + expect(child.kill).toHaveBeenCalled() + child.emit('message', ready) + child.emit('exit', 1) + } + expect(frame.postMessage.mock.calls).toEqual(notifications) + const { recordDurableCrashBreadcrumb } = + await import('../crash-reporting/breadcrumbs/durable-crash-breadcrumb') + expect(recordDurableCrashBreadcrumb).toHaveBeenLastCalledWith('backend_process_exited', { + pid: null, + incarnation: null, + exitCode: 1, + expected: true + }) + // A rejection after successful readiness cannot turn that owner into a failed startup. + children[4].emit('error', new Error('late error')) + rig.call.mockResolvedValueOnce({ inputTokens: 43 }) + await expect(read()).resolves.toEqual({ inputTokens: 43 }) + expect(children).toHaveLength(5) + const stopped = supervisor.stop() + let quit = false + void stopped.then(() => { + quit = true + }) + await Promise.resolve() + expect(quit).toBe(false) + children[4].emit('exit', 0) + await stopped + await expect(read()).rejects.toThrow('Orca is shutting down') + expect(children).toHaveLength(5) +}) + +it('rejects old requests when ownership changes across readiness instead of rerouting them', async () => { + const { children, supervisor, frame, read, request } = startupFixture() + const ready = { type: 'ready', protocol: 1, capabilities: ['usage'] } + children[0].emit('message', ready) + const oldRead = expect(read()).rejects.toThrow('Usage disconnected') + const oldPort = expect(request()).rejects.toThrow('The document was replaced') + children[0].emit('exit', 1) + const replacement = request() + children[1].emit('message', ready) + await oldRead + await oldPort + await replacement + expect(rig.call).not.toHaveBeenCalled() + expect( + frame.postMessage.mock.calls.filter(([channel]) => channel === BACKEND_PORT_DELIVERY) + ).toHaveLength(1) + const stopped = supervisor.stop() + children[1].emit('spawn') + expect(children[1].kill).toHaveBeenCalled() + children[1].emit('exit', 0) + await stopped +}) diff --git a/src/main/backend-desktop/backend-supervisor.ts b/src/main/backend-desktop/backend-supervisor.ts index a677fd63f55..4078fa1c92c 100644 --- a/src/main/backend-desktop/backend-supervisor.ts +++ b/src/main/backend-desktop/backend-supervisor.ts @@ -1,15 +1,5 @@ -import { - app, - BrowserWindow, - ipcMain, - MessageChannelMain, - utilityProcess, - type WebContents, - type WebFrameMain -} from 'electron' -import { join } from 'node:path' +import { app, ipcMain, MessageChannelMain, utilityProcess, type WebContents } from 'electron' import { recordDurableCrashBreadcrumb } from '../crash-reporting/breadcrumbs/durable-crash-breadcrumb' -import { pathToFileURL } from 'node:url' import { BackendPortRpc } from '../../shared/backend-port-rpc' import { BACKEND_CONNECTION_CHANGED, @@ -25,24 +15,27 @@ import { } from '../worker-threads/worker-thread-entry-path' import { ManagedOrcadEventWindow } from '../managed-server/lifecycle/orcad-managed-event-window' import { ORCAD_SHUTDOWN_DEADLINE_MS } from '../orcad/orcad-stop-deadlines' -import { isRendererDocumentNavigation } from '../window/renderer-document-navigation' -import { createBackendDesktopCapabilities } from './backend-desktop-capabilities' +import { + createBackendDesktopCapabilities, + trustsBackendRenderer, + trackBackendDocument, + type BackendDocument +} from './backend-desktop-capabilities' import type { UsageWorktreeSource } from '../usage/worktree-attribution/usage-worktree-metadata' export { captureBackendBootstrap } from './backend-bootstrap' import { capturedBootstrap } from './backend-bootstrap' -type Document = { contents: WebContents; frame: WebFrameMain; token: string; retire(): void } - export class BackendSupervisor { private child: ReturnType | null = null private control: BackendPortRpc | null = null private ready: Promise | null = null + private disposeFailedStartup?: () => void private desiredRunning = true private exited: Promise = Promise.resolve() private restartTimer: ReturnType | undefined private readonly failures = new ManagedOrcadEventWindow(10 * 60_000, Date.now) - private readonly documents = new Map() + private readonly documents = new Map() private stopping?: Promise private readonly retireRenderer = (_event: unknown, contents: WebContents): void => @@ -64,57 +57,37 @@ export class BackendSupervisor { throw new Error('Invalid backend connection request.') } const { sender, senderFrame } = event - if (!senderFrame || senderFrame !== sender.mainFrame || !this.trusts(sender)) { + if (!senderFrame || senderFrame !== sender.mainFrame || !trustsBackendRenderer(sender)) { throw new Error('This document cannot connect to Orca.') } - this.documents.get(sender.id)?.retire() - const document: Document = { - contents: sender, - frame: senderFrame, + const document = trackBackendDocument( + this.documents, + sender, + senderFrame, token, - retire: () => {} - } - const retire = (): void => { - if (this.documents.get(sender.id) !== document) { - return - } - this.documents.delete(sender.id) - this.child?.postMessage({ type: 'retire', token: document.token }) - sender.off('did-frame-navigate', navigation) - sender.off('destroyed', retire) - } - const navigation = ( - _event: unknown, - _url: string, - _httpCode: number, - _httpStatus: string, - mainFrame: boolean - ): void => { - if (mainFrame) { - retire() - } - } - document.retire = retire - this.documents.set(sender.id, document) - sender.on('did-frame-navigate', navigation) - sender.once('destroyed', retire) + (retiredToken) => this.child?.postMessage({ type: 'retire', token: retiredToken }) + ) if (!this.desiredRunning) { throw new Error('Orca is shutting down.') } const ready = purpose === 'recovery' ? this.ready : this.ensureReady(true) + const child = this.child if (!ready) { throw new Error('Usage is unavailable. Try refreshing the usage view.') } await ready if ( this.documents.get(sender.id) !== document || + !this.desiredRunning || + !child || + this.child !== child || sender.isDestroyed() || sender.mainFrame !== senderFrame ) { throw new Error('The document was replaced before Orca connected.') } const { port1, port2 } = new MessageChannelMain() - this.child!.postMessage({ type: 'client', token: document.token }, [port1]) + child.postMessage({ type: 'client', token: document.token }, [port1]) senderFrame.postMessage(BACKEND_PORT_DELIVERY, { token: document.token }, [port2]) return document.token } @@ -125,9 +98,14 @@ export class BackendSupervisor { automationClient(provider: 'claudeUsage' | 'codexUsage'): AutomationUsageClient { return { getAutomationRunUsage: async (input) => { - await this.ensureReady(true) + const ready = this.ensureReady(true) + const control = this.control + await ready + if (!this.desiredRunning || !control || this.control !== control) { + throw new Error('Usage disconnected. Try again after it reconnects.') + } // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The private usage service returns its typed automation result for this provider method. - return (await this.control!.call(`${provider}:getAutomationRunUsage`, input)) as Awaited< + return (await control.call(`${provider}:getAutomationRunUsage`, input)) as Awaited< ReturnType > } @@ -161,23 +139,6 @@ export class BackendSupervisor { return this.stopping } - private trusts(contents: WebContents): boolean { - if ( - contents.isDestroyed() || - contents.getType() !== 'window' || - !BrowserWindow.fromWebContents(contents) - ) { - return false - } - const layout = currentWorkerEntryLayout(__dirname) - const roots = process.env.ELECTRON_RENDERER_URL - ? [process.env.ELECTRON_RENDERER_URL] - : ['index.html', 'space.html', 'popout.html'].map( - (name) => pathToFileURL(join(layout.moduleDir, '../renderer', name)).href - ) - return roots.some((root) => isRendererDocumentNavigation(root, contents.getURL())) - } - private notify(connected: boolean, unavailable = false): void { for (const document of this.documents.values()) { if (!document.contents.isDestroyed() && document.contents.mainFrame === document.frame) { @@ -190,6 +151,9 @@ export class BackendSupervisor { if (!this.desiredRunning) { throw new Error('Orca is shutting down.') } + if (explicit) { + this.disposeFailedStartup?.() + } if (this.ready) { return this.ready } @@ -219,6 +183,9 @@ export class BackendSupervisor { let childPid = child.pid child.once('spawn', () => { childPid = child.pid + if (this.child !== child || !this.desiredRunning) { + child.kill() + } }) let incarnation: string | undefined let settleExit!: () => void @@ -226,8 +193,31 @@ export class BackendSupervisor { settleExit = resolve }) let failed!: (error: Error) => void + let settled = false + let disposed = false const ready = new Promise((resolve, reject) => { - failed = reject + failed = (error) => { + if (settled) { + return + } + settled = true + if (this.child === child && this.ready === ready) { + this.disposeFailedStartup = () => { + disposed = true + const control = this.control + this.child = null + this.control = null + this.ready = null + this.disposeFailedStartup = undefined + clearTimeout(startupDeadline) + control?.close() + try { + child.kill() + } catch {} + } + } + reject(error) + } child.on( 'message', (message: { @@ -236,7 +226,7 @@ export class BackendSupervisor { capabilities?: string[] incarnation?: string }) => { - if (this.child !== child) { + if (this.child !== child || settled || !this.desiredRunning) { return } if ( @@ -245,12 +235,13 @@ export class BackendSupervisor { message.capabilities?.includes('usage') ) { incarnation = message.incarnation + settled = true clearTimeout(startupDeadline) resolve() this.notify(true) } if (message.type === 'failed') { - reject(new Error('Usage could not start. Try refreshing the usage view.')) + failed(new Error('Usage could not start. Try refreshing the usage view.')) child.kill() } } @@ -274,7 +265,7 @@ export class BackendSupervisor { pid: childPid ?? null, incarnation: incarnation ?? null, exitCode: code, - expected: !this.desiredRunning + expected: disposed || !this.desiredRunning }) clearTimeout(startupDeadline) failed(new Error('Usage disconnected. Try again after it reconnects.')) @@ -283,6 +274,7 @@ export class BackendSupervisor { this.control = null this.child = null this.ready = null + this.disposeFailedStartup = undefined if (this.desiredRunning) { this.failures.record('backend') }