fix(usage): retry failed backend startup on demand

This commit is contained in:
Brennan Benson
2026-11-11 15:32:23 -08:00
parent a55dfc5f19
commit 08dedd2cdb
3 changed files with 256 additions and 71 deletions
@@ -1,8 +1,30 @@
import { BrowserWindow, type WebContents, type WebFrameMain } from 'electron'
import { join } from 'node:path'
import { pathToFileURL } from 'node:url'
import { agentTokenUsageSchema } from '../../shared/telemetry-agent-token-usage-schema'
import { claudeProfileTranscriptDirs } from '../claude-usage/transcript-file-discovery'
import { loadKnownUsageWorktreesByRepo } from '../usage/worktree-attribution/usage-worktree-metadata'
import { isTelemetryEnabled, track } from '../telemetry/client'
import type { UsageWorktreeSource } from '../usage/worktree-attribution/usage-worktree-metadata'
import { currentWorkerEntryLayout } from '../worker-threads/worker-thread-entry-path'
import { isRendererDocumentNavigation } from '../window/renderer-document-navigation'
export function trustsBackendRenderer(contents: WebContents): boolean {
if (
contents.isDestroyed() ||
contents.getType() !== 'window' ||
!BrowserWindow.fromWebContents(contents)
) {
return false
}
const layout = currentWorkerEntryLayout(__dirname)
const roots = process.env.ELECTRON_RENDERER_URL
? [process.env.ELECTRON_RENDERER_URL]
: ['index.html', 'space.html', 'popout.html'].map(
(name) => pathToFileURL(join(layout.moduleDir, '../renderer', name)).href
)
return roots.some((root) => isRendererDocumentNavigation(root, contents.getURL()))
}
export function createBackendDesktopCapabilities(
store: UsageWorktreeSource
@@ -28,3 +50,46 @@ export function createBackendDesktopCapabilities(
throw new Error('This desktop capability is not supported.')
}
}
export type BackendDocument = {
contents: WebContents
frame: WebFrameMain
token: string
retire(): void
}
export function trackBackendDocument(
documents: Map<number, BackendDocument>,
contents: WebContents,
frame: WebFrameMain,
token: string,
retireToken: (token: string) => void
): BackendDocument {
documents.get(contents.id)?.retire()
const document: BackendDocument = { contents, frame, token, retire: () => {} }
const retire = (): void => {
if (documents.get(contents.id) !== document) {
return
}
documents.delete(contents.id)
retireToken(document.token)
contents.off('did-frame-navigate', navigation)
contents.off('destroyed', retire)
}
const navigation = (
_event: unknown,
_url: string,
_httpCode: number,
_httpStatus: string,
mainFrame: boolean
): void => {
if (mainFrame) {
retire()
}
}
document.retire = retire
documents.set(contents.id, document)
contents.on('did-frame-navigate', navigation)
contents.once('destroyed', retire)
return document
}
@@ -232,3 +232,131 @@ it('retires only the exited renderer document and rejects its pending port deliv
child.emit('exit', 0)
await stopped
})
function startupFixture() {
const children: (EventEmitter & {
postMessage: ReturnType<typeof vi.fn>
kill: ReturnType<typeof vi.fn>
})[] = []
rig.fork.mockImplementation(() => {
const child = Object.assign(new EventEmitter(), {
postMessage: vi.fn(),
kill: vi.fn(() => false)
})
children.push(child)
return child
})
const supervisor = new BackendSupervisor({
getRepos: () => [],
getAllWorktreeMeta: () => ({}),
getRememberedWorktreeDisplayNames: () => new Map(),
whenLoaded: async () => {}
})
const frame = { postMessage: vi.fn() }
const sender = Object.assign(new EventEmitter(), {
id: 4,
mainFrame: frame,
isDestroyed: () => false,
getType: () => 'window',
getURL: () => 'file:///index.html'
})
const request = rig.handle.mock.calls.find(([channel]) => channel === BACKEND_PORT_REQUEST)?.[1]
const token = '44444444-4444-4444-4444-444444444444'
const read = () =>
supervisor.automationClient('claudeUsage').getAutomationRunUsage({
worktreeId: null,
terminalSessionId: null,
startedAt: null,
completedAt: null
})
return {
children,
supervisor,
frame,
read,
request: (purpose = 'demand') => request({ sender, senderFrame: frame }, token, purpose)
}
}
it('retries a rejected signal-free launch on new demand without replay or false retirement', async () => {
vi.useFakeTimers()
const { children, supervisor, frame, read, request } = startupFixture()
const ready = { type: 'ready', protocol: 1, capabilities: ['usage'] }
for (let attempt = 0; attempt < 4; attempt++) {
const demand = expect(read()).rejects.toThrow('Usage is still unavailable')
const joined = expect(request('recovery')).rejects.toThrow('Usage is still unavailable')
await vi.advanceTimersByTimeAsync(30_000)
await demand
await joined
children[attempt].emit('message', ready)
expect(frame.postMessage).not.toHaveBeenCalledWith(BACKEND_CONNECTION_CHANGED, {
connected: true,
unavailable: false
})
await expect(request('recovery')).rejects.toThrow('Usage is still unavailable')
expect(children).toHaveLength(attempt + 1)
expect(rig.call).not.toHaveBeenCalled()
}
const retry = read()
const port = request()
children[4].emit('message', ready)
rig.call.mockResolvedValueOnce({ inputTokens: 42 })
await expect(retry).resolves.toEqual({ inputTokens: 42 })
await port
expect(rig.call).toHaveBeenCalledTimes(1)
const notifications = frame.postMessage.mock.calls.slice()
for (const child of children.slice(0, 4)) {
child.emit('spawn')
expect(child.kill).toHaveBeenCalled()
child.emit('message', ready)
child.emit('exit', 1)
}
expect(frame.postMessage.mock.calls).toEqual(notifications)
const { recordDurableCrashBreadcrumb } =
await import('../crash-reporting/breadcrumbs/durable-crash-breadcrumb')
expect(recordDurableCrashBreadcrumb).toHaveBeenLastCalledWith('backend_process_exited', {
pid: null,
incarnation: null,
exitCode: 1,
expected: true
})
// A rejection after successful readiness cannot turn that owner into a failed startup.
children[4].emit('error', new Error('late error'))
rig.call.mockResolvedValueOnce({ inputTokens: 43 })
await expect(read()).resolves.toEqual({ inputTokens: 43 })
expect(children).toHaveLength(5)
const stopped = supervisor.stop()
let quit = false
void stopped.then(() => {
quit = true
})
await Promise.resolve()
expect(quit).toBe(false)
children[4].emit('exit', 0)
await stopped
await expect(read()).rejects.toThrow('Orca is shutting down')
expect(children).toHaveLength(5)
})
it('rejects old requests when ownership changes across readiness instead of rerouting them', async () => {
const { children, supervisor, frame, read, request } = startupFixture()
const ready = { type: 'ready', protocol: 1, capabilities: ['usage'] }
children[0].emit('message', ready)
const oldRead = expect(read()).rejects.toThrow('Usage disconnected')
const oldPort = expect(request()).rejects.toThrow('The document was replaced')
children[0].emit('exit', 1)
const replacement = request()
children[1].emit('message', ready)
await oldRead
await oldPort
await replacement
expect(rig.call).not.toHaveBeenCalled()
expect(
frame.postMessage.mock.calls.filter(([channel]) => channel === BACKEND_PORT_DELIVERY)
).toHaveLength(1)
const stopped = supervisor.stop()
children[1].emit('spawn')
expect(children[1].kill).toHaveBeenCalled()
children[1].emit('exit', 0)
await stopped
})
+63 -71
View File
@@ -1,15 +1,5 @@
import {
app,
BrowserWindow,
ipcMain,
MessageChannelMain,
utilityProcess,
type WebContents,
type WebFrameMain
} from 'electron'
import { join } from 'node:path'
import { app, ipcMain, MessageChannelMain, utilityProcess, type WebContents } from 'electron'
import { recordDurableCrashBreadcrumb } from '../crash-reporting/breadcrumbs/durable-crash-breadcrumb'
import { pathToFileURL } from 'node:url'
import { BackendPortRpc } from '../../shared/backend-port-rpc'
import {
BACKEND_CONNECTION_CHANGED,
@@ -25,24 +15,27 @@ import {
} from '../worker-threads/worker-thread-entry-path'
import { ManagedOrcadEventWindow } from '../managed-server/lifecycle/orcad-managed-event-window'
import { ORCAD_SHUTDOWN_DEADLINE_MS } from '../orcad/orcad-stop-deadlines'
import { isRendererDocumentNavigation } from '../window/renderer-document-navigation'
import { createBackendDesktopCapabilities } from './backend-desktop-capabilities'
import {
createBackendDesktopCapabilities,
trustsBackendRenderer,
trackBackendDocument,
type BackendDocument
} from './backend-desktop-capabilities'
import type { UsageWorktreeSource } from '../usage/worktree-attribution/usage-worktree-metadata'
export { captureBackendBootstrap } from './backend-bootstrap'
import { capturedBootstrap } from './backend-bootstrap'
type Document = { contents: WebContents; frame: WebFrameMain; token: string; retire(): void }
export class BackendSupervisor {
private child: ReturnType<typeof utilityProcess.fork> | null = null
private control: BackendPortRpc | null = null
private ready: Promise<void> | null = null
private disposeFailedStartup?: () => void
private desiredRunning = true
private exited: Promise<void> = Promise.resolve()
private restartTimer: ReturnType<typeof setTimeout> | undefined
private readonly failures = new ManagedOrcadEventWindow(10 * 60_000, Date.now)
private readonly documents = new Map<number, Document>()
private readonly documents = new Map<number, BackendDocument>()
private stopping?: Promise<void>
private readonly retireRenderer = (_event: unknown, contents: WebContents): void =>
@@ -64,57 +57,37 @@ export class BackendSupervisor {
throw new Error('Invalid backend connection request.')
}
const { sender, senderFrame } = event
if (!senderFrame || senderFrame !== sender.mainFrame || !this.trusts(sender)) {
if (!senderFrame || senderFrame !== sender.mainFrame || !trustsBackendRenderer(sender)) {
throw new Error('This document cannot connect to Orca.')
}
this.documents.get(sender.id)?.retire()
const document: Document = {
contents: sender,
frame: senderFrame,
const document = trackBackendDocument(
this.documents,
sender,
senderFrame,
token,
retire: () => {}
}
const retire = (): void => {
if (this.documents.get(sender.id) !== document) {
return
}
this.documents.delete(sender.id)
this.child?.postMessage({ type: 'retire', token: document.token })
sender.off('did-frame-navigate', navigation)
sender.off('destroyed', retire)
}
const navigation = (
_event: unknown,
_url: string,
_httpCode: number,
_httpStatus: string,
mainFrame: boolean
): void => {
if (mainFrame) {
retire()
}
}
document.retire = retire
this.documents.set(sender.id, document)
sender.on('did-frame-navigate', navigation)
sender.once('destroyed', retire)
(retiredToken) => this.child?.postMessage({ type: 'retire', token: retiredToken })
)
if (!this.desiredRunning) {
throw new Error('Orca is shutting down.')
}
const ready = purpose === 'recovery' ? this.ready : this.ensureReady(true)
const child = this.child
if (!ready) {
throw new Error('Usage is unavailable. Try refreshing the usage view.')
}
await ready
if (
this.documents.get(sender.id) !== document ||
!this.desiredRunning ||
!child ||
this.child !== child ||
sender.isDestroyed() ||
sender.mainFrame !== senderFrame
) {
throw new Error('The document was replaced before Orca connected.')
}
const { port1, port2 } = new MessageChannelMain()
this.child!.postMessage({ type: 'client', token: document.token }, [port1])
child.postMessage({ type: 'client', token: document.token }, [port1])
senderFrame.postMessage(BACKEND_PORT_DELIVERY, { token: document.token }, [port2])
return document.token
}
@@ -125,9 +98,14 @@ export class BackendSupervisor {
automationClient(provider: 'claudeUsage' | 'codexUsage'): AutomationUsageClient {
return {
getAutomationRunUsage: async (input) => {
await this.ensureReady(true)
const ready = this.ensureReady(true)
const control = this.control
await ready
if (!this.desiredRunning || !control || this.control !== control) {
throw new Error('Usage disconnected. Try again after it reconnects.')
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The private usage service returns its typed automation result for this provider method.
return (await this.control!.call(`${provider}:getAutomationRunUsage`, input)) as Awaited<
return (await control.call(`${provider}:getAutomationRunUsage`, input)) as Awaited<
ReturnType<AutomationUsageClient['getAutomationRunUsage']>
>
}
@@ -161,23 +139,6 @@ export class BackendSupervisor {
return this.stopping
}
private trusts(contents: WebContents): boolean {
if (
contents.isDestroyed() ||
contents.getType() !== 'window' ||
!BrowserWindow.fromWebContents(contents)
) {
return false
}
const layout = currentWorkerEntryLayout(__dirname)
const roots = process.env.ELECTRON_RENDERER_URL
? [process.env.ELECTRON_RENDERER_URL]
: ['index.html', 'space.html', 'popout.html'].map(
(name) => pathToFileURL(join(layout.moduleDir, '../renderer', name)).href
)
return roots.some((root) => isRendererDocumentNavigation(root, contents.getURL()))
}
private notify(connected: boolean, unavailable = false): void {
for (const document of this.documents.values()) {
if (!document.contents.isDestroyed() && document.contents.mainFrame === document.frame) {
@@ -190,6 +151,9 @@ export class BackendSupervisor {
if (!this.desiredRunning) {
throw new Error('Orca is shutting down.')
}
if (explicit) {
this.disposeFailedStartup?.()
}
if (this.ready) {
return this.ready
}
@@ -219,6 +183,9 @@ export class BackendSupervisor {
let childPid = child.pid
child.once('spawn', () => {
childPid = child.pid
if (this.child !== child || !this.desiredRunning) {
child.kill()
}
})
let incarnation: string | undefined
let settleExit!: () => void
@@ -226,8 +193,31 @@ export class BackendSupervisor {
settleExit = resolve
})
let failed!: (error: Error) => void
let settled = false
let disposed = false
const ready = new Promise<void>((resolve, reject) => {
failed = reject
failed = (error) => {
if (settled) {
return
}
settled = true
if (this.child === child && this.ready === ready) {
this.disposeFailedStartup = () => {
disposed = true
const control = this.control
this.child = null
this.control = null
this.ready = null
this.disposeFailedStartup = undefined
clearTimeout(startupDeadline)
control?.close()
try {
child.kill()
} catch {}
}
}
reject(error)
}
child.on(
'message',
(message: {
@@ -236,7 +226,7 @@ export class BackendSupervisor {
capabilities?: string[]
incarnation?: string
}) => {
if (this.child !== child) {
if (this.child !== child || settled || !this.desiredRunning) {
return
}
if (
@@ -245,12 +235,13 @@ export class BackendSupervisor {
message.capabilities?.includes('usage')
) {
incarnation = message.incarnation
settled = true
clearTimeout(startupDeadline)
resolve()
this.notify(true)
}
if (message.type === 'failed') {
reject(new Error('Usage could not start. Try refreshing the usage view.'))
failed(new Error('Usage could not start. Try refreshing the usage view.'))
child.kill()
}
}
@@ -274,7 +265,7 @@ export class BackendSupervisor {
pid: childPid ?? null,
incarnation: incarnation ?? null,
exitCode: code,
expected: !this.desiredRunning
expected: disposed || !this.desiredRunning
})
clearTimeout(startupDeadline)
failed(new Error('Usage disconnected. Try again after it reconnects.'))
@@ -283,6 +274,7 @@ export class BackendSupervisor {
this.control = null
this.child = null
this.ready = null
this.disposeFailedStartup = undefined
if (this.desiredRunning) {
this.failures.record('backend')
}