mirror of
https://github.com/stablyai/orca.git
synced 2026-11-15 18:37:24 +00:00
fix(usage): retry failed backend startup on demand
This commit is contained in:
@@ -1,8 +1,30 @@
|
||||
import { BrowserWindow, type WebContents, type WebFrameMain } from 'electron'
|
||||
import { join } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { agentTokenUsageSchema } from '../../shared/telemetry-agent-token-usage-schema'
|
||||
import { claudeProfileTranscriptDirs } from '../claude-usage/transcript-file-discovery'
|
||||
import { loadKnownUsageWorktreesByRepo } from '../usage/worktree-attribution/usage-worktree-metadata'
|
||||
import { isTelemetryEnabled, track } from '../telemetry/client'
|
||||
import type { UsageWorktreeSource } from '../usage/worktree-attribution/usage-worktree-metadata'
|
||||
import { currentWorkerEntryLayout } from '../worker-threads/worker-thread-entry-path'
|
||||
import { isRendererDocumentNavigation } from '../window/renderer-document-navigation'
|
||||
|
||||
export function trustsBackendRenderer(contents: WebContents): boolean {
|
||||
if (
|
||||
contents.isDestroyed() ||
|
||||
contents.getType() !== 'window' ||
|
||||
!BrowserWindow.fromWebContents(contents)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const layout = currentWorkerEntryLayout(__dirname)
|
||||
const roots = process.env.ELECTRON_RENDERER_URL
|
||||
? [process.env.ELECTRON_RENDERER_URL]
|
||||
: ['index.html', 'space.html', 'popout.html'].map(
|
||||
(name) => pathToFileURL(join(layout.moduleDir, '../renderer', name)).href
|
||||
)
|
||||
return roots.some((root) => isRendererDocumentNavigation(root, contents.getURL()))
|
||||
}
|
||||
|
||||
export function createBackendDesktopCapabilities(
|
||||
store: UsageWorktreeSource
|
||||
@@ -28,3 +50,46 @@ export function createBackendDesktopCapabilities(
|
||||
throw new Error('This desktop capability is not supported.')
|
||||
}
|
||||
}
|
||||
|
||||
export type BackendDocument = {
|
||||
contents: WebContents
|
||||
frame: WebFrameMain
|
||||
token: string
|
||||
retire(): void
|
||||
}
|
||||
|
||||
export function trackBackendDocument(
|
||||
documents: Map<number, BackendDocument>,
|
||||
contents: WebContents,
|
||||
frame: WebFrameMain,
|
||||
token: string,
|
||||
retireToken: (token: string) => void
|
||||
): BackendDocument {
|
||||
documents.get(contents.id)?.retire()
|
||||
const document: BackendDocument = { contents, frame, token, retire: () => {} }
|
||||
const retire = (): void => {
|
||||
if (documents.get(contents.id) !== document) {
|
||||
return
|
||||
}
|
||||
documents.delete(contents.id)
|
||||
retireToken(document.token)
|
||||
contents.off('did-frame-navigate', navigation)
|
||||
contents.off('destroyed', retire)
|
||||
}
|
||||
const navigation = (
|
||||
_event: unknown,
|
||||
_url: string,
|
||||
_httpCode: number,
|
||||
_httpStatus: string,
|
||||
mainFrame: boolean
|
||||
): void => {
|
||||
if (mainFrame) {
|
||||
retire()
|
||||
}
|
||||
}
|
||||
document.retire = retire
|
||||
documents.set(contents.id, document)
|
||||
contents.on('did-frame-navigate', navigation)
|
||||
contents.once('destroyed', retire)
|
||||
return document
|
||||
}
|
||||
|
||||
@@ -232,3 +232,131 @@ it('retires only the exited renderer document and rejects its pending port deliv
|
||||
child.emit('exit', 0)
|
||||
await stopped
|
||||
})
|
||||
|
||||
function startupFixture() {
|
||||
const children: (EventEmitter & {
|
||||
postMessage: ReturnType<typeof vi.fn>
|
||||
kill: ReturnType<typeof vi.fn>
|
||||
})[] = []
|
||||
rig.fork.mockImplementation(() => {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
postMessage: vi.fn(),
|
||||
kill: vi.fn(() => false)
|
||||
})
|
||||
children.push(child)
|
||||
return child
|
||||
})
|
||||
const supervisor = new BackendSupervisor({
|
||||
getRepos: () => [],
|
||||
getAllWorktreeMeta: () => ({}),
|
||||
getRememberedWorktreeDisplayNames: () => new Map(),
|
||||
whenLoaded: async () => {}
|
||||
})
|
||||
const frame = { postMessage: vi.fn() }
|
||||
const sender = Object.assign(new EventEmitter(), {
|
||||
id: 4,
|
||||
mainFrame: frame,
|
||||
isDestroyed: () => false,
|
||||
getType: () => 'window',
|
||||
getURL: () => 'file:///index.html'
|
||||
})
|
||||
const request = rig.handle.mock.calls.find(([channel]) => channel === BACKEND_PORT_REQUEST)?.[1]
|
||||
const token = '44444444-4444-4444-4444-444444444444'
|
||||
const read = () =>
|
||||
supervisor.automationClient('claudeUsage').getAutomationRunUsage({
|
||||
worktreeId: null,
|
||||
terminalSessionId: null,
|
||||
startedAt: null,
|
||||
completedAt: null
|
||||
})
|
||||
return {
|
||||
children,
|
||||
supervisor,
|
||||
frame,
|
||||
read,
|
||||
request: (purpose = 'demand') => request({ sender, senderFrame: frame }, token, purpose)
|
||||
}
|
||||
}
|
||||
|
||||
it('retries a rejected signal-free launch on new demand without replay or false retirement', async () => {
|
||||
vi.useFakeTimers()
|
||||
const { children, supervisor, frame, read, request } = startupFixture()
|
||||
const ready = { type: 'ready', protocol: 1, capabilities: ['usage'] }
|
||||
for (let attempt = 0; attempt < 4; attempt++) {
|
||||
const demand = expect(read()).rejects.toThrow('Usage is still unavailable')
|
||||
const joined = expect(request('recovery')).rejects.toThrow('Usage is still unavailable')
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
await demand
|
||||
await joined
|
||||
children[attempt].emit('message', ready)
|
||||
expect(frame.postMessage).not.toHaveBeenCalledWith(BACKEND_CONNECTION_CHANGED, {
|
||||
connected: true,
|
||||
unavailable: false
|
||||
})
|
||||
await expect(request('recovery')).rejects.toThrow('Usage is still unavailable')
|
||||
expect(children).toHaveLength(attempt + 1)
|
||||
expect(rig.call).not.toHaveBeenCalled()
|
||||
}
|
||||
const retry = read()
|
||||
const port = request()
|
||||
children[4].emit('message', ready)
|
||||
rig.call.mockResolvedValueOnce({ inputTokens: 42 })
|
||||
await expect(retry).resolves.toEqual({ inputTokens: 42 })
|
||||
await port
|
||||
expect(rig.call).toHaveBeenCalledTimes(1)
|
||||
const notifications = frame.postMessage.mock.calls.slice()
|
||||
for (const child of children.slice(0, 4)) {
|
||||
child.emit('spawn')
|
||||
expect(child.kill).toHaveBeenCalled()
|
||||
child.emit('message', ready)
|
||||
child.emit('exit', 1)
|
||||
}
|
||||
expect(frame.postMessage.mock.calls).toEqual(notifications)
|
||||
const { recordDurableCrashBreadcrumb } =
|
||||
await import('../crash-reporting/breadcrumbs/durable-crash-breadcrumb')
|
||||
expect(recordDurableCrashBreadcrumb).toHaveBeenLastCalledWith('backend_process_exited', {
|
||||
pid: null,
|
||||
incarnation: null,
|
||||
exitCode: 1,
|
||||
expected: true
|
||||
})
|
||||
// A rejection after successful readiness cannot turn that owner into a failed startup.
|
||||
children[4].emit('error', new Error('late error'))
|
||||
rig.call.mockResolvedValueOnce({ inputTokens: 43 })
|
||||
await expect(read()).resolves.toEqual({ inputTokens: 43 })
|
||||
expect(children).toHaveLength(5)
|
||||
const stopped = supervisor.stop()
|
||||
let quit = false
|
||||
void stopped.then(() => {
|
||||
quit = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(quit).toBe(false)
|
||||
children[4].emit('exit', 0)
|
||||
await stopped
|
||||
await expect(read()).rejects.toThrow('Orca is shutting down')
|
||||
expect(children).toHaveLength(5)
|
||||
})
|
||||
|
||||
it('rejects old requests when ownership changes across readiness instead of rerouting them', async () => {
|
||||
const { children, supervisor, frame, read, request } = startupFixture()
|
||||
const ready = { type: 'ready', protocol: 1, capabilities: ['usage'] }
|
||||
children[0].emit('message', ready)
|
||||
const oldRead = expect(read()).rejects.toThrow('Usage disconnected')
|
||||
const oldPort = expect(request()).rejects.toThrow('The document was replaced')
|
||||
children[0].emit('exit', 1)
|
||||
const replacement = request()
|
||||
children[1].emit('message', ready)
|
||||
await oldRead
|
||||
await oldPort
|
||||
await replacement
|
||||
expect(rig.call).not.toHaveBeenCalled()
|
||||
expect(
|
||||
frame.postMessage.mock.calls.filter(([channel]) => channel === BACKEND_PORT_DELIVERY)
|
||||
).toHaveLength(1)
|
||||
const stopped = supervisor.stop()
|
||||
children[1].emit('spawn')
|
||||
expect(children[1].kill).toHaveBeenCalled()
|
||||
children[1].emit('exit', 0)
|
||||
await stopped
|
||||
})
|
||||
|
||||
@@ -1,15 +1,5 @@
|
||||
import {
|
||||
app,
|
||||
BrowserWindow,
|
||||
ipcMain,
|
||||
MessageChannelMain,
|
||||
utilityProcess,
|
||||
type WebContents,
|
||||
type WebFrameMain
|
||||
} from 'electron'
|
||||
import { join } from 'node:path'
|
||||
import { app, ipcMain, MessageChannelMain, utilityProcess, type WebContents } from 'electron'
|
||||
import { recordDurableCrashBreadcrumb } from '../crash-reporting/breadcrumbs/durable-crash-breadcrumb'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { BackendPortRpc } from '../../shared/backend-port-rpc'
|
||||
import {
|
||||
BACKEND_CONNECTION_CHANGED,
|
||||
@@ -25,24 +15,27 @@ import {
|
||||
} from '../worker-threads/worker-thread-entry-path'
|
||||
import { ManagedOrcadEventWindow } from '../managed-server/lifecycle/orcad-managed-event-window'
|
||||
import { ORCAD_SHUTDOWN_DEADLINE_MS } from '../orcad/orcad-stop-deadlines'
|
||||
import { isRendererDocumentNavigation } from '../window/renderer-document-navigation'
|
||||
import { createBackendDesktopCapabilities } from './backend-desktop-capabilities'
|
||||
import {
|
||||
createBackendDesktopCapabilities,
|
||||
trustsBackendRenderer,
|
||||
trackBackendDocument,
|
||||
type BackendDocument
|
||||
} from './backend-desktop-capabilities'
|
||||
import type { UsageWorktreeSource } from '../usage/worktree-attribution/usage-worktree-metadata'
|
||||
|
||||
export { captureBackendBootstrap } from './backend-bootstrap'
|
||||
import { capturedBootstrap } from './backend-bootstrap'
|
||||
|
||||
type Document = { contents: WebContents; frame: WebFrameMain; token: string; retire(): void }
|
||||
|
||||
export class BackendSupervisor {
|
||||
private child: ReturnType<typeof utilityProcess.fork> | null = null
|
||||
private control: BackendPortRpc | null = null
|
||||
private ready: Promise<void> | null = null
|
||||
private disposeFailedStartup?: () => void
|
||||
private desiredRunning = true
|
||||
private exited: Promise<void> = Promise.resolve()
|
||||
private restartTimer: ReturnType<typeof setTimeout> | undefined
|
||||
private readonly failures = new ManagedOrcadEventWindow(10 * 60_000, Date.now)
|
||||
private readonly documents = new Map<number, Document>()
|
||||
private readonly documents = new Map<number, BackendDocument>()
|
||||
private stopping?: Promise<void>
|
||||
|
||||
private readonly retireRenderer = (_event: unknown, contents: WebContents): void =>
|
||||
@@ -64,57 +57,37 @@ export class BackendSupervisor {
|
||||
throw new Error('Invalid backend connection request.')
|
||||
}
|
||||
const { sender, senderFrame } = event
|
||||
if (!senderFrame || senderFrame !== sender.mainFrame || !this.trusts(sender)) {
|
||||
if (!senderFrame || senderFrame !== sender.mainFrame || !trustsBackendRenderer(sender)) {
|
||||
throw new Error('This document cannot connect to Orca.')
|
||||
}
|
||||
this.documents.get(sender.id)?.retire()
|
||||
const document: Document = {
|
||||
contents: sender,
|
||||
frame: senderFrame,
|
||||
const document = trackBackendDocument(
|
||||
this.documents,
|
||||
sender,
|
||||
senderFrame,
|
||||
token,
|
||||
retire: () => {}
|
||||
}
|
||||
const retire = (): void => {
|
||||
if (this.documents.get(sender.id) !== document) {
|
||||
return
|
||||
}
|
||||
this.documents.delete(sender.id)
|
||||
this.child?.postMessage({ type: 'retire', token: document.token })
|
||||
sender.off('did-frame-navigate', navigation)
|
||||
sender.off('destroyed', retire)
|
||||
}
|
||||
const navigation = (
|
||||
_event: unknown,
|
||||
_url: string,
|
||||
_httpCode: number,
|
||||
_httpStatus: string,
|
||||
mainFrame: boolean
|
||||
): void => {
|
||||
if (mainFrame) {
|
||||
retire()
|
||||
}
|
||||
}
|
||||
document.retire = retire
|
||||
this.documents.set(sender.id, document)
|
||||
sender.on('did-frame-navigate', navigation)
|
||||
sender.once('destroyed', retire)
|
||||
(retiredToken) => this.child?.postMessage({ type: 'retire', token: retiredToken })
|
||||
)
|
||||
if (!this.desiredRunning) {
|
||||
throw new Error('Orca is shutting down.')
|
||||
}
|
||||
const ready = purpose === 'recovery' ? this.ready : this.ensureReady(true)
|
||||
const child = this.child
|
||||
if (!ready) {
|
||||
throw new Error('Usage is unavailable. Try refreshing the usage view.')
|
||||
}
|
||||
await ready
|
||||
if (
|
||||
this.documents.get(sender.id) !== document ||
|
||||
!this.desiredRunning ||
|
||||
!child ||
|
||||
this.child !== child ||
|
||||
sender.isDestroyed() ||
|
||||
sender.mainFrame !== senderFrame
|
||||
) {
|
||||
throw new Error('The document was replaced before Orca connected.')
|
||||
}
|
||||
const { port1, port2 } = new MessageChannelMain()
|
||||
this.child!.postMessage({ type: 'client', token: document.token }, [port1])
|
||||
child.postMessage({ type: 'client', token: document.token }, [port1])
|
||||
senderFrame.postMessage(BACKEND_PORT_DELIVERY, { token: document.token }, [port2])
|
||||
return document.token
|
||||
}
|
||||
@@ -125,9 +98,14 @@ export class BackendSupervisor {
|
||||
automationClient(provider: 'claudeUsage' | 'codexUsage'): AutomationUsageClient {
|
||||
return {
|
||||
getAutomationRunUsage: async (input) => {
|
||||
await this.ensureReady(true)
|
||||
const ready = this.ensureReady(true)
|
||||
const control = this.control
|
||||
await ready
|
||||
if (!this.desiredRunning || !control || this.control !== control) {
|
||||
throw new Error('Usage disconnected. Try again after it reconnects.')
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The private usage service returns its typed automation result for this provider method.
|
||||
return (await this.control!.call(`${provider}:getAutomationRunUsage`, input)) as Awaited<
|
||||
return (await control.call(`${provider}:getAutomationRunUsage`, input)) as Awaited<
|
||||
ReturnType<AutomationUsageClient['getAutomationRunUsage']>
|
||||
>
|
||||
}
|
||||
@@ -161,23 +139,6 @@ export class BackendSupervisor {
|
||||
return this.stopping
|
||||
}
|
||||
|
||||
private trusts(contents: WebContents): boolean {
|
||||
if (
|
||||
contents.isDestroyed() ||
|
||||
contents.getType() !== 'window' ||
|
||||
!BrowserWindow.fromWebContents(contents)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const layout = currentWorkerEntryLayout(__dirname)
|
||||
const roots = process.env.ELECTRON_RENDERER_URL
|
||||
? [process.env.ELECTRON_RENDERER_URL]
|
||||
: ['index.html', 'space.html', 'popout.html'].map(
|
||||
(name) => pathToFileURL(join(layout.moduleDir, '../renderer', name)).href
|
||||
)
|
||||
return roots.some((root) => isRendererDocumentNavigation(root, contents.getURL()))
|
||||
}
|
||||
|
||||
private notify(connected: boolean, unavailable = false): void {
|
||||
for (const document of this.documents.values()) {
|
||||
if (!document.contents.isDestroyed() && document.contents.mainFrame === document.frame) {
|
||||
@@ -190,6 +151,9 @@ export class BackendSupervisor {
|
||||
if (!this.desiredRunning) {
|
||||
throw new Error('Orca is shutting down.')
|
||||
}
|
||||
if (explicit) {
|
||||
this.disposeFailedStartup?.()
|
||||
}
|
||||
if (this.ready) {
|
||||
return this.ready
|
||||
}
|
||||
@@ -219,6 +183,9 @@ export class BackendSupervisor {
|
||||
let childPid = child.pid
|
||||
child.once('spawn', () => {
|
||||
childPid = child.pid
|
||||
if (this.child !== child || !this.desiredRunning) {
|
||||
child.kill()
|
||||
}
|
||||
})
|
||||
let incarnation: string | undefined
|
||||
let settleExit!: () => void
|
||||
@@ -226,8 +193,31 @@ export class BackendSupervisor {
|
||||
settleExit = resolve
|
||||
})
|
||||
let failed!: (error: Error) => void
|
||||
let settled = false
|
||||
let disposed = false
|
||||
const ready = new Promise<void>((resolve, reject) => {
|
||||
failed = reject
|
||||
failed = (error) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
if (this.child === child && this.ready === ready) {
|
||||
this.disposeFailedStartup = () => {
|
||||
disposed = true
|
||||
const control = this.control
|
||||
this.child = null
|
||||
this.control = null
|
||||
this.ready = null
|
||||
this.disposeFailedStartup = undefined
|
||||
clearTimeout(startupDeadline)
|
||||
control?.close()
|
||||
try {
|
||||
child.kill()
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
reject(error)
|
||||
}
|
||||
child.on(
|
||||
'message',
|
||||
(message: {
|
||||
@@ -236,7 +226,7 @@ export class BackendSupervisor {
|
||||
capabilities?: string[]
|
||||
incarnation?: string
|
||||
}) => {
|
||||
if (this.child !== child) {
|
||||
if (this.child !== child || settled || !this.desiredRunning) {
|
||||
return
|
||||
}
|
||||
if (
|
||||
@@ -245,12 +235,13 @@ export class BackendSupervisor {
|
||||
message.capabilities?.includes('usage')
|
||||
) {
|
||||
incarnation = message.incarnation
|
||||
settled = true
|
||||
clearTimeout(startupDeadline)
|
||||
resolve()
|
||||
this.notify(true)
|
||||
}
|
||||
if (message.type === 'failed') {
|
||||
reject(new Error('Usage could not start. Try refreshing the usage view.'))
|
||||
failed(new Error('Usage could not start. Try refreshing the usage view.'))
|
||||
child.kill()
|
||||
}
|
||||
}
|
||||
@@ -274,7 +265,7 @@ export class BackendSupervisor {
|
||||
pid: childPid ?? null,
|
||||
incarnation: incarnation ?? null,
|
||||
exitCode: code,
|
||||
expected: !this.desiredRunning
|
||||
expected: disposed || !this.desiredRunning
|
||||
})
|
||||
clearTimeout(startupDeadline)
|
||||
failed(new Error('Usage disconnected. Try again after it reconnects.'))
|
||||
@@ -283,6 +274,7 @@ export class BackendSupervisor {
|
||||
this.control = null
|
||||
this.child = null
|
||||
this.ready = null
|
||||
this.disposeFailedStartup = undefined
|
||||
if (this.desiredRunning) {
|
||||
this.failures.record('backend')
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user