mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 11:54:46 +00:00
fix(agent-picker): keep loading models when an older WSL host isn't ready yet (#43497)
* fix(agent-picker): retry model catalog reads an older host refused before its chat host started Hosts before 1.4.248 refuse a session-less agentSession.modelCatalog read with structured_agent_session_unsupported (hostDisabled) until their chat host is built. The picker treated that as permanently unsupported and fell back to built-in models, which are empty for Pi and OMP. Treat that refusal as transient and retry failed catalog reads a few times while the picker is open. * fix(agent-picker): retry only the old-host refusal and dropped connections Other catalog read errors, including timeouts, show at once as before.
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RuntimeRpcResponse } from '../../../../shared/remote-runtime/rpc/runtime-rpc-envelope'
|
||||
import type { AgentPickerEntry } from '../../../../shared/agent-launch/picker/agent-launch-picker'
|
||||
import { agentPickerModels } from '../../../../shared/agent-launch/picker/agent-launch-picker-catalog'
|
||||
import { unwrapRuntimeRpcResult } from '@/runtime/rpc/calls/runtime-rpc-result'
|
||||
import { observeAgentPickerAcquisition } from './agent-launch-picker-acquisition'
|
||||
import { AGENT_PICKER_CATALOG_RETRY_DELAYS_MS } from './agent-picker-catalog-jobs'
|
||||
import { agentPickerModelCatalogReads } from './agent-launch-picker-host-reads'
|
||||
|
||||
const mocks = vi.hoisted(() => {
|
||||
const responses: RuntimeRpcResponse<unknown>[] = []
|
||||
const calls: { method: string; params: unknown }[] = []
|
||||
return { responses, calls }
|
||||
})
|
||||
|
||||
// The remote runtime transport: each call answers with the next wire envelope the host sent.
|
||||
vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({
|
||||
callRuntimeRpc: async (_target: unknown, method: string, params: unknown) => {
|
||||
mocks.calls.push({ method, params })
|
||||
const response = mocks.responses.shift()
|
||||
if (!response) {
|
||||
throw new Error('unexpected call')
|
||||
}
|
||||
return unwrapRuntimeRpcResult(response)
|
||||
}
|
||||
}))
|
||||
|
||||
const meta = { runtimeId: 'wsl-runtime' }
|
||||
/** What a host before 1.4.248 answers a session-less catalog read before its chat host is built. */
|
||||
function refusal(reason: 'hostDisabled' | 'clientCapabilityMissing'): RuntimeRpcResponse<unknown> {
|
||||
return {
|
||||
id: 'r',
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'structured_agent_session_unsupported',
|
||||
message: 'structured_agent_session_unsupported',
|
||||
data: { refusal: { code: 'structured_agent_session_unsupported', details: { reason } } }
|
||||
},
|
||||
_meta: meta
|
||||
}
|
||||
}
|
||||
function failure(code: string, message: string): RuntimeRpcResponse<unknown> {
|
||||
return { id: 'r', ok: false, error: { code, message }, _meta: meta }
|
||||
}
|
||||
const piCatalog: RuntimeRpcResponse<unknown> = {
|
||||
id: 'r',
|
||||
ok: true,
|
||||
result: {
|
||||
origin: 'probe',
|
||||
fetchedAt: 1,
|
||||
models: [
|
||||
{ id: 'gateway/model-a', label: 'Model A', isDefault: true, efforts: [] },
|
||||
{ id: 'gateway/model-b', label: 'Model B', isDefault: false, efforts: [] }
|
||||
]
|
||||
},
|
||||
_meta: meta
|
||||
}
|
||||
|
||||
const pi: AgentPickerEntry = {
|
||||
agent: 'pi',
|
||||
label: 'Pi',
|
||||
rows: [],
|
||||
surfaces: ['chat', 'terminal'],
|
||||
defaultSurface: 'chat',
|
||||
terminal: {},
|
||||
structured: {},
|
||||
catalog: { origin: 'unknown' }
|
||||
}
|
||||
|
||||
function observe(key: string) {
|
||||
let latest: AgentPickerEntry[] = []
|
||||
let invalidate = () => {}
|
||||
const reads = agentPickerModelCatalogReads({
|
||||
target: { kind: 'environment', environmentId: 'wsl-env' },
|
||||
worktree: 'id:repo::/home/me/project'
|
||||
})
|
||||
const stop = observeAgentPickerAcquisition(
|
||||
key,
|
||||
[pi],
|
||||
{
|
||||
catalog: reads.catalog,
|
||||
refreshOnInvalidation: () => true,
|
||||
subscribeInvalidation: (listener) => {
|
||||
invalidate = listener
|
||||
return () => {}
|
||||
},
|
||||
support: async () => undefined
|
||||
},
|
||||
{ selected: () => undefined, publish: (entries) => (latest = entries) }
|
||||
)
|
||||
return { stop, entry: () => latest[0], invalidate: () => invalidate() }
|
||||
}
|
||||
const BACKOFF_TOTAL_MS = AGENT_PICKER_CATALOG_RETRY_DELAYS_MS.reduce((sum, delay) => sum + delay, 0)
|
||||
|
||||
describe('agent picker catalog acquisition against a remote runtime', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
mocks.responses.length = 0
|
||||
mocks.calls.length = 0
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('retries an older host that refused the read before its chat host was built', async () => {
|
||||
mocks.responses.push(refusal('hostDisabled'), piCatalog)
|
||||
const picker = observe('host-not-built')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
// Still loading rather than settled on Pi's empty built-in list.
|
||||
expect(picker.entry().catalogRead).toBe('pending')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(AGENT_PICKER_CATALOG_RETRY_DELAYS_MS[0])
|
||||
expect(mocks.calls.map(({ method }) => method)).toEqual([
|
||||
'agentSession.modelCatalog',
|
||||
'agentSession.modelCatalog'
|
||||
])
|
||||
expect(picker.entry().catalogRead).toBe('settled')
|
||||
expect(agentPickerModels(picker.entry(), 'chat').map(({ id }) => id)).toEqual(
|
||||
expect.arrayContaining(['gateway/model-a', 'gateway/model-b'])
|
||||
)
|
||||
picker.stop()
|
||||
})
|
||||
|
||||
it('gives up after its bounded retries', async () => {
|
||||
mocks.responses.push(...Array.from({ length: 4 }, () => refusal('hostDisabled')))
|
||||
const picker = observe('host-never-built')
|
||||
await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS)
|
||||
expect(mocks.calls).toHaveLength(4)
|
||||
expect(picker.entry().catalogRead).toBe('failed')
|
||||
picker.stop()
|
||||
})
|
||||
|
||||
it('does not retry a client the host will never serve', async () => {
|
||||
mocks.responses.push(refusal('clientCapabilityMissing'))
|
||||
const picker = observe('client-unsupported')
|
||||
await vi.advanceTimersByTimeAsync(20_000)
|
||||
expect(mocks.calls).toHaveLength(1)
|
||||
expect(picker.entry().catalogRead).toBe('unsupported')
|
||||
picker.stop()
|
||||
})
|
||||
|
||||
it('stops retrying once the picker closes', async () => {
|
||||
mocks.responses.push(refusal('hostDisabled'))
|
||||
const picker = observe('closed')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
picker.stop()
|
||||
await vi.advanceTimersByTimeAsync(20_000)
|
||||
expect(mocks.calls).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('retries a read the dropped connection lost', async () => {
|
||||
mocks.responses.push(failure('reconnecting', 'Remote Orca runtime is reconnecting'), piCatalog)
|
||||
const picker = observe('reconnecting')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(picker.entry().catalogRead).toBe('pending')
|
||||
await vi.advanceTimersByTimeAsync(AGENT_PICKER_CATALOG_RETRY_DELAYS_MS[0])
|
||||
expect(mocks.calls).toHaveLength(2)
|
||||
expect(picker.entry().catalogRead).toBe('settled')
|
||||
picker.stop()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['a generic host error', failure('runtime_error', 'listing crashed')],
|
||||
['a timeout', failure('runtime_timeout', 'Timed out waiting for the remote Orca runtime')]
|
||||
])('shows %s at once without retrying', async (_name, response) => {
|
||||
mocks.responses.push(response)
|
||||
const picker = observe(`no-retry-${response.ok ? '' : response.error.code}`)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(picker.entry().catalogRead).toBe('failed')
|
||||
await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS)
|
||||
expect(mocks.calls).toHaveLength(1)
|
||||
picker.stop()
|
||||
})
|
||||
|
||||
it('a refresh during backoff reads at once and drops the pending retry', async () => {
|
||||
mocks.responses.push(refusal('hostDisabled'), piCatalog)
|
||||
const picker = observe('refresh-during-backoff')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
picker.invalidate()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(mocks.calls).toHaveLength(2)
|
||||
expect(picker.entry().catalogRead).toBe('settled')
|
||||
await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS)
|
||||
expect(mocks.calls).toHaveLength(2)
|
||||
expect(picker.entry().catalogRead).toBe('settled')
|
||||
picker.stop()
|
||||
})
|
||||
|
||||
it('a reopen during backoff reads at once and drops the pending retry', async () => {
|
||||
mocks.responses.push(refusal('hostDisabled'), piCatalog)
|
||||
const first = observe('reopen-during-backoff')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(first.entry().catalogRead).toBe('pending')
|
||||
first.stop()
|
||||
const reopened = observe('reopen-during-backoff')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(mocks.calls).toHaveLength(2)
|
||||
expect(reopened.entry().catalogRead).toBe('settled')
|
||||
await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS)
|
||||
expect(mocks.calls).toHaveLength(2)
|
||||
reopened.stop()
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
agentPickerCatalogJobs,
|
||||
agentPickerCatalogFailure,
|
||||
AgentPickerCatalogRetries,
|
||||
updateAgentPickerCatalog,
|
||||
type AgentPickerCatalogJob
|
||||
} from './agent-picker-catalog-jobs'
|
||||
@@ -36,6 +37,7 @@ class AgentPickerAcquisition {
|
||||
private defaults: AgentPickerEntry[]
|
||||
private stopInvalidation?: () => void
|
||||
private physicalCatalogs = 0
|
||||
private retries = new AgentPickerCatalogRetries()
|
||||
|
||||
constructor(
|
||||
readonly key: string,
|
||||
@@ -120,6 +122,7 @@ class AgentPickerAcquisition {
|
||||
this.observers.delete(observer)
|
||||
if (!this.observers.size) {
|
||||
this.demand?.abort()
|
||||
this.retries.cancel()
|
||||
this.stopInvalidation?.()
|
||||
this.stopInvalidation = undefined
|
||||
this.queued.refresh.clear()
|
||||
@@ -228,8 +231,18 @@ class AgentPickerAcquisition {
|
||||
kind !== 'support'
|
||||
? this.readCatalog(entry, index, kind === 'refresh').catch((error: unknown) => {
|
||||
if (job === this.jobs[index] && !signal.aborted) {
|
||||
const failure = agentPickerCatalogFailure(error)
|
||||
const retrying =
|
||||
failure === 'retry' &&
|
||||
this.observers.size > 0 &&
|
||||
this.retries.schedule(job, () => {
|
||||
if (job === this.jobs[index] && !signal.aborted && this.observers.size) {
|
||||
this.queued.catalog.add(index)
|
||||
this.admit('catalog')
|
||||
}
|
||||
})
|
||||
this.entries = updateAgentPickerCatalog(this.entries, job, {
|
||||
catalogRead: agentPickerCatalogFailure(error)
|
||||
catalogRead: retrying ? 'pending' : failure === 'retry' ? 'failed' : failure
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
import { shallow } from 'zustand/shallow'
|
||||
import type { AgentPickerEntry } from '../../../../shared/agent-launch/picker/agent-launch-picker'
|
||||
import { readAgentSessionErrorRefusal } from '../../../../shared/agent-session/failures/agent-session-write-failure'
|
||||
import {
|
||||
isRecoverableRemoteRuntimeConnectionError,
|
||||
isRuntimeRpcQueueOverloadError,
|
||||
toRemoteRuntimeClientErrorLike
|
||||
} from '../../../../shared/remote-runtime/remote-runtime-client-error-classification'
|
||||
|
||||
export type AgentPickerCatalogJob = { index: number }
|
||||
export function agentPickerCatalogJobs(entries: AgentPickerEntry[]): AgentPickerCatalogJob[] {
|
||||
@@ -7,15 +13,39 @@ export function agentPickerCatalogJobs(entries: AgentPickerEntry[]): AgentPicker
|
||||
entry.surfaces.length && entry.catalogRead !== 'unsupported' ? [{ index }] : []
|
||||
)
|
||||
}
|
||||
export function agentPickerCatalogFailure(error: unknown): 'unsupported' | 'failed' {
|
||||
return typeof error === 'object' &&
|
||||
// Only contact lost or being re-established retries; a timeout or a busy client fails fast.
|
||||
function agentPickerCatalogReconnecting(error: unknown): boolean {
|
||||
const like = toRemoteRuntimeClientErrorLike(error)
|
||||
const message = like.message.toLowerCase()
|
||||
return (
|
||||
isRecoverableRemoteRuntimeConnectionError(like) &&
|
||||
!isRuntimeRpcQueueOverloadError(like) &&
|
||||
(like.code === undefined
|
||||
? !/timed out|limit reached/.test(message)
|
||||
: !['runtime_timeout', 'timeout', 'remote_runtime_busy'].includes(like.code))
|
||||
)
|
||||
}
|
||||
export function agentPickerCatalogFailure(error: unknown): 'unsupported' | 'failed' | 'retry' {
|
||||
// Why: hosts before 1.4.248 refuse a session-less catalog read until something starts their
|
||||
// chat host, so that refusal is retryable, not final.
|
||||
const refusal = readAgentSessionErrorRefusal(error)
|
||||
if (
|
||||
refusal?.code === 'structured_agent_session_unsupported' &&
|
||||
refusal.details?.reason === 'hostDisabled'
|
||||
) {
|
||||
return 'retry'
|
||||
}
|
||||
if (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error.code === 'method_not_found' ||
|
||||
error.code === 'structured_agent_session_unsupported' ||
|
||||
error.code === 'forbidden')
|
||||
? 'unsupported'
|
||||
: 'failed'
|
||||
) {
|
||||
return 'unsupported'
|
||||
}
|
||||
return agentPickerCatalogReconnecting(error) ? 'retry' : 'failed'
|
||||
}
|
||||
export function updateAgentPickerCatalog(
|
||||
entries: AgentPickerEntry[],
|
||||
@@ -28,3 +58,32 @@ export function updateAgentPickerCatalog(
|
||||
? entries
|
||||
: entries.map((row, index) => (index === job.index ? next : row))
|
||||
}
|
||||
|
||||
// Transient failures (an older host still starting its chat host, a dropped bridge) recover
|
||||
// within seconds; a few spaced retries keep an open picker from settling on built-in models.
|
||||
export const AGENT_PICKER_CATALOG_RETRY_DELAYS_MS = [1_000, 3_000, 10_000] as const
|
||||
|
||||
/** Bounded retries per job; a new job (reopen or invalidation) starts over. */
|
||||
export class AgentPickerCatalogRetries {
|
||||
private attempts = new WeakMap<AgentPickerCatalogJob, number>()
|
||||
private timers = new Set<ReturnType<typeof setTimeout>>()
|
||||
|
||||
schedule(job: AgentPickerCatalogJob, retry: () => void): boolean {
|
||||
const attempt = this.attempts.get(job) ?? 0
|
||||
if (attempt >= AGENT_PICKER_CATALOG_RETRY_DELAYS_MS.length) {
|
||||
return false
|
||||
}
|
||||
this.attempts.set(job, attempt + 1)
|
||||
const timer = setTimeout(() => {
|
||||
this.timers.delete(timer)
|
||||
retry()
|
||||
}, AGENT_PICKER_CATALOG_RETRY_DELAYS_MS[attempt])
|
||||
this.timers.add(timer)
|
||||
return true
|
||||
}
|
||||
|
||||
cancel() {
|
||||
this.timers.forEach(clearTimeout)
|
||||
this.timers.clear()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user