diff --git a/src/renderer/src/components/agent/agent-launch-picker-acquisition.test.ts b/src/renderer/src/components/agent/agent-launch-picker-acquisition.test.ts new file mode 100644 index 00000000000..aa902400b91 --- /dev/null +++ b/src/renderer/src/components/agent/agent-launch-picker-acquisition.test.ts @@ -0,0 +1,204 @@ +// @vitest-environment happy-dom +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RuntimeRpcResponse } from '../../../../shared/remote-runtime/rpc/runtime-rpc-envelope' +import type { AgentPickerEntry } from '../../../../shared/agent-launch/picker/agent-launch-picker' +import { agentPickerModels } from '../../../../shared/agent-launch/picker/agent-launch-picker-catalog' +import { unwrapRuntimeRpcResult } from '@/runtime/rpc/calls/runtime-rpc-result' +import { observeAgentPickerAcquisition } from './agent-launch-picker-acquisition' +import { AGENT_PICKER_CATALOG_RETRY_DELAYS_MS } from './agent-picker-catalog-jobs' +import { agentPickerModelCatalogReads } from './agent-launch-picker-host-reads' + +const mocks = vi.hoisted(() => { + const responses: RuntimeRpcResponse[] = [] + const calls: { method: string; params: unknown }[] = [] + return { responses, calls } +}) + +// The remote runtime transport: each call answers with the next wire envelope the host sent. +vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({ + callRuntimeRpc: async (_target: unknown, method: string, params: unknown) => { + mocks.calls.push({ method, params }) + const response = mocks.responses.shift() + if (!response) { + throw new Error('unexpected call') + } + return unwrapRuntimeRpcResult(response) + } +})) + +const meta = { runtimeId: 'wsl-runtime' } +/** What a host before 1.4.248 answers a session-less catalog read before its chat host is built. */ +function refusal(reason: 'hostDisabled' | 'clientCapabilityMissing'): RuntimeRpcResponse { + return { + id: 'r', + ok: false, + error: { + code: 'structured_agent_session_unsupported', + message: 'structured_agent_session_unsupported', + data: { refusal: { code: 'structured_agent_session_unsupported', details: { reason } } } + }, + _meta: meta + } +} +function failure(code: string, message: string): RuntimeRpcResponse { + return { id: 'r', ok: false, error: { code, message }, _meta: meta } +} +const piCatalog: RuntimeRpcResponse = { + id: 'r', + ok: true, + result: { + origin: 'probe', + fetchedAt: 1, + models: [ + { id: 'gateway/model-a', label: 'Model A', isDefault: true, efforts: [] }, + { id: 'gateway/model-b', label: 'Model B', isDefault: false, efforts: [] } + ] + }, + _meta: meta +} + +const pi: AgentPickerEntry = { + agent: 'pi', + label: 'Pi', + rows: [], + surfaces: ['chat', 'terminal'], + defaultSurface: 'chat', + terminal: {}, + structured: {}, + catalog: { origin: 'unknown' } +} + +function observe(key: string) { + let latest: AgentPickerEntry[] = [] + let invalidate = () => {} + const reads = agentPickerModelCatalogReads({ + target: { kind: 'environment', environmentId: 'wsl-env' }, + worktree: 'id:repo::/home/me/project' + }) + const stop = observeAgentPickerAcquisition( + key, + [pi], + { + catalog: reads.catalog, + refreshOnInvalidation: () => true, + subscribeInvalidation: (listener) => { + invalidate = listener + return () => {} + }, + support: async () => undefined + }, + { selected: () => undefined, publish: (entries) => (latest = entries) } + ) + return { stop, entry: () => latest[0], invalidate: () => invalidate() } +} +const BACKOFF_TOTAL_MS = AGENT_PICKER_CATALOG_RETRY_DELAYS_MS.reduce((sum, delay) => sum + delay, 0) + +describe('agent picker catalog acquisition against a remote runtime', () => { + beforeEach(() => { + vi.useFakeTimers() + mocks.responses.length = 0 + mocks.calls.length = 0 + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('retries an older host that refused the read before its chat host was built', async () => { + mocks.responses.push(refusal('hostDisabled'), piCatalog) + const picker = observe('host-not-built') + await vi.advanceTimersByTimeAsync(0) + // Still loading rather than settled on Pi's empty built-in list. + expect(picker.entry().catalogRead).toBe('pending') + + await vi.advanceTimersByTimeAsync(AGENT_PICKER_CATALOG_RETRY_DELAYS_MS[0]) + expect(mocks.calls.map(({ method }) => method)).toEqual([ + 'agentSession.modelCatalog', + 'agentSession.modelCatalog' + ]) + expect(picker.entry().catalogRead).toBe('settled') + expect(agentPickerModels(picker.entry(), 'chat').map(({ id }) => id)).toEqual( + expect.arrayContaining(['gateway/model-a', 'gateway/model-b']) + ) + picker.stop() + }) + + it('gives up after its bounded retries', async () => { + mocks.responses.push(...Array.from({ length: 4 }, () => refusal('hostDisabled'))) + const picker = observe('host-never-built') + await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS) + expect(mocks.calls).toHaveLength(4) + expect(picker.entry().catalogRead).toBe('failed') + picker.stop() + }) + + it('does not retry a client the host will never serve', async () => { + mocks.responses.push(refusal('clientCapabilityMissing')) + const picker = observe('client-unsupported') + await vi.advanceTimersByTimeAsync(20_000) + expect(mocks.calls).toHaveLength(1) + expect(picker.entry().catalogRead).toBe('unsupported') + picker.stop() + }) + + it('stops retrying once the picker closes', async () => { + mocks.responses.push(refusal('hostDisabled')) + const picker = observe('closed') + await vi.advanceTimersByTimeAsync(0) + picker.stop() + await vi.advanceTimersByTimeAsync(20_000) + expect(mocks.calls).toHaveLength(1) + }) + + it('retries a read the dropped connection lost', async () => { + mocks.responses.push(failure('reconnecting', 'Remote Orca runtime is reconnecting'), piCatalog) + const picker = observe('reconnecting') + await vi.advanceTimersByTimeAsync(0) + expect(picker.entry().catalogRead).toBe('pending') + await vi.advanceTimersByTimeAsync(AGENT_PICKER_CATALOG_RETRY_DELAYS_MS[0]) + expect(mocks.calls).toHaveLength(2) + expect(picker.entry().catalogRead).toBe('settled') + picker.stop() + }) + + it.each([ + ['a generic host error', failure('runtime_error', 'listing crashed')], + ['a timeout', failure('runtime_timeout', 'Timed out waiting for the remote Orca runtime')] + ])('shows %s at once without retrying', async (_name, response) => { + mocks.responses.push(response) + const picker = observe(`no-retry-${response.ok ? '' : response.error.code}`) + await vi.advanceTimersByTimeAsync(0) + expect(picker.entry().catalogRead).toBe('failed') + await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS) + expect(mocks.calls).toHaveLength(1) + picker.stop() + }) + + it('a refresh during backoff reads at once and drops the pending retry', async () => { + mocks.responses.push(refusal('hostDisabled'), piCatalog) + const picker = observe('refresh-during-backoff') + await vi.advanceTimersByTimeAsync(0) + picker.invalidate() + await vi.advanceTimersByTimeAsync(0) + expect(mocks.calls).toHaveLength(2) + expect(picker.entry().catalogRead).toBe('settled') + await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS) + expect(mocks.calls).toHaveLength(2) + expect(picker.entry().catalogRead).toBe('settled') + picker.stop() + }) + + it('a reopen during backoff reads at once and drops the pending retry', async () => { + mocks.responses.push(refusal('hostDisabled'), piCatalog) + const first = observe('reopen-during-backoff') + await vi.advanceTimersByTimeAsync(0) + expect(first.entry().catalogRead).toBe('pending') + first.stop() + const reopened = observe('reopen-during-backoff') + await vi.advanceTimersByTimeAsync(0) + expect(mocks.calls).toHaveLength(2) + expect(reopened.entry().catalogRead).toBe('settled') + await vi.advanceTimersByTimeAsync(BACKOFF_TOTAL_MS) + expect(mocks.calls).toHaveLength(2) + reopened.stop() + }) +}) diff --git a/src/renderer/src/components/agent/agent-launch-picker-acquisition.ts b/src/renderer/src/components/agent/agent-launch-picker-acquisition.ts index bbc08d9c020..106611441f5 100644 --- a/src/renderer/src/components/agent/agent-launch-picker-acquisition.ts +++ b/src/renderer/src/components/agent/agent-launch-picker-acquisition.ts @@ -1,6 +1,7 @@ import { agentPickerCatalogJobs, agentPickerCatalogFailure, + AgentPickerCatalogRetries, updateAgentPickerCatalog, type AgentPickerCatalogJob } from './agent-picker-catalog-jobs' @@ -36,6 +37,7 @@ class AgentPickerAcquisition { private defaults: AgentPickerEntry[] private stopInvalidation?: () => void private physicalCatalogs = 0 + private retries = new AgentPickerCatalogRetries() constructor( readonly key: string, @@ -120,6 +122,7 @@ class AgentPickerAcquisition { this.observers.delete(observer) if (!this.observers.size) { this.demand?.abort() + this.retries.cancel() this.stopInvalidation?.() this.stopInvalidation = undefined this.queued.refresh.clear() @@ -228,8 +231,18 @@ class AgentPickerAcquisition { kind !== 'support' ? this.readCatalog(entry, index, kind === 'refresh').catch((error: unknown) => { if (job === this.jobs[index] && !signal.aborted) { + const failure = agentPickerCatalogFailure(error) + const retrying = + failure === 'retry' && + this.observers.size > 0 && + this.retries.schedule(job, () => { + if (job === this.jobs[index] && !signal.aborted && this.observers.size) { + this.queued.catalog.add(index) + this.admit('catalog') + } + }) this.entries = updateAgentPickerCatalog(this.entries, job, { - catalogRead: agentPickerCatalogFailure(error) + catalogRead: retrying ? 'pending' : failure === 'retry' ? 'failed' : failure }) } }) diff --git a/src/renderer/src/components/agent/agent-picker-catalog-jobs.ts b/src/renderer/src/components/agent/agent-picker-catalog-jobs.ts index 39ac2a7614b..bc9f1704ec1 100644 --- a/src/renderer/src/components/agent/agent-picker-catalog-jobs.ts +++ b/src/renderer/src/components/agent/agent-picker-catalog-jobs.ts @@ -1,5 +1,11 @@ import { shallow } from 'zustand/shallow' import type { AgentPickerEntry } from '../../../../shared/agent-launch/picker/agent-launch-picker' +import { readAgentSessionErrorRefusal } from '../../../../shared/agent-session/failures/agent-session-write-failure' +import { + isRecoverableRemoteRuntimeConnectionError, + isRuntimeRpcQueueOverloadError, + toRemoteRuntimeClientErrorLike +} from '../../../../shared/remote-runtime/remote-runtime-client-error-classification' export type AgentPickerCatalogJob = { index: number } export function agentPickerCatalogJobs(entries: AgentPickerEntry[]): AgentPickerCatalogJob[] { @@ -7,15 +13,39 @@ export function agentPickerCatalogJobs(entries: AgentPickerEntry[]): AgentPicker entry.surfaces.length && entry.catalogRead !== 'unsupported' ? [{ index }] : [] ) } -export function agentPickerCatalogFailure(error: unknown): 'unsupported' | 'failed' { - return typeof error === 'object' && +// Only contact lost or being re-established retries; a timeout or a busy client fails fast. +function agentPickerCatalogReconnecting(error: unknown): boolean { + const like = toRemoteRuntimeClientErrorLike(error) + const message = like.message.toLowerCase() + return ( + isRecoverableRemoteRuntimeConnectionError(like) && + !isRuntimeRpcQueueOverloadError(like) && + (like.code === undefined + ? !/timed out|limit reached/.test(message) + : !['runtime_timeout', 'timeout', 'remote_runtime_busy'].includes(like.code)) + ) +} +export function agentPickerCatalogFailure(error: unknown): 'unsupported' | 'failed' | 'retry' { + // Why: hosts before 1.4.248 refuse a session-less catalog read until something starts their + // chat host, so that refusal is retryable, not final. + const refusal = readAgentSessionErrorRefusal(error) + if ( + refusal?.code === 'structured_agent_session_unsupported' && + refusal.details?.reason === 'hostDisabled' + ) { + return 'retry' + } + if ( + typeof error === 'object' && error !== null && 'code' in error && (error.code === 'method_not_found' || error.code === 'structured_agent_session_unsupported' || error.code === 'forbidden') - ? 'unsupported' - : 'failed' + ) { + return 'unsupported' + } + return agentPickerCatalogReconnecting(error) ? 'retry' : 'failed' } export function updateAgentPickerCatalog( entries: AgentPickerEntry[], @@ -28,3 +58,32 @@ export function updateAgentPickerCatalog( ? entries : entries.map((row, index) => (index === job.index ? next : row)) } + +// Transient failures (an older host still starting its chat host, a dropped bridge) recover +// within seconds; a few spaced retries keep an open picker from settling on built-in models. +export const AGENT_PICKER_CATALOG_RETRY_DELAYS_MS = [1_000, 3_000, 10_000] as const + +/** Bounded retries per job; a new job (reopen or invalidation) starts over. */ +export class AgentPickerCatalogRetries { + private attempts = new WeakMap() + private timers = new Set>() + + schedule(job: AgentPickerCatalogJob, retry: () => void): boolean { + const attempt = this.attempts.get(job) ?? 0 + if (attempt >= AGENT_PICKER_CATALOG_RETRY_DELAYS_MS.length) { + return false + } + this.attempts.set(job, attempt + 1) + const timer = setTimeout(() => { + this.timers.delete(timer) + retry() + }, AGENT_PICKER_CATALOG_RETRY_DELAYS_MS[attempt]) + this.timers.add(timer) + return true + } + + cancel() { + this.timers.forEach(clearTimeout) + this.timers.clear() + } +}