diff --git a/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts b/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts index fd210c5e7cd..d9b44299c4f 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts @@ -2,6 +2,16 @@ const generationByEnvironment = new Map() const MAX_TRACKED_ENVIRONMENTS = 512 let generationSequence = 0 let evictedGeneration = 0 +const retirementListeners = new Set<(environmentId: string) => void>() + +/** Streams opened under an earlier generation can no longer deliver (their frames are discarded), + * so whoever owns them must close them when the generation moves on. */ +export function onRuntimeEnvironmentTransportGenerationRetired( + listener: (environmentId: string) => void +): () => void { + retirementListeners.add(listener) + return () => retirementListeners.delete(listener) +} export function getRuntimeEnvironmentTransportGeneration(environmentId: string): number { return generationByEnvironment.get(environmentId) ?? evictedGeneration @@ -18,6 +28,9 @@ export function advanceRuntimeEnvironmentTransportGeneration(environmentId: stri evictedGeneration = Math.max(evictedGeneration, generationByEnvironment.get(oldest.value) ?? 0) generationByEnvironment.delete(oldest.value) } + for (const listener of retirementListeners) { + listener(environmentId) + } } export function _getRuntimeEnvironmentTransportGenerationCacheSize(): number { diff --git a/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts b/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts index ed19056d5f2..8e09a9237d2 100644 --- a/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts +++ b/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts @@ -1,4 +1,5 @@ import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' +import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -186,4 +187,45 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) expect(close).toHaveBeenCalled() }) + + // #39926: a rebuilt control connection advances the transport generation, after which every + // frame of an existing stream was discarded without a close, so a chat view sat on a silent stream. + it('closes and reports a stream whose transport generation was retired', async () => { + registerRuntimeEnvironmentHandlers(store as never) + const close = vi.fn() + subscribeRemoteRuntimeRequestMock.mockResolvedValue({ + requestId: 'stream-1', + close, + sendBinary: vi.fn() + }) + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const { environment } = await add(null, { name: 'desk', pairingCode: pairingCode() }) + const send = vi.fn() + const subscribe = handler< + { selector: string; method: string; params?: unknown; subscriptionId?: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + await subscribe( + { + sender: { id: 1, isDestroyed: () => false, send, on: vi.fn(), removeListener: vi.fn() } + }, + { + selector: 'desk', + method: 'agentSession.subscribe', + params: { sessionId: 'chat-1' }, + subscriptionId: 'chat-sub' + } + ) + + advanceRuntimeEnvironmentTransportGeneration(environment.id) + + expect(close).toHaveBeenCalled() + expect(send).toHaveBeenCalledWith('runtimeEnvironments:subscriptionEvent', { + subscriptionId: 'chat-sub', + type: 'close' + }) + }) }) diff --git a/src/main/ipc/runtime-environments/runtime-environments.ts b/src/main/ipc/runtime-environments/runtime-environments.ts index fb21fb8e798..cf205a1a3c1 100644 --- a/src/main/ipc/runtime-environments/runtime-environments.ts +++ b/src/main/ipc/runtime-environments/runtime-environments.ts @@ -29,7 +29,10 @@ import { setManagedOrcadPairingRepairedListener } from '../../managed-server/lif import { forgetManagedOrcadAutomaticFailures } from '../../managed-server/lifecycle/orcad-managed-tunnel' import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler' import { registerRuntimeEnvironmentRepairHandler } from './runtime-environment-repair-handler' -import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' +import { + advanceRuntimeEnvironmentTransportGeneration, + onRuntimeEnvironmentTransportGenerationRetired +} from './runtime-environment-transport-generation' import { resetSharedControlSupport } from './runtime-environment-shared-control-support' import { RUNTIME_ENVIRONMENT_HANDLER_CHANNELS } from './runtime-environment-handler-channels' import { registerOrcadRuntimeLifecycleHandlers } from './orcad-runtime-lifecycle-handlers' @@ -59,11 +62,17 @@ function closeSubscriptionsForEnvironment(environmentId: string): void { pending.close() } } + closeEstablishedSubscriptionsForEnvironment(environmentId) +} + +function closeEstablishedSubscriptionsForEnvironment(environmentId: string): number { + let closed = 0 // Removed runtimes must release terminal/browser sockets before renderer teardown. for (const [subscriptionId, subscription] of remoteRuntimeSubscriptions) { if (subscription.environmentId !== environmentId) { continue } + closed += 1 remoteRuntimeSubscriptions.delete(subscriptionId) // Isolate close and notification failures so neither strands this environment's sibling subscriptions. try { @@ -78,7 +87,20 @@ function closeSubscriptionsForEnvironment(environmentId: string): void { console.warn('[runtime-environments] subscription close notice failed:', error) } } + return closed } +// A retired generation's frames are discarded (runtime-environment-subscription-handlers.ts), so its +// streams must close and tell their renderers, which resubscribe from where they left off. Streams +// still being set up fail their own generation check instead. +onRuntimeEnvironmentTransportGenerationRetired((environmentId) => { + const closed = closeEstablishedSubscriptionsForEnvironment(environmentId) + if (closed > 0) { + console.warn(`[runtime-environments] transport retired; reopening ${closed} stream(s)`, { + environmentId + }) + } +}) + /** Returns once the environment's client-hosted browser pages have been released. */ export function invalidateRuntimeEnvironmentTransport(environmentId: string): Promise { // Same-ID re-pairing must retire every transport authenticated as the former peer. diff --git a/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts b/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts index 9dd5194b84f..b61ce30fa86 100644 --- a/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts +++ b/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts @@ -10,6 +10,8 @@ export type AgentSessionSubscribeInput = { id: string sessionId: string emit: (event: AgentSessionSubscribeEvent) => void + /** Ends the subscriber's stream once the host stops delivering to it, so the client resubscribes. */ + end?: (cause: unknown) => void cursor?: AgentJournalCursor /** Bound each frame to the phone bridge’s limit; RPC enables this for every phone. */ boundedFrames?: boolean @@ -26,6 +28,7 @@ export type AgentSessionSubscriberEmit = (event: AgentSessionSubscribeEvent) => export type Subscriber = { id: string emit: AgentSessionSubscriberEmit + end?: (cause: unknown) => void cursor: AgentJournalCursor fence: number forkMarkerView?: string diff --git a/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts index 99e49d59fdd..0f27517ec4b 100644 --- a/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts @@ -307,8 +307,10 @@ export class AgentSessionSubscribers { if (options?.historyView !== undefined && this.isActive(subscriber)) { subscriber.historyView = options.historyView } - } catch { + } catch (error) { + // A dropped subscriber gets no more frames, so end its stream or the client never resubscribes. this.drop(subscriber) + subscriber.end?.(error) } } diff --git a/src/main/native-chat/agent-session-wire/publication/subscribers/testing/structured-agent-session-subscriber-failed-frame.test.ts b/src/main/native-chat/agent-session-wire/publication/subscribers/testing/structured-agent-session-subscriber-failed-frame.test.ts new file mode 100644 index 00000000000..0ad0cc7dc0e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/publication/subscribers/testing/structured-agent-session-subscriber-failed-frame.test.ts @@ -0,0 +1,66 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../../../../shared/agent-session/journal/agent-session-journal-types' +import { claudeProviderHandle } from '../../../../../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' +import type { AgentSessionSubscribeEvent } from '../../../../../../shared/agent-session/agent-session-wire' +import { createTrackedJournalOpener } from '../../../../agent-session-journal/store-lifecycle/journal-host-database-test-support' +import { AgentSessionSubscribers } from '../structured-agent-session-subscribers' + +const SESSION = 'failed-frame-session' +let root: string +const journals = createTrackedJournalOpener() + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-subscriber-failed-frame-')) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +// #39926: a frame that failed to send dropped the subscriber but left its stream open, so the chat +// view sat on a healthy-looking, silent stream until the tab was reopened. +it('ends the stream of a subscriber dropped after a frame fails to send', async () => { + const journal = await journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'claude', + providerHandle: claudeProviderHandle('provider-1', null) + }, + stateDirectory: join(root, 'journal') + }) + const subscribers = new AgentSessionSubscribers() + const received: AgentSessionSubscribeEvent['type'][] = [] + const end = vi.fn() + subscribers.open({ + id: 'chat-pane', + sessionId: SESSION, + journal, + fence: 1, + end, + emit: (event) => { + if (event.type === 'batch') { + throw new Error('frame could not be sent') + } + received.push(event.type) + } + }) + expect(received).toEqual(['snapshot']) + + await journal.appendItem( + { provider: 'orca', clientMessageId: 'reply' }, + { kind: 'status', text: 'reply' }, + { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + subscribers.publish(SESSION, journal) + + expect(subscribers.subscriberCountForTests(SESSION)).toBe(0) + expect(end).toHaveBeenCalledTimes(1) +}) diff --git a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts index ffc5121bb0b..4e493fdce88 100644 --- a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts +++ b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts @@ -15,7 +15,7 @@ export function bindStructuredAgentSessionStream( ctx: RpcContext, subscriptionId: string, onClose: () => void -): { isClosed: () => boolean } { +): { isClosed: () => boolean; close: () => void } { let closed = false let releaseTransportSubscription = (): void => {} const onTransportAbort = (): void => releaseTransportSubscription() @@ -40,7 +40,7 @@ export function bindStructuredAgentSessionStream( if (ctx.signal?.aborted) { onTransportAbort() } - return { isClosed: () => closed } + return { isClosed: () => closed, close: () => releaseTransportSubscription() } } export const STRUCTURED_AGENT_SESSION_STATUS_METHODS = [ diff --git a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts index a09545a50bd..8e3046fb5cc 100644 --- a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts +++ b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts @@ -49,6 +49,18 @@ export const STRUCTURED_AGENT_SESSION_SUBSCRIBE_METHOD = defineStreamingMethod({ ctx ) ), + end: (cause) => { + if (stream.isClosed()) { + return + } + console.warn('[agent-session] ending stream after a failed frame', params.sessionId, cause) + try { + emitFrame({ type: 'end' }) + } catch { + // The transport is gone; releasing the stream below is all that is left to do. + } + stream.close() + }, ...(params.cursor ? { cursor: params.cursor } : {}), snapshotCatchUp: clientSupportsSnapshotCatchUp(ctx), // Every phone's app, whatever its version, ends a stream over a frame its bridge cannot carry. diff --git a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts index 90656512b8c..1890ac6ed5e 100644 --- a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts +++ b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts @@ -80,3 +80,37 @@ it('drops the subscriber a client unsubscribes, and keeps its sibling (U-02)', a await vi.waitFor(() => expect(kept.length).toBeGreaterThan(1)) expect(ended.slice(before).every((frame) => JSON.stringify(frame).includes('"end"'))).toBe(true) }) + +// #39926: a frame the transport fails to send must end the stream, never leave it open and silent. +// The opening catch-up already sends a batch after the snapshot, so this covers a failure inside open. +it('ends the stream once when a frame fails to send', async () => { + await foundRestTestChat(rig) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const frames: RpcResponse[] = [] + await dispatcher.dispatchStreaming( + { + id: 'frame-a', + authToken: 'token', + method: 'agentSession.subscribe', + params: { sessionId: SESSION } + }, + (raw) => { + if (raw.includes('"type":"batch"')) { + throw new Error('frame could not be sent') + } + frames.push(JSON.parse(raw)) + }, + CLIENT + ) + + expect(subscriberCount()).toBe(0) + expect(frames.map((frame) => JSON.stringify(frame).match(/"type":"(\w+)"/)?.[1])).toEqual([ + 'snapshot', + 'end' + ]) + + // Later publications reach no one and end nothing twice. + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + await rig.host.send(CALLER, restTestSend('after the failed frame', fence)) + expect(frames.filter((frame) => JSON.stringify(frame).includes('"type":"end"'))).toHaveLength(1) +}) diff --git a/src/shared/agent-session/structured-agent-session-live-read.ts b/src/shared/agent-session/structured-agent-session-live-read.ts index 728aab8ee2a..99ea4e34bc8 100644 --- a/src/shared/agent-session/structured-agent-session-live-read.ts +++ b/src/shared/agent-session/structured-agent-session-live-read.ts @@ -163,9 +163,9 @@ export function startStructuredAgentSessionLiveRead(args: { return } clearUnattachedReadGrace() - // Reset on delivered frames; local subscribe can resolve before the host refuses opening. + // Note delivered frames; local subscribe can resolve before the host refuses opening. if (event.type !== 'end') { - reconnectScheduler.reset() + reconnectScheduler.noteDelivered() } if (event.type === 'snapshot' || event.type === 'reset') { requestReader.invalidate() diff --git a/src/shared/agent-session/structured-agent-session-reconnect.test.ts b/src/shared/agent-session/structured-agent-session-reconnect.test.ts new file mode 100644 index 00000000000..e27511d3818 --- /dev/null +++ b/src/shared/agent-session/structured-agent-session-reconnect.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + createReconnectScheduler, + RECONNECT_STABLE_STREAM_MS +} from './structured-agent-session-reconnect' + +describe('stream reconnect backoff', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + function scheduler() { + const reconnect = vi.fn() + const retry = createReconnectScheduler({ shouldStop: () => false, reconnect }) + /** Run one reconnect cycle and report how long it waited. */ + const cycle = (): number => { + const start = Date.now() + retry.schedule() + vi.advanceTimersToNextTimer() + return Date.now() - start + } + return { retry, cycle } + } + + // #39926: a host that ends a stream right after its first frames must not cause a 750 ms storm. + it('keeps growing the wait for a stream that ends right after delivering', () => { + const { retry, cycle } = scheduler() + retry.noteDelivered() + expect(cycle()).toBe(750) + retry.noteDelivered() + expect(cycle()).toBe(1_500) + retry.noteDelivered() + expect(cycle()).toBe(3_000) + }) + + it('retries soon again after a stream that stayed up', () => { + const { retry, cycle } = scheduler() + expect(cycle()).toBe(750) + expect(cycle()).toBe(1_500) + retry.noteDelivered() + vi.advanceTimersByTime(RECONNECT_STABLE_STREAM_MS) + expect(cycle()).toBe(750) + }) +}) diff --git a/src/shared/agent-session/structured-agent-session-reconnect.ts b/src/shared/agent-session/structured-agent-session-reconnect.ts index 8e9ff528a15..102e315c295 100644 --- a/src/shared/agent-session/structured-agent-session-reconnect.ts +++ b/src/shared/agent-session/structured-agent-session-reconnect.ts @@ -1,5 +1,7 @@ const RECONNECT_FIRST_DELAY_MS = 750 const RECONNECT_MAX_DELAY_MS = 30_000 +/** A stream must stay up this long before its next failure counts as fresh rather than a repeat. */ +export const RECONNECT_STABLE_STREAM_MS = 10_000 /** Each reconnect waits twice the last, up to the cap, until a read delivers. */ export function createReconnectScheduler(args: { @@ -8,11 +10,16 @@ export function createReconnectScheduler(args: { }) { let timer: ReturnType | null = null let nextDelay = RECONNECT_FIRST_DELAY_MS + let deliveringSince: number | null = null return { schedule(): void { if (args.shouldStop() || timer) { return } + if (deliveringSince !== null && Date.now() - deliveringSince >= RECONNECT_STABLE_STREAM_MS) { + nextDelay = RECONNECT_FIRST_DELAY_MS + } + deliveringSince = null const delay = nextDelay nextDelay = Math.min(delay * 2, RECONNECT_MAX_DELAY_MS) timer = setTimeout(() => { @@ -22,7 +29,12 @@ export function createReconnectScheduler(args: { } }, delay) }, - /** A read delivered, so the next failure is retried soon again. */ + /** A stream delivered. Only one that then stays up resets the wait, so a stream the host ends + * right after its first frames cannot retry every 750 ms forever. */ + noteDelivered(): void { + deliveringSince ??= Date.now() + }, + /** A one-shot read delivered, so the next failure is retried soon again. */ reset(): void { nextDelay = RECONNECT_FIRST_DELAY_MS },