From f60b84dcbaeca345437ea79aa39867bbfd8f2c60 Mon Sep 17 00:00:00 2001 From: Neil Date: Mon, 9 Nov 2026 23:03:44 -0800 Subject: [PATCH] fix(native-chat): end chat streams that stop delivering instead of leaving them silent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A remote chat view could sit on a stream that looked healthy but carried no frames, so replies never rendered and sends stayed on "Sending…" until the tab was reopened. Two paths left a stream open after delivery had stopped: - The host dropped a chat subscriber whose frame failed to send, but never ended its stream or logged why. It now logs the failure and ends the stream once, so the client resubscribes from its cursor. - The desktop discards frames from a retired transport generation, but a rebuilt control connection advanced the generation without closing the streams opened under the old one. Retiring a generation now closes those streams, tells their windows (which resubscribe) and logs how many. The client's reconnect wait now resets only after a stream has stayed up for 10 s, so a host that ends a stream right after its first frames cannot drive a resubscribe every 750 ms. Refs #39926 --- ...untime-environment-transport-generation.ts | 13 ++++ ...nvironments-subscription-lifecycle.test.ts | 42 ++++++++++++ .../runtime-environments.ts | 24 ++++++- .../agent-session-subscribe-input.ts | 3 + .../structured-agent-session-subscribers.ts | 4 +- ...nt-session-subscriber-failed-frame.test.ts | 66 +++++++++++++++++++ .../structured-agent-session-status-stream.ts | 4 +- ...ructured-agent-session-subscribe-method.ts | 12 ++++ ...ructured-agent-session-unsubscribe.test.ts | 34 ++++++++++ .../structured-agent-session-live-read.ts | 4 +- ...structured-agent-session-reconnect.test.ts | 43 ++++++++++++ .../structured-agent-session-reconnect.ts | 14 +++- 12 files changed, 256 insertions(+), 7 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/publication/subscribers/testing/structured-agent-session-subscriber-failed-frame.test.ts create mode 100644 src/shared/agent-session/structured-agent-session-reconnect.test.ts diff --git a/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts b/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts index fd210c5e7cd..d9b44299c4f 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-transport-generation.ts @@ -2,6 +2,16 @@ const generationByEnvironment = new Map() const MAX_TRACKED_ENVIRONMENTS = 512 let generationSequence = 0 let evictedGeneration = 0 +const retirementListeners = new Set<(environmentId: string) => void>() + +/** Streams opened under an earlier generation can no longer deliver (their frames are discarded), + * so whoever owns them must close them when the generation moves on. */ +export function onRuntimeEnvironmentTransportGenerationRetired( + listener: (environmentId: string) => void +): () => void { + retirementListeners.add(listener) + return () => retirementListeners.delete(listener) +} export function getRuntimeEnvironmentTransportGeneration(environmentId: string): number { return generationByEnvironment.get(environmentId) ?? evictedGeneration @@ -18,6 +28,9 @@ export function advanceRuntimeEnvironmentTransportGeneration(environmentId: stri evictedGeneration = Math.max(evictedGeneration, generationByEnvironment.get(oldest.value) ?? 0) generationByEnvironment.delete(oldest.value) } + for (const listener of retirementListeners) { + listener(environmentId) + } } export function _getRuntimeEnvironmentTransportGenerationCacheSize(): number { diff --git a/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts b/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts index ed19056d5f2..8e09a9237d2 100644 --- a/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts +++ b/src/main/ipc/runtime-environments/runtime-environments-subscription-lifecycle.test.ts @@ -1,4 +1,5 @@ import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' +import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -186,4 +187,45 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) expect(close).toHaveBeenCalled() }) + + // #39926: a rebuilt control connection advances the transport generation, after which every + // frame of an existing stream was discarded without a close, so a chat view sat on a silent stream. + it('closes and reports a stream whose transport generation was retired', async () => { + registerRuntimeEnvironmentHandlers(store as never) + const close = vi.fn() + subscribeRemoteRuntimeRequestMock.mockResolvedValue({ + requestId: 'stream-1', + close, + sendBinary: vi.fn() + }) + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const { environment } = await add(null, { name: 'desk', pairingCode: pairingCode() }) + const send = vi.fn() + const subscribe = handler< + { selector: string; method: string; params?: unknown; subscriptionId?: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + await subscribe( + { + sender: { id: 1, isDestroyed: () => false, send, on: vi.fn(), removeListener: vi.fn() } + }, + { + selector: 'desk', + method: 'agentSession.subscribe', + params: { sessionId: 'chat-1' }, + subscriptionId: 'chat-sub' + } + ) + + advanceRuntimeEnvironmentTransportGeneration(environment.id) + + expect(close).toHaveBeenCalled() + expect(send).toHaveBeenCalledWith('runtimeEnvironments:subscriptionEvent', { + subscriptionId: 'chat-sub', + type: 'close' + }) + }) }) diff --git a/src/main/ipc/runtime-environments/runtime-environments.ts b/src/main/ipc/runtime-environments/runtime-environments.ts index fb21fb8e798..cf205a1a3c1 100644 --- a/src/main/ipc/runtime-environments/runtime-environments.ts +++ b/src/main/ipc/runtime-environments/runtime-environments.ts @@ -29,7 +29,10 @@ import { setManagedOrcadPairingRepairedListener } from '../../managed-server/lif import { forgetManagedOrcadAutomaticFailures } from '../../managed-server/lifecycle/orcad-managed-tunnel' import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler' import { registerRuntimeEnvironmentRepairHandler } from './runtime-environment-repair-handler' -import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' +import { + advanceRuntimeEnvironmentTransportGeneration, + onRuntimeEnvironmentTransportGenerationRetired +} from './runtime-environment-transport-generation' import { resetSharedControlSupport } from './runtime-environment-shared-control-support' import { RUNTIME_ENVIRONMENT_HANDLER_CHANNELS } from './runtime-environment-handler-channels' import { registerOrcadRuntimeLifecycleHandlers } from './orcad-runtime-lifecycle-handlers' @@ -59,11 +62,17 @@ function closeSubscriptionsForEnvironment(environmentId: string): void { pending.close() } } + closeEstablishedSubscriptionsForEnvironment(environmentId) +} + +function closeEstablishedSubscriptionsForEnvironment(environmentId: string): number { + let closed = 0 // Removed runtimes must release terminal/browser sockets before renderer teardown. for (const [subscriptionId, subscription] of remoteRuntimeSubscriptions) { if (subscription.environmentId !== environmentId) { continue } + closed += 1 remoteRuntimeSubscriptions.delete(subscriptionId) // Isolate close and notification failures so neither strands this environment's sibling subscriptions. try { @@ -78,7 +87,20 @@ function closeSubscriptionsForEnvironment(environmentId: string): void { console.warn('[runtime-environments] subscription close notice failed:', error) } } + return closed } +// A retired generation's frames are discarded (runtime-environment-subscription-handlers.ts), so its +// streams must close and tell their renderers, which resubscribe from where they left off. Streams +// still being set up fail their own generation check instead. +onRuntimeEnvironmentTransportGenerationRetired((environmentId) => { + const closed = closeEstablishedSubscriptionsForEnvironment(environmentId) + if (closed > 0) { + console.warn(`[runtime-environments] transport retired; reopening ${closed} stream(s)`, { + environmentId + }) + } +}) + /** Returns once the environment's client-hosted browser pages have been released. */ export function invalidateRuntimeEnvironmentTransport(environmentId: string): Promise { // Same-ID re-pairing must retire every transport authenticated as the former peer. diff --git a/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts b/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts index 9dd5194b84f..b61ce30fa86 100644 --- a/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts +++ b/src/main/native-chat/agent-session-wire/publication/subscribers/agent-session-subscribe-input.ts @@ -10,6 +10,8 @@ export type AgentSessionSubscribeInput = { id: string sessionId: string emit: (event: AgentSessionSubscribeEvent) => void + /** Ends the subscriber's stream once the host stops delivering to it, so the client resubscribes. */ + end?: (cause: unknown) => void cursor?: AgentJournalCursor /** Bound each frame to the phone bridge’s limit; RPC enables this for every phone. */ boundedFrames?: boolean @@ -26,6 +28,7 @@ export type AgentSessionSubscriberEmit = (event: AgentSessionSubscribeEvent) => export type Subscriber = { id: string emit: AgentSessionSubscriberEmit + end?: (cause: unknown) => void cursor: AgentJournalCursor fence: number forkMarkerView?: string diff --git a/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts index 99e49d59fdd..0f27517ec4b 100644 --- a/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/publication/subscribers/structured-agent-session-subscribers.ts @@ -307,8 +307,10 @@ export class AgentSessionSubscribers { if (options?.historyView !== undefined && this.isActive(subscriber)) { subscriber.historyView = options.historyView } - } catch { + } catch (error) { + // A dropped subscriber gets no more frames, so end its stream or the client never resubscribes. this.drop(subscriber) + subscriber.end?.(error) } } diff --git a/src/main/native-chat/agent-session-wire/publication/subscribers/testing/structured-agent-session-subscriber-failed-frame.test.ts b/src/main/native-chat/agent-session-wire/publication/subscribers/testing/structured-agent-session-subscriber-failed-frame.test.ts new file mode 100644 index 00000000000..0ad0cc7dc0e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/publication/subscribers/testing/structured-agent-session-subscriber-failed-frame.test.ts @@ -0,0 +1,66 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../../../../shared/agent-session/journal/agent-session-journal-types' +import { claudeProviderHandle } from '../../../../../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' +import type { AgentSessionSubscribeEvent } from '../../../../../../shared/agent-session/agent-session-wire' +import { createTrackedJournalOpener } from '../../../../agent-session-journal/store-lifecycle/journal-host-database-test-support' +import { AgentSessionSubscribers } from '../structured-agent-session-subscribers' + +const SESSION = 'failed-frame-session' +let root: string +const journals = createTrackedJournalOpener() + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-subscriber-failed-frame-')) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) + +afterEach(async () => { + vi.restoreAllMocks() + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +// #39926: a frame that failed to send dropped the subscriber but left its stream open, so the chat +// view sat on a healthy-looking, silent stream until the tab was reopened. +it('ends the stream of a subscriber dropped after a frame fails to send', async () => { + const journal = await journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'claude', + providerHandle: claudeProviderHandle('provider-1', null) + }, + stateDirectory: join(root, 'journal') + }) + const subscribers = new AgentSessionSubscribers() + const received: AgentSessionSubscribeEvent['type'][] = [] + const end = vi.fn() + subscribers.open({ + id: 'chat-pane', + sessionId: SESSION, + journal, + fence: 1, + end, + emit: (event) => { + if (event.type === 'batch') { + throw new Error('frame could not be sent') + } + received.push(event.type) + } + }) + expect(received).toEqual(['snapshot']) + + await journal.appendItem( + { provider: 'orca', clientMessageId: 'reply' }, + { kind: 'status', text: 'reply' }, + { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + subscribers.publish(SESSION, journal) + + expect(subscribers.subscriberCountForTests(SESSION)).toBe(0) + expect(end).toHaveBeenCalledTimes(1) +}) diff --git a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts index ffc5121bb0b..4e493fdce88 100644 --- a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts +++ b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-status-stream.ts @@ -15,7 +15,7 @@ export function bindStructuredAgentSessionStream( ctx: RpcContext, subscriptionId: string, onClose: () => void -): { isClosed: () => boolean } { +): { isClosed: () => boolean; close: () => void } { let closed = false let releaseTransportSubscription = (): void => {} const onTransportAbort = (): void => releaseTransportSubscription() @@ -40,7 +40,7 @@ export function bindStructuredAgentSessionStream( if (ctx.signal?.aborted) { onTransportAbort() } - return { isClosed: () => closed } + return { isClosed: () => closed, close: () => releaseTransportSubscription() } } export const STRUCTURED_AGENT_SESSION_STATUS_METHODS = [ diff --git a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts index a09545a50bd..8e3046fb5cc 100644 --- a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts +++ b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-subscribe-method.ts @@ -49,6 +49,18 @@ export const STRUCTURED_AGENT_SESSION_SUBSCRIBE_METHOD = defineStreamingMethod({ ctx ) ), + end: (cause) => { + if (stream.isClosed()) { + return + } + console.warn('[agent-session] ending stream after a failed frame', params.sessionId, cause) + try { + emitFrame({ type: 'end' }) + } catch { + // The transport is gone; releasing the stream below is all that is left to do. + } + stream.close() + }, ...(params.cursor ? { cursor: params.cursor } : {}), snapshotCatchUp: clientSupportsSnapshotCatchUp(ctx), // Every phone's app, whatever its version, ends a stream over a frame its bridge cannot carry. diff --git a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts index 90656512b8c..1890ac6ed5e 100644 --- a/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts +++ b/src/main/runtime/rpc/methods/structured-sessions/subscriptions/structured-agent-session-unsubscribe.test.ts @@ -80,3 +80,37 @@ it('drops the subscriber a client unsubscribes, and keeps its sibling (U-02)', a await vi.waitFor(() => expect(kept.length).toBeGreaterThan(1)) expect(ended.slice(before).every((frame) => JSON.stringify(frame).includes('"end"'))).toBe(true) }) + +// #39926: a frame the transport fails to send must end the stream, never leave it open and silent. +// The opening catch-up already sends a batch after the snapshot, so this covers a failure inside open. +it('ends the stream once when a frame fails to send', async () => { + await foundRestTestChat(rig) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const frames: RpcResponse[] = [] + await dispatcher.dispatchStreaming( + { + id: 'frame-a', + authToken: 'token', + method: 'agentSession.subscribe', + params: { sessionId: SESSION } + }, + (raw) => { + if (raw.includes('"type":"batch"')) { + throw new Error('frame could not be sent') + } + frames.push(JSON.parse(raw)) + }, + CLIENT + ) + + expect(subscriberCount()).toBe(0) + expect(frames.map((frame) => JSON.stringify(frame).match(/"type":"(\w+)"/)?.[1])).toEqual([ + 'snapshot', + 'end' + ]) + + // Later publications reach no one and end nothing twice. + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + await rig.host.send(CALLER, restTestSend('after the failed frame', fence)) + expect(frames.filter((frame) => JSON.stringify(frame).includes('"type":"end"'))).toHaveLength(1) +}) diff --git a/src/shared/agent-session/structured-agent-session-live-read.ts b/src/shared/agent-session/structured-agent-session-live-read.ts index 728aab8ee2a..99ea4e34bc8 100644 --- a/src/shared/agent-session/structured-agent-session-live-read.ts +++ b/src/shared/agent-session/structured-agent-session-live-read.ts @@ -163,9 +163,9 @@ export function startStructuredAgentSessionLiveRead(args: { return } clearUnattachedReadGrace() - // Reset on delivered frames; local subscribe can resolve before the host refuses opening. + // Note delivered frames; local subscribe can resolve before the host refuses opening. if (event.type !== 'end') { - reconnectScheduler.reset() + reconnectScheduler.noteDelivered() } if (event.type === 'snapshot' || event.type === 'reset') { requestReader.invalidate() diff --git a/src/shared/agent-session/structured-agent-session-reconnect.test.ts b/src/shared/agent-session/structured-agent-session-reconnect.test.ts new file mode 100644 index 00000000000..e27511d3818 --- /dev/null +++ b/src/shared/agent-session/structured-agent-session-reconnect.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + createReconnectScheduler, + RECONNECT_STABLE_STREAM_MS +} from './structured-agent-session-reconnect' + +describe('stream reconnect backoff', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + function scheduler() { + const reconnect = vi.fn() + const retry = createReconnectScheduler({ shouldStop: () => false, reconnect }) + /** Run one reconnect cycle and report how long it waited. */ + const cycle = (): number => { + const start = Date.now() + retry.schedule() + vi.advanceTimersToNextTimer() + return Date.now() - start + } + return { retry, cycle } + } + + // #39926: a host that ends a stream right after its first frames must not cause a 750 ms storm. + it('keeps growing the wait for a stream that ends right after delivering', () => { + const { retry, cycle } = scheduler() + retry.noteDelivered() + expect(cycle()).toBe(750) + retry.noteDelivered() + expect(cycle()).toBe(1_500) + retry.noteDelivered() + expect(cycle()).toBe(3_000) + }) + + it('retries soon again after a stream that stayed up', () => { + const { retry, cycle } = scheduler() + expect(cycle()).toBe(750) + expect(cycle()).toBe(1_500) + retry.noteDelivered() + vi.advanceTimersByTime(RECONNECT_STABLE_STREAM_MS) + expect(cycle()).toBe(750) + }) +}) diff --git a/src/shared/agent-session/structured-agent-session-reconnect.ts b/src/shared/agent-session/structured-agent-session-reconnect.ts index 8e9ff528a15..102e315c295 100644 --- a/src/shared/agent-session/structured-agent-session-reconnect.ts +++ b/src/shared/agent-session/structured-agent-session-reconnect.ts @@ -1,5 +1,7 @@ const RECONNECT_FIRST_DELAY_MS = 750 const RECONNECT_MAX_DELAY_MS = 30_000 +/** A stream must stay up this long before its next failure counts as fresh rather than a repeat. */ +export const RECONNECT_STABLE_STREAM_MS = 10_000 /** Each reconnect waits twice the last, up to the cap, until a read delivers. */ export function createReconnectScheduler(args: { @@ -8,11 +10,16 @@ export function createReconnectScheduler(args: { }) { let timer: ReturnType | null = null let nextDelay = RECONNECT_FIRST_DELAY_MS + let deliveringSince: number | null = null return { schedule(): void { if (args.shouldStop() || timer) { return } + if (deliveringSince !== null && Date.now() - deliveringSince >= RECONNECT_STABLE_STREAM_MS) { + nextDelay = RECONNECT_FIRST_DELAY_MS + } + deliveringSince = null const delay = nextDelay nextDelay = Math.min(delay * 2, RECONNECT_MAX_DELAY_MS) timer = setTimeout(() => { @@ -22,7 +29,12 @@ export function createReconnectScheduler(args: { } }, delay) }, - /** A read delivered, so the next failure is retried soon again. */ + /** A stream delivered. Only one that then stays up resets the wait, so a stream the host ends + * right after its first frames cannot retry every 750 ms forever. */ + noteDelivered(): void { + deliveringSince ??= Date.now() + }, + /** A one-shot read delivered, so the next failure is retried soon again. */ reset(): void { nextDelay = RECONNECT_FIRST_DELAY_MS },