mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
fix(native-chat): end chat streams that stop delivering instead of leaving them silent
A remote chat view could sit on a stream that looked healthy but carried no frames, so replies never rendered and sends stayed on "Sending…" until the tab was reopened. Two paths left a stream open after delivery had stopped: - The host dropped a chat subscriber whose frame failed to send, but never ended its stream or logged why. It now logs the failure and ends the stream once, so the client resubscribes from its cursor. - The desktop discards frames from a retired transport generation, but a rebuilt control connection advanced the generation without closing the streams opened under the old one. Retiring a generation now closes those streams, tells their windows (which resubscribe) and logs how many. The client's reconnect wait now resets only after a stream has stayed up for 10 s, so a host that ends a stream right after its first frames cannot drive a resubscribe every 750 ms. Refs #39926
This commit is contained in:
@@ -2,6 +2,16 @@ const generationByEnvironment = new Map<string, number>()
|
||||
const MAX_TRACKED_ENVIRONMENTS = 512
|
||||
let generationSequence = 0
|
||||
let evictedGeneration = 0
|
||||
const retirementListeners = new Set<(environmentId: string) => void>()
|
||||
|
||||
/** Streams opened under an earlier generation can no longer deliver (their frames are discarded),
|
||||
* so whoever owns them must close them when the generation moves on. */
|
||||
export function onRuntimeEnvironmentTransportGenerationRetired(
|
||||
listener: (environmentId: string) => void
|
||||
): () => void {
|
||||
retirementListeners.add(listener)
|
||||
return () => retirementListeners.delete(listener)
|
||||
}
|
||||
|
||||
export function getRuntimeEnvironmentTransportGeneration(environmentId: string): number {
|
||||
return generationByEnvironment.get(environmentId) ?? evictedGeneration
|
||||
@@ -18,6 +28,9 @@ export function advanceRuntimeEnvironmentTransportGeneration(environmentId: stri
|
||||
evictedGeneration = Math.max(evictedGeneration, generationByEnvironment.get(oldest.value) ?? 0)
|
||||
generationByEnvironment.delete(oldest.value)
|
||||
}
|
||||
for (const listener of retirementListeners) {
|
||||
listener(environmentId)
|
||||
}
|
||||
}
|
||||
|
||||
export function _getRuntimeEnvironmentTransportGenerationCacheSize(): number {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
|
||||
import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -186,4 +187,45 @@ describe('registerRuntimeEnvironmentHandlers', () => {
|
||||
})
|
||||
expect(close).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// #39926: a rebuilt control connection advances the transport generation, after which every
|
||||
// frame of an existing stream was discarded without a close, so a chat view sat on a silent stream.
|
||||
it('closes and reports a stream whose transport generation was retired', async () => {
|
||||
registerRuntimeEnvironmentHandlers(store as never)
|
||||
const close = vi.fn()
|
||||
subscribeRemoteRuntimeRequestMock.mockResolvedValue({
|
||||
requestId: 'stream-1',
|
||||
close,
|
||||
sendBinary: vi.fn()
|
||||
})
|
||||
const add = handler<
|
||||
{ name: string; pairingCode: string },
|
||||
{ environment: { id: string; name: string } }
|
||||
>('runtimeEnvironments:addFromPairingCode')
|
||||
const { environment } = await add(null, { name: 'desk', pairingCode: pairingCode() })
|
||||
const send = vi.fn()
|
||||
const subscribe = handler<
|
||||
{ selector: string; method: string; params?: unknown; subscriptionId?: string },
|
||||
{ subscriptionId: string; requestId: string }
|
||||
>('runtimeEnvironments:subscribe')
|
||||
await subscribe(
|
||||
{
|
||||
sender: { id: 1, isDestroyed: () => false, send, on: vi.fn(), removeListener: vi.fn() }
|
||||
},
|
||||
{
|
||||
selector: 'desk',
|
||||
method: 'agentSession.subscribe',
|
||||
params: { sessionId: 'chat-1' },
|
||||
subscriptionId: 'chat-sub'
|
||||
}
|
||||
)
|
||||
|
||||
advanceRuntimeEnvironmentTransportGeneration(environment.id)
|
||||
|
||||
expect(close).toHaveBeenCalled()
|
||||
expect(send).toHaveBeenCalledWith('runtimeEnvironments:subscriptionEvent', {
|
||||
subscriptionId: 'chat-sub',
|
||||
type: 'close'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -29,7 +29,10 @@ import { setManagedOrcadPairingRepairedListener } from '../../managed-server/lif
|
||||
import { forgetManagedOrcadAutomaticFailures } from '../../managed-server/lifecycle/orcad-managed-tunnel'
|
||||
import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler'
|
||||
import { registerRuntimeEnvironmentRepairHandler } from './runtime-environment-repair-handler'
|
||||
import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation'
|
||||
import {
|
||||
advanceRuntimeEnvironmentTransportGeneration,
|
||||
onRuntimeEnvironmentTransportGenerationRetired
|
||||
} from './runtime-environment-transport-generation'
|
||||
import { resetSharedControlSupport } from './runtime-environment-shared-control-support'
|
||||
import { RUNTIME_ENVIRONMENT_HANDLER_CHANNELS } from './runtime-environment-handler-channels'
|
||||
import { registerOrcadRuntimeLifecycleHandlers } from './orcad-runtime-lifecycle-handlers'
|
||||
@@ -59,11 +62,17 @@ function closeSubscriptionsForEnvironment(environmentId: string): void {
|
||||
pending.close()
|
||||
}
|
||||
}
|
||||
closeEstablishedSubscriptionsForEnvironment(environmentId)
|
||||
}
|
||||
|
||||
function closeEstablishedSubscriptionsForEnvironment(environmentId: string): number {
|
||||
let closed = 0
|
||||
// Removed runtimes must release terminal/browser sockets before renderer teardown.
|
||||
for (const [subscriptionId, subscription] of remoteRuntimeSubscriptions) {
|
||||
if (subscription.environmentId !== environmentId) {
|
||||
continue
|
||||
}
|
||||
closed += 1
|
||||
remoteRuntimeSubscriptions.delete(subscriptionId)
|
||||
// Isolate close and notification failures so neither strands this environment's sibling subscriptions.
|
||||
try {
|
||||
@@ -78,7 +87,20 @@ function closeSubscriptionsForEnvironment(environmentId: string): void {
|
||||
console.warn('[runtime-environments] subscription close notice failed:', error)
|
||||
}
|
||||
}
|
||||
return closed
|
||||
}
|
||||
// A retired generation's frames are discarded (runtime-environment-subscription-handlers.ts), so its
|
||||
// streams must close and tell their renderers, which resubscribe from where they left off. Streams
|
||||
// still being set up fail their own generation check instead.
|
||||
onRuntimeEnvironmentTransportGenerationRetired((environmentId) => {
|
||||
const closed = closeEstablishedSubscriptionsForEnvironment(environmentId)
|
||||
if (closed > 0) {
|
||||
console.warn(`[runtime-environments] transport retired; reopening ${closed} stream(s)`, {
|
||||
environmentId
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
/** Returns once the environment's client-hosted browser pages have been released. */
|
||||
export function invalidateRuntimeEnvironmentTransport(environmentId: string): Promise<void> {
|
||||
// Same-ID re-pairing must retire every transport authenticated as the former peer.
|
||||
|
||||
+3
@@ -10,6 +10,8 @@ export type AgentSessionSubscribeInput = {
|
||||
id: string
|
||||
sessionId: string
|
||||
emit: (event: AgentSessionSubscribeEvent) => void
|
||||
/** Ends the subscriber's stream once the host stops delivering to it, so the client resubscribes. */
|
||||
end?: (cause: unknown) => void
|
||||
cursor?: AgentJournalCursor
|
||||
/** Bound each frame to the phone bridge’s limit; RPC enables this for every phone. */
|
||||
boundedFrames?: boolean
|
||||
@@ -26,6 +28,7 @@ export type AgentSessionSubscriberEmit = (event: AgentSessionSubscribeEvent) =>
|
||||
export type Subscriber = {
|
||||
id: string
|
||||
emit: AgentSessionSubscriberEmit
|
||||
end?: (cause: unknown) => void
|
||||
cursor: AgentJournalCursor
|
||||
fence: number
|
||||
forkMarkerView?: string
|
||||
|
||||
+3
-1
@@ -307,8 +307,10 @@ export class AgentSessionSubscribers {
|
||||
if (options?.historyView !== undefined && this.isActive(subscriber)) {
|
||||
subscriber.historyView = options.historyView
|
||||
}
|
||||
} catch {
|
||||
} catch (error) {
|
||||
// A dropped subscriber gets no more frames, so end its stream or the client never resubscribes.
|
||||
this.drop(subscriber)
|
||||
subscriber.end?.(error)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import { claudeProviderHandle } from '../../../../../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding'
|
||||
import type { AgentSessionSubscribeEvent } from '../../../../../../shared/agent-session/agent-session-wire'
|
||||
import { createTrackedJournalOpener } from '../../../../agent-session-journal/store-lifecycle/journal-host-database-test-support'
|
||||
import { AgentSessionSubscribers } from '../structured-agent-session-subscribers'
|
||||
|
||||
const SESSION = 'failed-frame-session'
|
||||
let root: string
|
||||
const journals = createTrackedJournalOpener()
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'orca-subscriber-failed-frame-'))
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
await journals.closeAll()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
// #39926: a frame that failed to send dropped the subscriber but left its stream open, so the chat
|
||||
// view sat on a healthy-looking, silent stream until the tab was reopened.
|
||||
it('ends the stream of a subscriber dropped after a frame fails to send', async () => {
|
||||
const journal = await journals.open({
|
||||
identity: {
|
||||
sessionId: SESSION,
|
||||
workspaceId: 'workspace-1',
|
||||
hostId: 'local',
|
||||
agent: 'claude',
|
||||
providerHandle: claudeProviderHandle('provider-1', null)
|
||||
},
|
||||
stateDirectory: join(root, 'journal')
|
||||
})
|
||||
const subscribers = new AgentSessionSubscribers()
|
||||
const received: AgentSessionSubscribeEvent['type'][] = []
|
||||
const end = vi.fn()
|
||||
subscribers.open({
|
||||
id: 'chat-pane',
|
||||
sessionId: SESSION,
|
||||
journal,
|
||||
fence: 1,
|
||||
end,
|
||||
emit: (event) => {
|
||||
if (event.type === 'batch') {
|
||||
throw new Error('frame could not be sent')
|
||||
}
|
||||
received.push(event.type)
|
||||
}
|
||||
})
|
||||
expect(received).toEqual(['snapshot'])
|
||||
|
||||
await journal.appendItem(
|
||||
{ provider: 'orca', clientMessageId: 'reply' },
|
||||
{ kind: 'status', text: 'reply' },
|
||||
{ fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE }
|
||||
)
|
||||
subscribers.publish(SESSION, journal)
|
||||
|
||||
expect(subscribers.subscriberCountForTests(SESSION)).toBe(0)
|
||||
expect(end).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
+2
-2
@@ -15,7 +15,7 @@ export function bindStructuredAgentSessionStream(
|
||||
ctx: RpcContext,
|
||||
subscriptionId: string,
|
||||
onClose: () => void
|
||||
): { isClosed: () => boolean } {
|
||||
): { isClosed: () => boolean; close: () => void } {
|
||||
let closed = false
|
||||
let releaseTransportSubscription = (): void => {}
|
||||
const onTransportAbort = (): void => releaseTransportSubscription()
|
||||
@@ -40,7 +40,7 @@ export function bindStructuredAgentSessionStream(
|
||||
if (ctx.signal?.aborted) {
|
||||
onTransportAbort()
|
||||
}
|
||||
return { isClosed: () => closed }
|
||||
return { isClosed: () => closed, close: () => releaseTransportSubscription() }
|
||||
}
|
||||
|
||||
export const STRUCTURED_AGENT_SESSION_STATUS_METHODS = [
|
||||
|
||||
+12
@@ -49,6 +49,18 @@ export const STRUCTURED_AGENT_SESSION_SUBSCRIBE_METHOD = defineStreamingMethod({
|
||||
ctx
|
||||
)
|
||||
),
|
||||
end: (cause) => {
|
||||
if (stream.isClosed()) {
|
||||
return
|
||||
}
|
||||
console.warn('[agent-session] ending stream after a failed frame', params.sessionId, cause)
|
||||
try {
|
||||
emitFrame({ type: 'end' })
|
||||
} catch {
|
||||
// The transport is gone; releasing the stream below is all that is left to do.
|
||||
}
|
||||
stream.close()
|
||||
},
|
||||
...(params.cursor ? { cursor: params.cursor } : {}),
|
||||
snapshotCatchUp: clientSupportsSnapshotCatchUp(ctx),
|
||||
// Every phone's app, whatever its version, ends a stream over a frame its bridge cannot carry.
|
||||
|
||||
+34
@@ -80,3 +80,37 @@ it('drops the subscriber a client unsubscribes, and keeps its sibling (U-02)', a
|
||||
await vi.waitFor(() => expect(kept.length).toBeGreaterThan(1))
|
||||
expect(ended.slice(before).every((frame) => JSON.stringify(frame).includes('"end"'))).toBe(true)
|
||||
})
|
||||
|
||||
// #39926: a frame the transport fails to send must end the stream, never leave it open and silent.
|
||||
// The opening catch-up already sends a batch after the snapshot, so this covers a failure inside open.
|
||||
it('ends the stream once when a frame fails to send', async () => {
|
||||
await foundRestTestChat(rig)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const frames: RpcResponse[] = []
|
||||
await dispatcher.dispatchStreaming(
|
||||
{
|
||||
id: 'frame-a',
|
||||
authToken: 'token',
|
||||
method: 'agentSession.subscribe',
|
||||
params: { sessionId: SESSION }
|
||||
},
|
||||
(raw) => {
|
||||
if (raw.includes('"type":"batch"')) {
|
||||
throw new Error('frame could not be sent')
|
||||
}
|
||||
frames.push(JSON.parse(raw))
|
||||
},
|
||||
CLIENT
|
||||
)
|
||||
|
||||
expect(subscriberCount()).toBe(0)
|
||||
expect(frames.map((frame) => JSON.stringify(frame).match(/"type":"(\w+)"/)?.[1])).toEqual([
|
||||
'snapshot',
|
||||
'end'
|
||||
])
|
||||
|
||||
// Later publications reach no one and end nothing twice.
|
||||
const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence
|
||||
await rig.host.send(CALLER, restTestSend('after the failed frame', fence))
|
||||
expect(frames.filter((frame) => JSON.stringify(frame).includes('"type":"end"'))).toHaveLength(1)
|
||||
})
|
||||
|
||||
@@ -163,9 +163,9 @@ export function startStructuredAgentSessionLiveRead(args: {
|
||||
return
|
||||
}
|
||||
clearUnattachedReadGrace()
|
||||
// Reset on delivered frames; local subscribe can resolve before the host refuses opening.
|
||||
// Note delivered frames; local subscribe can resolve before the host refuses opening.
|
||||
if (event.type !== 'end') {
|
||||
reconnectScheduler.reset()
|
||||
reconnectScheduler.noteDelivered()
|
||||
}
|
||||
if (event.type === 'snapshot' || event.type === 'reset') {
|
||||
requestReader.invalidate()
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
createReconnectScheduler,
|
||||
RECONNECT_STABLE_STREAM_MS
|
||||
} from './structured-agent-session-reconnect'
|
||||
|
||||
describe('stream reconnect backoff', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
function scheduler() {
|
||||
const reconnect = vi.fn()
|
||||
const retry = createReconnectScheduler({ shouldStop: () => false, reconnect })
|
||||
/** Run one reconnect cycle and report how long it waited. */
|
||||
const cycle = (): number => {
|
||||
const start = Date.now()
|
||||
retry.schedule()
|
||||
vi.advanceTimersToNextTimer()
|
||||
return Date.now() - start
|
||||
}
|
||||
return { retry, cycle }
|
||||
}
|
||||
|
||||
// #39926: a host that ends a stream right after its first frames must not cause a 750 ms storm.
|
||||
it('keeps growing the wait for a stream that ends right after delivering', () => {
|
||||
const { retry, cycle } = scheduler()
|
||||
retry.noteDelivered()
|
||||
expect(cycle()).toBe(750)
|
||||
retry.noteDelivered()
|
||||
expect(cycle()).toBe(1_500)
|
||||
retry.noteDelivered()
|
||||
expect(cycle()).toBe(3_000)
|
||||
})
|
||||
|
||||
it('retries soon again after a stream that stayed up', () => {
|
||||
const { retry, cycle } = scheduler()
|
||||
expect(cycle()).toBe(750)
|
||||
expect(cycle()).toBe(1_500)
|
||||
retry.noteDelivered()
|
||||
vi.advanceTimersByTime(RECONNECT_STABLE_STREAM_MS)
|
||||
expect(cycle()).toBe(750)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,7 @@
|
||||
const RECONNECT_FIRST_DELAY_MS = 750
|
||||
const RECONNECT_MAX_DELAY_MS = 30_000
|
||||
/** A stream must stay up this long before its next failure counts as fresh rather than a repeat. */
|
||||
export const RECONNECT_STABLE_STREAM_MS = 10_000
|
||||
|
||||
/** Each reconnect waits twice the last, up to the cap, until a read delivers. */
|
||||
export function createReconnectScheduler(args: {
|
||||
@@ -8,11 +10,16 @@ export function createReconnectScheduler(args: {
|
||||
}) {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
let nextDelay = RECONNECT_FIRST_DELAY_MS
|
||||
let deliveringSince: number | null = null
|
||||
return {
|
||||
schedule(): void {
|
||||
if (args.shouldStop() || timer) {
|
||||
return
|
||||
}
|
||||
if (deliveringSince !== null && Date.now() - deliveringSince >= RECONNECT_STABLE_STREAM_MS) {
|
||||
nextDelay = RECONNECT_FIRST_DELAY_MS
|
||||
}
|
||||
deliveringSince = null
|
||||
const delay = nextDelay
|
||||
nextDelay = Math.min(delay * 2, RECONNECT_MAX_DELAY_MS)
|
||||
timer = setTimeout(() => {
|
||||
@@ -22,7 +29,12 @@ export function createReconnectScheduler(args: {
|
||||
}
|
||||
}, delay)
|
||||
},
|
||||
/** A read delivered, so the next failure is retried soon again. */
|
||||
/** A stream delivered. Only one that then stays up resets the wait, so a stream the host ends
|
||||
* right after its first frames cannot retry every 750 ms forever. */
|
||||
noteDelivered(): void {
|
||||
deliveringSince ??= Date.now()
|
||||
},
|
||||
/** A one-shot read delivered, so the next failure is retried soon again. */
|
||||
reset(): void {
|
||||
nextDelay = RECONNECT_FIRST_DELAY_MS
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user