fix(cloud): N-02 full depth only when the desktop has no local copy (historyDepth hint, 2-min per-pane backstop); reconnects and resync keep the 1000-row cut

This commit is contained in:
Neil
2026-11-11 13:24:53 -08:00
parent a7b4937fa0
commit f81e500d40
11 changed files with 238 additions and 16 deletions
@@ -31,7 +31,10 @@ export const TerminalMultiplexSubscribeFrame = TerminalHandle.extend({
// Initial cadence, so a resubscribe resumes coalescing without a follow-up frame.
outputCadence: z.enum(['focused', 'background']).optional(),
// Name the client input sequence domain across stream replacement so replay deduplicates.
inputSessionId: z.string().min(1).max(128).optional()
inputSessionId: z.string().min(1).max(128).optional(),
// N-02: the desktop holds no local copy of this pane's history (fresh mount, app reopen), so the
// first snapshot carries a local terminal's depth. Absent, every reconnect gets the 1000-row cut.
historyDepth: z.literal('full').optional()
})
export const TerminalMultiplexLegacyAckFrame = z
@@ -0,0 +1,30 @@
// N-02 backstop: a desktop asks for a full-depth snapshot (5000 rows / 2 MB) only when it holds no
// local copy of the pane, but a client that asks on every reconnect would turn a flaky link into a
// stream of 2 MB snapshots. So each pane gets at most one full-depth snapshot per interval; the rest
// get the 1000-row cut, which is what a reconnecting pane needs to fill its gap.
export const FULL_DEPTH_SNAPSHOT_INTERVAL_MS = 2 * 60_000
// Prune once the map holds this many panes; entries older than the interval no longer limit anything.
const PRUNE_AT_SIZE = 1024
const lastFullDepthAtByPtyId = new Map<string, number>()
/** True when this pane may have a full-depth snapshot now, and records it. */
export function admitFullDepthSnapshot(ptyId: string, now: number = Date.now()): boolean {
const last = lastFullDepthAtByPtyId.get(ptyId)
if (last !== undefined && now - last < FULL_DEPTH_SNAPSHOT_INTERVAL_MS) {
return false
}
if (lastFullDepthAtByPtyId.size >= PRUNE_AT_SIZE) {
for (const [id, at] of lastFullDepthAtByPtyId) {
if (now - at >= FULL_DEPTH_SNAPSHOT_INTERVAL_MS) {
lastFullDepthAtByPtyId.delete(id)
}
}
}
lastFullDepthAtByPtyId.set(ptyId, now)
return true
}
export function resetFullDepthSnapshotLimitForTests(): void {
lastFullDepthAtByPtyId.clear()
}
@@ -4,6 +4,7 @@ import {
serializeBudgetedMobileSnapshot
} from './terminal-snapshot-publication'
import { getOutputAfterSnapshotSeq } from './terminal-stream-replay'
import { admitFullDepthSnapshot } from './terminal-full-depth-snapshot-limit'
import { isMovedPaneHistorySessionId } from '../../../../../shared/orcad-migration/orcad-migration-terminal-history'
import {
DESKTOP_SNAPSHOT_BYTE_BUDGET,
@@ -42,14 +43,17 @@ export async function publishMultiplexInitialSnapshot(
const forcedInitialSnapshotTruncated =
process.env.ORCA_E2E_FORCE_REMOTE_TERMINAL_INITIAL_SNAPSHOT_TRUNCATED === '1'
// A WSL-moved pane replays its whole carried history once; the first view must hold all of it.
// A desktop keeps a local terminal's depth (N-02: a cloud pane reattaching after an app restart
// lost everything above the phone-sized 1000 rows); phones keep the phone-sized cut.
// A desktop with no local copy of the pane (fresh mount, app reopen) asks for a local terminal's
// depth (N-02), at most once per pane per interval. Every other subscribe (phones, reconnects)
// gets the 1000-row cut: a deep snapshot per reconnect would flood a flaky link.
const fullDepth =
!isMobile && request.historyDepth === 'full' && admitFullDepthSnapshot(ptyId)
const firstRows = isMovedPaneHistorySessionId(ptyId)
? MOVED_PANE_SUBSCRIBE_SCROLLBACK_ROWS
: isMobile
? MOBILE_SUBSCRIBE_SCROLLBACK_ROWS
: DESKTOP_SUBSCRIBE_SCROLLBACK_ROWS
const rawBudgetBytes = isMobile ? MOBILE_SNAPSHOT_BYTE_BUDGET : DESKTOP_SNAPSHOT_BYTE_BUDGET
: fullDepth
? DESKTOP_SUBSCRIBE_SCROLLBACK_ROWS
: MOBILE_SUBSCRIBE_SCROLLBACK_ROWS
const rawBudgetBytes = fullDepth ? DESKTOP_SNAPSHOT_BYTE_BUDGET : MOBILE_SNAPSHOT_BYTE_BUDGET
let read = await runtime.readTerminal(request.terminal)
let serialized = await serializeBudgetedMobileSnapshot(
runtime,
@@ -338,10 +338,9 @@ describe('terminal multiplex RPC', () => {
).toMatchObject({
requestId: 7
})
// Desktop initial snapshots carry history, not just the screen (#20158), at a local
// terminal's depth so a reopen keeps it (N-02); overflow recovery stays at 1000.
// Desktop initial snapshots carry history, not just the screen (#20158).
expect(runtime.serializeTerminalBuffer).toHaveBeenCalledWith('pty-1', {
scrollbackRows: 5000
scrollbackRows: 1000
})
expect(runtime.serializeAuthoritativeTerminalBuffer).toHaveBeenLastCalledWith('pty-1', {
scrollbackRows: 5000
@@ -0,0 +1,68 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
FULL_DEPTH_SNAPSHOT_INTERVAL_MS,
resetFullDepthSnapshotLimitForTests
} from './methods/terminal/terminal-full-depth-snapshot-limit'
import {
sendDesktopMultiplexSubscribe,
startDesktopMultiplexSubscribe
} from './terminal-multiplex-test-harness'
const DESKTOP_CAPS = { ackOutput: 1, desktopViewportClaims: 1 } as const
/** One connection's subscribe; returns the scrollback depth its initial snapshot asked the host for. */
async function subscribeDepth(extra: Record<string, unknown> = {}): Promise<number | undefined> {
const serializeTerminalBuffer = vi.fn(
async (_ptyId: string, _opts?: { scrollbackRows?: number }) => ({
data: 'history\r\nscreen',
cols: 120,
rows: 40,
seq: 1
})
)
const harness = startDesktopMultiplexSubscribe({ serializeTerminalBuffer })
await vi.waitFor(() =>
expect(harness.messages.some((m) => JSON.parse(m).result?.type === 'ready')).toBe(true)
)
sendDesktopMultiplexSubscribe(harness.handlers, { ...DESKTOP_CAPS }, extra)
await vi.waitFor(() => expect(serializeTerminalBuffer).toHaveBeenCalled())
const depth = serializeTerminalBuffer.mock.calls[0]?.[1]?.scrollbackRows
harness.registry.cleanupSubscription('terminal-multiplex:conn-desktop-first-paint')
await harness.dispatchPromise
return depth
}
// N-02: full depth only when the desktop has no local copy of the pane; reconnects stay small.
describe('desktop subscribe history depth', () => {
beforeEach(() => {
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(new Date('2026-11-11T21:00:00Z'))
resetFullDepthSnapshotLimitForTests()
})
afterEach(() => {
vi.useRealTimers()
})
it('an app reopen (no local copy) gets a local terminal depth', async () => {
expect(await subscribeDepth({ historyDepth: 'full' })).toBe(5000)
})
it('a reconnect storm gets the 1000-row cut every time', async () => {
const depths: Array<number | undefined> = []
for (let i = 0; i < 6; i++) {
depths.push(await subscribeDepth())
}
expect(depths).toEqual([1000, 1000, 1000, 1000, 1000, 1000])
})
it('a client that asks for full depth on every reconnect gets it once per pane per interval', async () => {
// The harness's waits move the fake clock a little per subscribe, so the edges keep a 1 s margin.
const start = Date.now()
expect(await subscribeDepth({ historyDepth: 'full' })).toBe(5000)
expect(await subscribeDepth({ historyDepth: 'full' })).toBe(1000)
vi.setSystemTime(start + FULL_DEPTH_SNAPSHOT_INTERVAL_MS - 1000)
expect(await subscribeDepth({ historyDepth: 'full' })).toBe(1000)
vi.setSystemTime(start + FULL_DEPTH_SNAPSHOT_INTERVAL_MS + 1000)
expect(await subscribeDepth({ historyDepth: 'full' })).toBe(5000)
})
})
@@ -128,7 +128,8 @@ export function startDesktopMultiplexSubscribe(
export function sendDesktopMultiplexSubscribe(
handlers: Map<number, (frame: NonNullable<ReturnType<typeof decodeTerminalStreamFrame>>) => void>,
capabilities: Record<string, 1> = { ackOutput: 1, desktopViewportClaims: 1 }
capabilities: Record<string, 1> = { ackOutput: 1, desktopViewportClaims: 1 },
extra: Record<string, unknown> = {}
) {
handlers.get(0)?.(
decodeTerminalStreamFrame(
@@ -141,7 +142,8 @@ export function sendDesktopMultiplexSubscribe(
terminal: 'terminal-1',
client: { id: 'desktop-1', type: 'desktop' },
capabilities,
viewport: { cols: 120, rows: 40 }
viewport: { cols: 120, rows: 40 },
...extra
})
})
)!
@@ -0,0 +1,106 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const appStore = vi.hoisted(() => ({
getState: vi.fn(() => {
throw new Error('These transport fixtures must not read app-store ownership')
})
}))
vi.mock('@/store/index', () => ({ useAppStore: { getState: appStore.getState } }))
afterEach(() => {
expect(appStore.getState).not.toHaveBeenCalled()
})
import { createRemoteRuntimeTransportMocks } from '../remote/remote-runtime-pty-transport-test-harness'
import type { MultiplexSubscriptionCallbacks } from '../remote/remote-runtime-pty-transport-stream-fixtures'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
const {
runtimeSubscribe,
subscriptionSendBinary,
latestSubscribePayload,
emitSnapshot,
subscribeFrameCount,
resetRemoteRuntimeTransport
} = createRemoteRuntimeTransportMocks({
getCallbacks: () => subscriptionCallbacks,
setCallbacks: (callbacks) => {
subscriptionCallbacks = callbacks
},
getResolvedPaneHandle: () => resolvedPaneHandle,
setResolvedPaneHandle: (handle) => {
resolvedPaneHandle = handle
}
})
beforeEach(() => {
resetRemoteRuntimeTransport()
runtimeSubscribe.mockImplementation(
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
subscriptionCallbacks = callbacks
queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } }))
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
})
function acknowledgeSubscribe(streamId: number): void {
subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'subscribed', streamId } })
}
function dropLink(): void {
subscriptionCallbacks?.onError?.({
code: 'remote_runtime_unavailable',
message: 'Remote runtime connection closed.'
})
}
async function connectedTransport() {
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
await transport.connect({ url: '', callbacks: {} })
await vi.waitFor(() => expect(latestSubscribePayload()).toBeDefined())
return transport
}
// N-02: only a pane with no local copy of the host's history asks for full depth.
describe('remote pane subscribe history depth', () => {
it('a fresh pane (mount or app reopen) asks for full depth', async () => {
const transport = await connectedTransport()
expect(latestSubscribePayload().historyDepth).toBe('full')
transport.destroy?.()
})
it('reconnects after the history landed fill the gap from the small cut', async () => {
const transport = await connectedTransport()
const first = latestSubscribePayload().streamId
acknowledgeSubscribe(first)
emitSnapshot(first, 'prompt$ ')
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
for (let drop = 2; drop <= 4; drop++) {
dropLink()
await vi.waitFor(() => expect(subscribeFrameCount()).toBe(drop))
expect(latestSubscribePayload().historyDepth).toBeUndefined()
const next = latestSubscribePayload().streamId
acknowledgeSubscribe(next)
emitSnapshot(next, 'prompt$ ')
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
}
transport.destroy?.()
})
it('asks again when the link dropped before the first history arrived', async () => {
const transport = await connectedTransport()
acknowledgeSubscribe(latestSubscribePayload().streamId)
dropLink()
await vi.waitFor(() => expect(subscribeFrameCount()).toBe(2))
expect(latestSubscribePayload().historyDepth).toBe('full')
transport.destroy?.()
})
})
@@ -326,6 +326,11 @@ export function createRemoteRuntimePtyTransport(
let sameHandleEndReuseAttachedAt: number | null = null
let attachGeneration = 0
let subscriptionGeneration = 0
// N-02: this pane mount holds no copy of the host's history until its first subscribe lands, so
// only that subscribe asks for full depth; reconnects already hold the history and fill the gap
// from the 1000-row cut.
let fullHistoryReceived = false
const FULL_HISTORY_DEPTH: 'full' = 'full'
// A terminal this pane just created has no history, so an empty first image there is not a gap.
let spawnedHandle: string | null = null
let inputReadySubscription: number | null = null
@@ -3075,6 +3080,7 @@ export function createRemoteRuntimePtyTransport(
inputSessionId: inputJournal.sessionId,
reportsPaneVisibility: true,
outputCadence: desiredOutputCadence,
...(fullHistoryReceived ? {} : { historyDepth: FULL_HISTORY_DEPTH }),
callbacks: {
onData: (data, meta) => {
if (isCurrentSubscription()) {
@@ -3089,6 +3095,7 @@ export function createRemoteRuntimePtyTransport(
if ((data || meta?.pendingEscapeTailAnsi) && isCurrentSubscription()) {
// Why printable: a host model holding only mode/cursor bytes paints a cursor-only pane.
subscriptionSnapshotHadContent ||= restoredSnapshotPaintsPrintableContent({ data })
fullHistoryReceived = true
if (subscribedPtyId && bufferPtyShutdownReplayData(subscribedPtyId, data)) {
return
}
@@ -21,6 +21,7 @@ type SubscribePayload = {
terminal: string
client: { id: string; type: string }
viewport?: { cols: number; rows: number }
historyDepth?: 'full'
capabilities?: {
ackOutput?: 1
ackOutputSourceRanges?: 1
@@ -2,7 +2,7 @@ import {
TerminalStreamOpcode,
encodeTerminalStreamJson
} from '../../../../../shared/terminal/stream/terminal-stream-protocol'
import { DESKTOP_TERMINAL_SCROLLBACK_ROWS_DEFAULT } from '../../../../../shared/terminal/replay/terminal-scrollback-policy'
import { TERMINAL_MULTIPLEX_RECOVERY_SCROLLBACK_ROWS } from '../../../../../shared/terminal/stream/terminal-multiplex-flow-control'
import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder'
import { RemoteRuntimeTerminalMultiplexerBase } from '../streams/transport/remote-runtime-terminal-multiplexer-base'
import {
@@ -65,8 +65,7 @@ export abstract class RemoteRuntimeTerminalSnapshotController extends RemoteRunt
stream.streamId,
TerminalStreamOpcode.SnapshotRequest,
// Why history: the gap lost output, so a screen-only answer would leave the pane no history to show.
// A desktop pane keeps a local terminal's depth after a gap (N-02); the host caps by bytes.
encodeTerminalStreamJson({ scrollbackRows: DESKTOP_TERMINAL_SCROLLBACK_ROWS_DEFAULT })
encodeTerminalStreamJson({ scrollbackRows: TERMINAL_MULTIPLEX_RECOVERY_SCROLLBACK_ROWS })
)
if (!sent) {
// Transport is down; the reconnect path re-subscribes from scratch.
@@ -26,6 +26,8 @@ export class RemoteRuntimeTerminalMultiplexer extends RemoteRuntimeTerminalBinar
reportsPaneVisibility?: boolean
/** Set by callers that retune cadence via setOutputCadence; absent leaves the capability unadvertised. */
outputCadence?: TerminalOutputCadence
/** N-02: this view holds no local copy of the pane, so the first snapshot carries full history. */
historyDepth?: 'full'
callbacks: RemoteRuntimeMultiplexedTerminalCallbacks
}): Promise<RemoteRuntimeMultiplexedTerminal> {
const streamId = this.allocateStreamId()
@@ -167,7 +169,8 @@ export class RemoteRuntimeTerminalMultiplexer extends RemoteRuntimeTerminalBinar
...(args.outputCadence ? { outputCadence: 1 } : {})
},
...(args.inputSessionId ? { inputSessionId: args.inputSessionId } : {}),
...(outputCadence !== 'focused' ? { outputCadence } : {})
...(outputCadence !== 'focused' ? { outputCadence } : {}),
...(args.historyDepth ? { historyDepth: args.historyDepth } : {})
})
)
if (!sent) {