diff --git a/src/main/runtime/rpc/methods/terminal/terminal-output-credit.ts b/src/main/runtime/rpc/methods/terminal/terminal-output-credit.ts new file mode 100644 index 00000000000..80fc8f63eca --- /dev/null +++ b/src/main/runtime/rpc/methods/terminal/terminal-output-credit.ts @@ -0,0 +1,207 @@ +import { + TERMINAL_MULTIPLEX_ACK_BATCH_BYTES, + TERMINAL_MULTIPLEX_PENDING_MAX_BYTES +} from '../../../../../shared/terminal/stream/terminal-multiplex-flow-control' +import { + EMPTY_TERMINAL_REPLY_QUERY_SCAN_STATE, + scanTerminalReplyQuerySequences +} from '../../../../../shared/terminal/protocol/queries/terminal-reply-query-scan' +import type { TerminalOutputMeta } from '../../terminal-output-frame-chunks' +import { terminalStreamByteLength } from '../../terminal-stream-byte-length' +import { getOutputAfterSnapshotSeq } from './terminal-stream-replay' +import type { TerminalOutputChunk } from './terminal-stream-types' + +export const TERMINAL_QUERY_REPLAY_MAX_CHARS = 16 * 1024 +// Fixed, in code units: the client counts decoded text. The multiplex's 2 MiB max queued ~2 s on an 8 Mbit/s relay. +const OUTPUT_CREDIT_WINDOW_UNITS = 512 * 1024 + +export type TerminalOutputCredit = { + deliver: (data: string, meta?: TerminalOutputMeta) => void + /** Units are UTF-16 code units of Output text the client has parsed since its previous ack. */ + acknowledge: (units: number) => void + /** A snapshot just sent shows this held output already; drop it instead of replaying it. */ + dropOutputCoveredBySnapshot: (snapshotSeq: number) => void + /** A snapshot of unknown seq was sent; a recovery replaces held output it may already show. */ + recoverHeldOutput: () => void + dispose: () => void +} + +/** Bounds Output a client has not parsed yet. Held output past the pending budget is replaced by + * one current-screen snapshot once the link drains, so a client skips stale output after Ctrl-C. */ +export function createTerminalOutputCredit(options: { + send: (data: string, meta?: TerminalOutputMeta) => void + /** Resolves the sent snapshot's output seq, or undefined when none was sent. */ + sendRecoverySnapshot: () => Promise +}): TerminalOutputCredit { + let inFlightUnits = 0 + let held: TerminalOutputChunk[] = [] + let heldBytes = 0 + let overflowed = false + let recovering = false + let disposed = false + // Snapshots omit control sequences; replay dropped queries so a waiting TUI still gets its reply. + let droppedQueries = '' + let droppedQueriesOverflowed = false + let queryScanState = EMPTY_TERMINAL_REPLY_QUERY_SCAN_STATE + + const collectDroppedQueries = (chunks: TerminalOutputChunk[]): void => { + for (const chunk of chunks) { + if (droppedQueriesOverflowed) { + return + } + const seq = chunk.meta?.seq + if (typeof seq !== 'number' || chunk.meta?.rawLength !== chunk.data.length) { + queryScanState = EMPTY_TERMINAL_REPLY_QUERY_SCAN_STATE + continue + } + const scan = scanTerminalReplyQuerySequences( + chunk.data, + seq - chunk.data.length, + queryScanState + ) + queryScanState = scan.state + for (const query of scan.queries) { + if (droppedQueries.length + query.data.length > TERMINAL_QUERY_REPLAY_MAX_CHARS) { + droppedQueriesOverflowed = true + break + } + droppedQueries += query.data + } + } + } + + const replayDroppedQueries = (): void => { + if (droppedQueries) { + const bytes = terminalStreamByteLength(droppedQueries) + held.unshift({ data: droppedQueries, bytes }) + heldBytes += bytes + } + droppedQueries = '' + droppedQueriesOverflowed = false + queryScanState = EMPTY_TERMINAL_REPLY_QUERY_SCAN_STATE + } + + const fits = (units: number): boolean => + inFlightUnits === 0 || inFlightUnits + units <= OUTPUT_CREDIT_WINDOW_UNITS + + const trimToBudget = (): void => { + let dropCount = 0 + while (heldBytes > TERMINAL_MULTIPLEX_PENDING_MAX_BYTES && dropCount < held.length) { + heldBytes -= held[dropCount]!.bytes + dropCount += 1 + } + if (dropCount > 0) { + collectDroppedQueries(held.splice(0, dropCount)) + overflowed = true + } + } + + const dropCovered = (snapshotSeq: number): void => { + const kept: TerminalOutputChunk[] = [] + const covered: TerminalOutputChunk[] = [] + for (const chunk of held) { + const uncovered = getOutputAfterSnapshotSeq(chunk, snapshotSeq) + if (uncovered === chunk) { + kept.push(chunk) + } else if (uncovered?.data) { + const prefix = chunk.data.slice(0, chunk.data.length - uncovered.data.length) + covered.push({ + data: prefix, + bytes: 0, + meta: { seq: snapshotSeq, rawLength: prefix.length } + }) + kept.push({ ...uncovered, bytes: terminalStreamByteLength(uncovered.data) }) + } else { + covered.push(chunk) + } + } + collectDroppedQueries(covered) + held = kept + heldBytes = kept.reduce((total, chunk) => total + chunk.bytes, 0) + replayDroppedQueries() + } + + const recover = async (): Promise => { + recovering = true + overflowed = false + try { + const snapshotSeq = await options.sendRecoverySnapshot() + if (disposed) { + return + } + if (typeof snapshotSeq === 'number') { + dropCovered(snapshotSeq) + } else { + replayDroppedQueries() + } + } catch { + // Keep streaming what is held; the next overflow tries again. + } finally { + recovering = false + pump() + } + } + + function pump(): void { + if (disposed || recovering) { + return + } + if (!overflowed) { + let sent = 0 + while (sent < held.length && fits(held[sent]!.data.length)) { + const chunk = held[sent]! + inFlightUnits += chunk.data.length + heldBytes -= chunk.bytes + sent += 1 + options.send(chunk.data, chunk.meta) + } + held.splice(0, sent) + } + // Trim after sending so output held across a recovery reaches the client before any new gap. + trimToBudget() + // Recover only once the link has drained, so a sustained flood cannot outrun the client with snapshots. + if (overflowed && inFlightUnits <= TERMINAL_MULTIPLEX_ACK_BATCH_BYTES) { + void recover() + } + } + + return { + deliver(data, meta) { + if (disposed) { + return + } + if (!overflowed && !recovering && held.length === 0 && fits(data.length)) { + inFlightUnits += data.length + options.send(data, meta) + return + } + const bytes = terminalStreamByteLength(data) + held.push({ data, bytes, meta }) + heldBytes += bytes + // During a recovery this returns early: held output follows its snapshot seq, and trimming it would open a new gap. + pump() + }, + acknowledge(units) { + inFlightUnits -= Math.min(inFlightUnits, units) + pump() + }, + dropOutputCoveredBySnapshot(snapshotSeq) { + if (disposed) { + return + } + dropCovered(snapshotSeq) + pump() + }, + recoverHeldOutput() { + if (disposed || held.length === 0) { + return + } + overflowed = true + pump() + }, + dispose() { + disposed = true + held = [] + } + } +} diff --git a/src/main/runtime/rpc/methods/terminal/terminal-stream-tokens.ts b/src/main/runtime/rpc/methods/terminal/terminal-stream-tokens.ts new file mode 100644 index 00000000000..f50daf43f1c --- /dev/null +++ b/src/main/runtime/rpc/methods/terminal/terminal-stream-tokens.ts @@ -0,0 +1,31 @@ +import { randomUUID } from 'node:crypto' + +type TerminalStreamTokenEntry = { + deviceId: string | undefined + acknowledgeOutput: (units: number) => void +} + +// Why tokens: relays renumber streamId, and a late ack keyed by terminal+client would credit its successor. +const liveStreams = new Map() + +export function registerTerminalStreamToken( + deviceId: string | undefined, + acknowledgeOutput: (units: number) => void +): { token: string; release: () => void } { + const token = randomUUID() + liveStreams.set(token, { deviceId, acknowledgeOutput }) + return { token, release: () => liveStreams.delete(token) } +} + +export function acknowledgeTerminalStreamOutput( + token: string, + deviceId: string | undefined, + units: number +): boolean { + const entry = liveStreams.get(token) + if (!entry || entry.deviceId !== deviceId) { + return false + } + entry.acknowledgeOutput(units) + return true +} diff --git a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-live.ts b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-live.ts index bda93cac810..6cca5279663 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-live.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-live.ts @@ -14,10 +14,12 @@ import type { TerminalSubscriptionArgs } from './terminal-subscribe-types' import { deepenHeadlessCopyToDesktopDepth } from './headless-copy-desktop-depth' +import type { TerminalOutputCredit } from './terminal-output-credit' export function activateTerminalSubscribeBinaryLive( args: TerminalSubscriptionArgs, - state: TerminalSubscribeBinaryState + state: TerminalSubscribeBinaryState, + outputCredit: TerminalOutputCredit | null ): void { const { params, runtime, emit, ptyId, clientId, isMobile, supportsDesktopViewportClaims } = args const lateRendererReady = state.lateRendererReadyPromise @@ -58,6 +60,9 @@ export function activateTerminalSubscribeBinaryLive( truncatedByByteBudget: recovery.truncatedByByteBudget, data: recovery.data }) + if (typeof recovery.seq === 'number') { + outputCredit?.dropOutputCoveredBySnapshot(recovery.seq) + } state.lastResizeCols = recovery.cols console.log('[mobile-terminal-stream] recovery snapshot', { terminal: params.terminal, @@ -117,6 +122,9 @@ export function activateTerminalSubscribeBinaryLive( } if (!restreamed) { sendResizedFrame(event) + } else { + // The restream reports no output seq, so held output may duplicate it. + outputCredit?.recoverHeldOutput() } }) // Even failed replay must report geometry. diff --git a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-snapshot.ts b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-snapshot.ts index f97ed628f14..2e0b783d8ea 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-snapshot.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary-snapshot.ts @@ -15,12 +15,15 @@ import type { TerminalSubscriptionArgs } from './terminal-subscribe-types' import { deepenHeadlessCopyToDesktopDepth } from './headless-copy-desktop-depth' +import { sendTerminalSubscribeRecoverySnapshot } from './terminal-subscribe-recovery-snapshot' const MOBILE_RENDERER_MOUNT_READY_TIMEOUT_MS = 3_000 export async function publishTerminalSubscribeBinarySnapshot( args: TerminalSubscriptionArgs, - state: TerminalSubscribeBinaryState + state: TerminalSubscribeBinaryState, + /** Set when this stream's output is credited; the client acks under this token. */ + streamToken: string | null ): Promise { const { params, @@ -193,7 +196,8 @@ export async function publishTerminalSubscribeBinarySnapshot( cols: serialized?.cols ?? size?.cols, rows: serialized?.rows ?? size?.rows, displayMode: state.displayMode, - seq: layoutSeq + seq: layoutSeq, + ...(streamToken ? { outputAck: true, streamToken } : {}) }) const snapshotStats = sendSnapshotFrames(state.sendFrame, { ...scrollbackFrame, @@ -226,51 +230,25 @@ export async function publishTerminalSubscribeBinarySnapshot( // Initial replay already rewrapped to these columns; seed resize replay gating. state.lastResizeCols = serialized?.cols ?? size?.cols let recoveryAttempts = 0 - // Use the recovery's actual resized frame for its budget. - const recoveryFrame = { kind: 'resized', reason: 'pending-output-overflow' } as const // Pre-subscribe overflow dropped a middle segment; only a fresh model snapshot covers it. while (state.pendingOutputOverflowed && recoveryAttempts < 2) { state.pendingOutputOverflowed = false recoveryAttempts += 1 - const recovery = await serializeBudgetedMobileSnapshot( - runtime, - ptyId, - mobileSnapshotByteBudget(params.snapshotByteBudget, state.streamId, recoveryFrame) + const recoverySeq = await sendTerminalSubscribeRecoverySnapshot( + args, + state, + 'pending-output-overflow' ) if (state.closed) { return } - if (!recovery) { + if (recoverySeq === undefined) { break } - // Without output seq, retain bounded replay rather than an unverifiable renderer snapshot. - if (typeof recovery.seq !== 'number') { - break - } - // Clients ignore repeat scrollback but apply resized inline; omit output seq to avoid polluting layout seq filtering. - const recoveryStats = sendSnapshotFrames(state.sendFrame, { - ...recoveryFrame, - cols: recovery.cols, - rows: recovery.rows, - displayMode: state.displayMode, - source: recovery.source, - truncated: false, - truncatedByByteBudget: recovery.truncatedByByteBudget, - data: recovery.data - }) - console.log('[mobile-terminal-stream] recovery snapshot', { - terminal: params.terminal, - streamId: state.streamId, - reason: 'pending-output-overflow', - bytes: recoveryStats.bytes, - chunks: recoveryStats.chunks, - scrollbackRows: recovery.scrollbackRows, - truncatedByByteBudget: recovery.truncatedByByteBudget === true - }) - const trimmed = trimPendingOutputCoveredBySnapshot(state.pendingOutput, recovery.seq) + const trimmed = trimPendingOutputCoveredBySnapshot(state.pendingOutput, recoverySeq) state.pendingOutput = trimmed.chunks state.pendingOutputBytes = trimmed.bytes - snapshotOutputSeq = recovery.seq + snapshotOutputSeq = recoverySeq } state.buffering = false const bufferedOutput = state.pendingOutput.splice(0) diff --git a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary.ts b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary.ts index b23c065fd54..0b64fcacfc0 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-binary.ts @@ -3,7 +3,10 @@ import { encodeTerminalStreamFrame, encodeTerminalStreamJson } from '../../../../../shared/terminal/stream/terminal-stream-protocol' -import { iterateTerminalOutputFrameChunks } from '../../terminal-output-frame-chunks' +import { + iterateTerminalOutputFrameChunks, + type TerminalOutputMeta +} from '../../terminal-output-frame-chunks' import { EMPTY_TERMINAL_REPLY_QUERY_SCAN_STATE, scanTerminalReplyQuerySequences, @@ -21,9 +24,15 @@ import type { } from './terminal-subscribe-types' import type { TerminalOutputChunk } from './terminal-stream-types' import { publishTerminalSubscribeBinarySnapshot } from './terminal-subscribe-binary-snapshot' +import { sendTerminalCreditRecoverySnapshot } from './terminal-subscribe-recovery-snapshot' import { activateTerminalSubscribeBinaryLive } from './terminal-subscribe-binary-live' import { registerTerminalSubscribeBinaryControlFrames } from './terminal-subscribe-binary-control-frames' -const TERMINAL_QUERY_REPLAY_MAX_CHARS = 16 * 1024 +import { + createTerminalOutputCredit, + TERMINAL_QUERY_REPLAY_MAX_CHARS, + type TerminalOutputCredit +} from './terminal-output-credit' +import { registerTerminalStreamToken } from './terminal-stream-tokens' export async function runTerminalBinarySubscription(args: TerminalSubscriptionArgs): Promise { const { runtime, registration, sendBinary, ptyId, clientId, viewerDeviceId, isMobile } = args if (!sendBinary) { @@ -53,13 +62,18 @@ export async function runTerminalBinarySubscription(args: TerminalSubscriptionAr let abortRendererMountWait = (): void => {} let lateRendererReadyPromise: Promise | null = null let outputBatcher: TerminalOutputBatcher | null = null + let outputCredit: TerminalOutputCredit | null = null + let streamToken: ReturnType | null = null let resolveStream = (): void => {} const streamClosed = new Promise((resolve) => { resolveStream = resolve }) registration.setTeardown(() => { + // Dispose credit first so the final flush cannot start a recovery on a closing stream. + outputCredit?.dispose() outputBatcher?.flush() outputBatcher?.dispose() + streamToken?.release() closed = true unsubscribeData() unsubscribeResize() @@ -87,7 +101,22 @@ export async function runTerminalBinarySubscription(args: TerminalSubscriptionAr compressible: true }) } + const sendOutputFrames = (data: string, meta?: TerminalOutputMeta): void => { + // Why text: phones before this release can't decode OutputSpan, so a span here is output they drop. + for (const chunk of iterateTerminalOutputFrameChunks(data, meta, 'text')) { + sendFrame(chunk.opcode ?? TerminalStreamOpcode.Output, chunk.bytes, chunk.seq) + } + } + // Only clients that ask get credit; others, and hosts before it, stream uncredited. + if (args.params.capabilities?.outputAck === 1) { + outputCredit = createTerminalOutputCredit({ + send: sendOutputFrames, + sendRecoverySnapshot: () => sendTerminalCreditRecoverySnapshot(args, state) + }) + streamToken = registerTerminalStreamToken(args.pairedDeviceId, outputCredit.acknowledge) + } outputBatcher = createTerminalOutputBatcher((data, meta) => { + // Metadata bypasses credit so dropped output never loses a cwd change. if (meta?.cwd !== undefined) { sendFrame( TerminalStreamOpcode.Metadata, @@ -95,9 +124,10 @@ export async function runTerminalBinarySubscription(args: TerminalSubscriptionAr meta.seq ) } - // Why text: phones before this release can't decode OutputSpan, so a span here is output they drop. - for (const chunk of iterateTerminalOutputFrameChunks(data, meta, 'text')) { - sendFrame(chunk.opcode ?? TerminalStreamOpcode.Output, chunk.bytes, chunk.seq) + if (outputCredit) { + outputCredit.deliver(data, meta) + } else { + sendOutputFrames(data, meta) } }) unregisterBinaryHandler = registerTerminalSubscribeBinaryControlFrames( @@ -266,11 +296,11 @@ export async function runTerminalBinarySubscription(args: TerminalSubscriptionAr streamClosed, sendFrame } - await publishTerminalSubscribeBinarySnapshot(args, state) + await publishTerminalSubscribeBinarySnapshot(args, state, streamToken?.token ?? null) if (registration.released) { return } args.onEstablished?.() - activateTerminalSubscribeBinaryLive(args, state) + activateTerminalSubscribeBinaryLive(args, state, outputCredit) await streamClosed } diff --git a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-method.ts b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-method.ts index 802b7add72d..038f2012172 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-method.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-method.ts @@ -101,6 +101,7 @@ export const TERMINAL_SUBSCRIBE_METHODS = [ ptyId, clientId, viewerDeviceId, + pairedDeviceId, isMobile, supportsDesktopViewportClaims: params.capabilities?.desktopViewportClaims === 1, supportsWriteUnavailable: params.capabilities?.writeUnavailable === 1, diff --git a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-recovery-snapshot.ts b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-recovery-snapshot.ts new file mode 100644 index 00000000000..09df5a434b4 --- /dev/null +++ b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-recovery-snapshot.ts @@ -0,0 +1,64 @@ +import { + mobileSnapshotByteBudget, + sendSnapshotFrames, + serializeBudgetedMobileSnapshot +} from './terminal-snapshot-publication' +import type { + TerminalSubscribeBinaryState, + TerminalSubscriptionArgs +} from './terminal-subscribe-types' + +/** Sends the current screen as an in-place `resized` snapshot; resolves its output seq, or undefined when none was sent. */ +export async function sendTerminalSubscribeRecoverySnapshot( + args: TerminalSubscriptionArgs, + state: TerminalSubscribeBinaryState, + reason: 'pending-output-overflow' | 'output-credit-overflow' +): Promise { + const { params, runtime, ptyId } = args + // Use the recovery's actual resized frame for its budget. + const recoveryFrame = { kind: 'resized', reason } as const + const recovery = await serializeBudgetedMobileSnapshot( + runtime, + ptyId, + mobileSnapshotByteBudget(params.snapshotByteBudget, state.streamId, recoveryFrame) + ) + // Without output seq, retain bounded replay rather than an unverifiable renderer snapshot. + if (state.closed || typeof recovery?.seq !== 'number') { + return undefined + } + // Clients ignore repeat scrollback but apply resized inline; omit output seq to avoid polluting layout seq filtering. + const recoveryStats = sendSnapshotFrames(state.sendFrame, { + ...recoveryFrame, + cols: recovery.cols, + rows: recovery.rows, + displayMode: state.displayMode, + source: recovery.source, + truncated: false, + truncatedByByteBudget: recovery.truncatedByByteBudget, + data: recovery.data + }) + console.log('[mobile-terminal-stream] recovery snapshot', { + terminal: params.terminal, + streamId: state.streamId, + reason, + bytes: recoveryStats.bytes, + chunks: recoveryStats.chunks, + scrollbackRows: recovery.scrollbackRows, + truncatedByByteBudget: recovery.truncatedByByteBudget === true + }) + return recovery.seq +} + +/** Credit overflow recovery: the snapshot also covers what the batcher still holds, so flush it into the held queue. */ +export async function sendTerminalCreditRecoverySnapshot( + args: TerminalSubscriptionArgs, + state: TerminalSubscribeBinaryState +): Promise { + const snapshotSeq = await sendTerminalSubscribeRecoverySnapshot( + args, + state, + 'output-credit-overflow' + ) + state.outputBatcher.flush() + return snapshotSeq +} diff --git a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-types.ts b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-types.ts index 111df5547dd..ed1bb102662 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-subscribe-types.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-subscribe-types.ts @@ -21,6 +21,7 @@ export type TerminalSubscriptionArgs = { clientId: string | undefined // The authenticated device, whose departure ends this view; request ids are client-declared. viewerDeviceId: string | undefined + pairedDeviceId: string | undefined isMobile: boolean supportsDesktopViewportClaims: boolean supportsWriteUnavailable: boolean diff --git a/src/main/runtime/rpc/methods/terminal/terminal-viewport-methods.ts b/src/main/runtime/rpc/methods/terminal/terminal-viewport-methods.ts index 8e540d53a88..aca8edec36c 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-viewport-methods.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-viewport-methods.ts @@ -8,6 +8,8 @@ import { } from '../../../../../shared/rpc-contract/terminal-viewport-schemas-params' import { updateViewportForClient } from './terminal-viewport-update' import { TerminalGetAutoRestoreFitParams } from '../../../../../shared/rpc-contract/terminal-viewport-methods-params' +import { TerminalAckOutput } from '../../../../../shared/rpc-contract/terminal-stream-params' +import { acknowledgeTerminalStreamOutput } from './terminal-stream-tokens' export const TERMINAL_VIEWPORT_METHODS_BEFORE_STREAMS = [ defineMethod({ @@ -127,5 +129,14 @@ export const TERMINAL_VIEWPORT_METHODS_AFTER_STREAMS = [ handler: async (params, { runtime }) => ({ ms: runtime.setMobileAutoRestoreFitMs(params.ms) }) + }), + defineMethod({ + name: 'terminal.ackOutput', + permission: 'workspace', + params: TerminalAckOutput, + // Late acks for an ended stream are normal; report them instead of failing the request. + handler: async (params, { pairedDeviceId }) => ({ + credited: acknowledgeTerminalStreamOutput(params.streamToken, pairedDeviceId, params.units) + }) }) ] diff --git a/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts b/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts index 46deedf58f7..f419583f6e7 100644 --- a/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts +++ b/src/main/runtime/rpc/server/runtime-rpc-mobile-method-routing.ts @@ -203,6 +203,7 @@ export const EXECUTION_HOST_METHODS = [ 'session.tabs.unsubscribe', 'session.tabs.unsubscribeAll', ...MOBILE_AGENT_SESSION_RPC_METHODS, + 'terminal.ackOutput', 'terminal.adoptOrphans', 'terminal.agentStatus', 'terminal.clearBuffer', diff --git a/src/main/runtime/rpc/terminal-subscribe-output-credit.test.ts b/src/main/runtime/rpc/terminal-subscribe-output-credit.test.ts new file mode 100644 index 00000000000..2b8062f9168 --- /dev/null +++ b/src/main/runtime/rpc/terminal-subscribe-output-credit.test.ts @@ -0,0 +1,164 @@ +import './unused-default-rpc-methods.test-fixture' +import { describe, expect, it, vi } from 'vitest' +import { RpcDispatcher } from './dispatcher' +import type { OrcaRuntimeService } from '../orca-runtime' +import { TERMINAL_METHODS } from './methods/terminal' +import { createSubscriptionRegistryDouble } from './subscription-registry-test-double' +import type { RuntimeTerminalWait } from '../../../shared/runtime-types' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + decodeTerminalStreamText +} from '../../../shared/terminal/stream/terminal-stream-protocol' + +const CHUNK = 64 * 1024 + +async function startCreditedSubscription() { + const frames: Uint8Array[] = [] + const messages: string[] = [] + const registry = createSubscriptionRegistryDouble() + let dataListener: ((data: string, meta?: { seq?: number; rawLength?: number }) => void) | null = + null + let outputSeq = 0 + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This partial runtime supplies the terminal RPC methods these tests invoke. + const runtime = { + getRuntimeId: () => 'test-runtime', + subscribeToPtyExit: vi.fn(() => vi.fn()), + registerRemoteTerminalViewSubscriber: () => () => {}, + requestRendererTerminalTabMount: () => false, + resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }), + handleMobileSubscribe: vi.fn().mockResolvedValue(true), + handleMobileUnsubscribe: vi.fn(), + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return vi.fn() + }), + readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }), + serializeTerminalBuffer: vi.fn(async () => ({ + data: 'screen', + cols: 80, + rows: 24, + seq: outputSeq + })), + getTerminalSize: vi.fn().mockReturnValue({ cols: 80, rows: 24 }), + getMobileDisplayMode: vi.fn().mockReturnValue('auto'), + getLayout: vi.fn().mockReturnValue({ seq: 1 }), + isTerminalAlternateScreen: vi.fn().mockReturnValue(false), + subscribeToTerminalResize: vi.fn().mockReturnValue(vi.fn()), + subscribeToFitOverrideChanges: vi.fn().mockReturnValue(vi.fn()), + registerSubscriptionCleanup: vi.fn(registry.registerSubscriptionCleanup), + registerOwnedSubscriptionCleanup: vi.fn(registry.registerOwnedSubscriptionCleanup), + cleanupSubscription: vi.fn(registry.cleanupSubscription), + waitForTerminal: vi.fn(() => new Promise(() => {})) + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS }) + const done = dispatcher.dispatchStreaming( + { + id: 'sub', + authToken: 'tok', + method: 'terminal.subscribe', + params: { + terminal: 'terminal-1', + client: { id: 'phone-1', type: 'mobile' }, + capabilities: { terminalBinaryStream: 1, outputAck: 1 } + } + }, + (msg) => messages.push(msg), + { + connectionId: 'conn-phone', + pairedDeviceId: 'phone-device', + sendBinary: (bytes) => { + frames.push(bytes) + }, + registerBinaryStreamHandler: vi.fn(() => vi.fn()) + } + ) + const subscribed = await vi.waitFor(() => { + const result = messages + .map((msg) => JSON.parse(msg).result) + .find((candidate) => candidate?.type === 'subscribed') + expect(result).toBeDefined() + return result + }) + await vi.waitFor(() => expect(dataListener).not.toBeNull()) + return { + subscribed, + output: (count: number, length = CHUNK) => { + for (let index = 0; index < count; index += 1) { + outputSeq += length + dataListener!('x'.repeat(length), { seq: outputSeq, rawLength: length }) + } + }, + sentOutputUnits: () => + frames.reduce((total, bytes) => { + const frame = decodeTerminalStreamFrame(bytes) + return frame?.opcode === TerminalStreamOpcode.Output + ? total + decodeTerminalStreamText(frame.payload).length + : total + }, 0), + snapshotKinds: () => + frames.flatMap((bytes) => { + const frame = decodeTerminalStreamFrame(bytes) + return frame?.opcode === TerminalStreamOpcode.SnapshotStart + ? [decodeTerminalStreamJson<{ kind: string }>(frame.payload)?.kind] + : [] + }), + ack: async (streamToken: string, units: number, pairedDeviceId = 'phone-device') => + ( + await dispatcher.dispatch( + { + id: 'ack', + authToken: 'tok', + method: 'terminal.ackOutput', + params: { streamToken, units } + }, + { connectionId: 'conn-phone', pairedDeviceId } + ) + ).ok, + stop: async () => { + runtime.cleanupSubscription('terminal-1:phone-1') + await done + } + } +} + +describe('terminal.subscribe output credit', () => { + it('holds output beyond the window until the stream token is acked', async () => { + const stream = await startCreditedSubscription() + expect(stream.subscribed).toMatchObject({ outputAck: true, streamToken: expect.any(String) }) + + stream.output(9) + expect(stream.sentOutputUnits()).toBe(512 * 1024) + + await stream.ack(stream.subscribed.streamToken, CHUNK) + expect(stream.sentOutputUnits()).toBe(576 * 1024) + await stream.stop() + }) + + it("credits nothing for another device's ack or an unknown token", async () => { + const stream = await startCreditedSubscription() + stream.output(9) + + await stream.ack(stream.subscribed.streamToken, CHUNK, 'other-device') + await stream.ack('stale-token', CHUNK) + expect(stream.sentOutputUnits()).toBe(512 * 1024) + await stream.stop() + }) + + it('replaces overflowed output with one recovery snapshot once the window drains', async () => { + const stream = await startCreditedSubscription() + // 512 KiB fills the window; the next 320 KiB overflows the 256 KiB hold; 100 units wait in the batcher. + stream.output(13) + stream.output(1, 100) + expect(stream.snapshotKinds()).toEqual(['scrollback']) + + await stream.ack(stream.subscribed.streamToken, 512 * 1024) + await vi.waitFor(() => expect(stream.snapshotKinds()).toEqual(['scrollback', 'resized'])) + // The snapshot shows everything held or batched, so only output after it is sent. + stream.output(1) + expect(stream.sentOutputUnits()).toBe(576 * 1024) + expect(stream.snapshotKinds()).toEqual(['scrollback', 'resized']) + await stream.stop() + }) +}) diff --git a/src/shared/rpc-contract/rpc-params-catalog.generated.ts b/src/shared/rpc-contract/rpc-params-catalog.generated.ts index f1618a7edc7..5faac8cf8fd 100644 --- a/src/shared/rpc-contract/rpc-params-catalog.generated.ts +++ b/src/shared/rpc-contract/rpc-params-catalog.generated.ts @@ -654,6 +654,7 @@ import { TerminalLimitUpdateParams } from './terminal-limit-params' import { TerminalAdoptOrphans } from './terminal-orphan-params' import { TerminalQuickCommandsUpdate } from './terminal-quick-command-params' import { + TerminalAckOutput, TerminalMultiplex, TerminalResizeForClient, TerminalSubscribe @@ -1494,6 +1495,7 @@ export const RPC_PARAMS_BY_METHOD = { 'ssh.updateTarget': SshTargetUpdate, 'stats.summary': null, 'status.get': StatusGetParams, + 'terminal.ackOutput': TerminalAckOutput, 'terminal.adoptOrphans': TerminalAdoptOrphans, 'terminal.agentStatus': TerminalHandle, 'terminal.clearBuffer': TerminalHandle, diff --git a/src/shared/rpc-contract/terminal-stream-params.ts b/src/shared/rpc-contract/terminal-stream-params.ts index 5786f18c39e..3367228094f 100644 --- a/src/shared/rpc-contract/terminal-stream-params.ts +++ b/src/shared/rpc-contract/terminal-stream-params.ts @@ -32,7 +32,9 @@ export const TerminalSubscribe = TerminalHandle.extend({ terminalBinaryStream: z.literal(1).optional(), desktopViewportClaims: z.literal(1).optional(), mobileInputLeaseOnly: z.literal(1).optional(), - writeUnavailable: z.literal(1).optional() + writeUnavailable: z.literal(1).optional(), + // Ask for credited output; the host echoes `outputAck: true` and a `streamToken` in `subscribed`. + outputAck: z.literal(1).optional() }) .optional(), /** Budget the JSON frame, not raw text: ESC expands sixfold and can push a 512 KiB snapshot past a 640 KiB bridge cap. @@ -40,4 +42,10 @@ export const TerminalSubscribe = TerminalHandle.extend({ snapshotByteBudget: z.number().int().positive().optional() }) +/** Credit for Output text the client has parsed, in UTF-16 code units since its previous ack. */ +export const TerminalAckOutput = z.object({ + streamToken: requiredString('Missing stream token'), + units: z.number().int().positive() +}) + export const TerminalMultiplex = z.object({})