fix(remote): harden terminal recovery lifecycle

This commit is contained in:
OrcaWin
2026-07-29 19:29:50 -07:00
parent 4ed700e160
commit b31ee22b6d
17 changed files with 720 additions and 251 deletions
+24 -7
View File
@@ -2996,10 +2996,10 @@
"https://github.com/stablyai/orca/issues/8652",
"https://github.com/stablyai/orca/pull/10625"
],
"invariant": "A host advertising terminal.paired-parking.v1 keeps the PTY and bounded authoritative history alive while an ordinary hidden-view park destroys the client xterm and releases its raw per-PTY stream. Reveal must restore up to the requested 5,000 rows, parked-time side effects/output, the same PTY identity, and continued input/output. Hosts without the capability must retain the existing limit/TTL force-parking fallback. A paired terminal stream whose delivery credits or command responses stop progressing must replace only that stream and repaint authoritative host output. Manual disconnect must retain pairing while preventing queued or passive calls and subscriptions from recreating transport until explicit Connect.",
"oracle": "Run one byte-identical six-terminal oracle against an isolated headed desktop host and an isolated headless `orca serve` host. Stage at least 1,000,000 xterm cells, enable ordinary parking with the lossy retention budget disabled, require exactly one mounted manager and five parked tabs, at most 45% retained cells, no more than 16 MiB heap growth, and under 500 ms timer drift. While a tab is parked, require authoritative terminal.read to observe new PTY output; reveal it and require the original PTY, a marker within the requested 5,000-row history, the parked marker, and post-reveal input/output. Admit 128 active streams, reject the 129th as retryable, release one stream, then require the retry to attach and publish a snapshot without multiplying retained subscribers. Disable terminal.paired-parking.v1 and require the same oracle to fail before parking, while the legacy limit-one fallback separately passes. Also preserve truncated first paint, ACK-starved same-PTY stream recovery, and queued manual-disconnect fencing.",
"invariant": "A host advertising terminal.paired-parking.v1 keeps the PTY and bounded authoritative history alive while an ordinary hidden-view park destroys the client xterm and releases its raw per-PTY stream. Reveal must restore up to the requested 5,000 rows, parked-time side effects/output, the same PTY identity, and continued input/output. Hosts without the capability must retain the existing limit/TTL force-parking fallback. A paired terminal stream whose delivery credits stop progressing must replace only that stream; command silence must first probe authoritative state and replace only if that probe times out. Manual disconnect must retain pairing while preventing queued or passive calls and subscriptions from recreating transport until explicit Connect.",
"oracle": "Run one byte-identical six-terminal oracle against an isolated headed desktop host and an isolated headless `orca serve` host. Stage at least 1,000,000 xterm cells, enable ordinary parking with the lossy retention budget disabled, require exactly one mounted manager and five parked tabs, at most 45% retained cells, no more than 16 MiB heap growth, and under 500 ms timer drift. While a tab is parked, require authoritative terminal.read to observe new PTY output; reveal it and require the original PTY, a marker within the requested 5,000-row history, the parked marker, and post-reveal input/output. Admit 128 active streams, reject the 129th as retryable, release one stream, then require the retry to attach and publish a snapshot without multiplying retained subscribers. Disable terminal.paired-parking.v1 and require the same oracle to fail before parking, while the legacy limit-one fallback separately passes. Also preserve truncated first paint, ACK-starved same-PTY recovery, responsive silent-command snapshot probes, dead-stream probe timeout recovery, and queued manual-disconnect fencing.",
"commands": [
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts",
"ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
"ORCA_E2E_WEB_CLIENT=1 ORCA_E2E_DISABLE_PAIRED_TERMINAL_PARKING=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1",
@@ -3022,6 +3022,7 @@
"src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts",
"src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts",
"src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts",
"src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts",
"src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts",
"src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts",
"src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts",
@@ -3081,7 +3082,8 @@
"assertions": [
"forwards facts over the shared client-event stream without a desktop renderer",
"prefers provider history over a partial headless mirror for requested snapshots",
"falls back to the available mirror when authoritative provider history is unavailable"
"falls back to the available mirror when authoritative provider history is unavailable",
"bounds a hung authoritative provider acquisition and reuses its fallback"
]
},
{
@@ -3108,13 +3110,28 @@
"rejects paired, fail-open, foreign, and null ptys without capability evidence"
]
},
{
"file": "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts",
"assertions": [
"keeps a mounted HUB mirror alive when the old stream ends before the replacement snapshot"
]
},
{
"file": "src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts",
"assertions": ["starts a fact watcher for snapshot-capable paired PTYs"]
},
{
"file": "src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts",
"assertions": [
"retains a continuing watcher and title while reconciling a reminted split leaf"
]
},
{
"file": "src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts",
"assertions": [
"restarts only the stream whose renderer delivery credit never settles",
"probes then restarts a stream when an entered command receives no frames",
"keeps a silent responsive stream after its authoritative snapshot probe",
"classifies a capacity rejection followed by end as recoverable transport pressure"
]
},
@@ -3133,10 +3150,10 @@
"date": "2026-07-29",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts",
"result": "passed",
"durationSeconds": 27.82,
"summary": "Sixteen focused files passed 1,935 tests with one existing skip, including 128 active streams, retryable rejection at 129, released-slot snapshot recovery, and balanced subscriber cleanup."
"durationSeconds": 19.54,
"summary": "Seventeen focused files passed 1,938 tests with one existing skip, including 128 active streams, retryable rejection at 129, released-slot snapshot recovery, bounded liveness probes, and balanced subscriber cleanup."
},
{
"date": "2026-07-29",
+68
View File
@@ -58,6 +58,7 @@ import { getBaseRefDefault, getBranchConflictKind } from '../git/repo'
import { OrchestrationDb } from './orchestration/db'
import type { MessagePriority, MessageRow, MessageType } from './orchestration/types'
import {
AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS,
appendNormalizedToTailBuffer,
appendRecentPtyPathCandidates,
buildPreview,
@@ -8544,6 +8545,73 @@ describe('OrcaRuntimeService', () => {
})
})
it('bounds a hung authoritative provider acquisition and reuses its fallback', async () => {
vi.useFakeTimers()
try {
let releaseProvider: (value: null) => void = () => {}
const hungProvider = new Promise<null>((resolve) => {
releaseProvider = resolve
})
const serializeProviderBuffer = vi
.fn()
.mockReturnValueOnce(hungProvider)
.mockResolvedValueOnce({
data: 'provider recovered\r\n',
cols: 100,
rows: 30,
seq: 200,
source: 'headless'
})
const { runtime } = createSideEffectRuntime()
runtime.setPtyController({
write: () => true,
kill: () => true,
getForegroundProcess: async () => null,
serializeProviderBuffer,
hasRendererSerializer: () => false
})
syncSinglePty(runtime)
runtime.onPtyData('pty-1', 'available mirror\r\n', 100)
const firstSnapshot = runtime.serializeAuthoritativeTerminalBuffer('pty-1', {
scrollbackRows: 5000
})
const concurrentSnapshot = runtime.serializeAuthoritativeTerminalBuffer('pty-1', {
scrollbackRows: 5000
})
expect(serializeProviderBuffer).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS)
await expect(firstSnapshot).resolves.toMatchObject({
data: expect.stringContaining('available mirror'),
source: 'headless'
})
await expect(concurrentSnapshot).resolves.toMatchObject({
data: expect.stringContaining('available mirror'),
source: 'headless'
})
await expect(
runtime.serializeAuthoritativeTerminalBuffer('pty-1', { scrollbackRows: 5000 })
).resolves.toMatchObject({
data: expect.stringContaining('available mirror'),
source: 'headless'
})
expect(serializeProviderBuffer).toHaveBeenCalledOnce()
releaseProvider(null)
await vi.advanceTimersByTimeAsync(0)
await expect(
runtime.serializeAuthoritativeTerminalBuffer('pty-1', { scrollbackRows: 5000 })
).resolves.toMatchObject({
data: 'provider recovered\r\n',
source: 'headless'
})
expect(serializeProviderBuffer).toHaveBeenCalledTimes(2)
} finally {
vi.useRealTimers()
}
})
it('falls back to provider history when a mounted renderer has not hydrated yet', async () => {
const { runtime } = createSideEffectRuntime()
const serializeBuffer = vi.fn().mockResolvedValue({
+28 -6
View File
@@ -1463,6 +1463,7 @@ type ProviderBufferAcquisition = {
generation: number
scrollbackRows: number
promise: Promise<PtyProviderBufferSnapshot | null>
timedOut: boolean
}
type RuntimeTerminalBufferSnapshot = {
@@ -10012,7 +10013,10 @@ export class OrcaRuntimeService {
ptyId: string,
opts: { scrollbackRows?: number } = {}
): Promise<RuntimeTerminalBufferSnapshot | null> {
const providerSnapshot = await this.serializeProviderTerminalBuffer(ptyId, opts)
const providerSnapshot = await this.serializeProviderTerminalBuffer(ptyId, opts, {
timeoutMs: AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS,
retireOnTimeout: true
})
if (providerSnapshot) {
return providerSnapshot
}
@@ -10579,7 +10583,7 @@ export class OrcaRuntimeService {
private async serializeProviderTerminalBuffer(
ptyId: string,
opts: { scrollbackRows?: number } = {},
wait: { timeoutMs?: number } = {}
wait: { timeoutMs?: number; retireOnTimeout?: boolean } = {}
): Promise<PtyProviderBufferSnapshot | null> {
const generation = this.getPtyLifecycleGeneration(ptyId)
const scrollbackRows = Math.max(0, Math.floor(opts.scrollbackRows ?? 0))
@@ -10590,7 +10594,7 @@ export class OrcaRuntimeService {
acquisition.scrollbackRows < scrollbackRows
) {
const promise = this.captureProviderTerminalBuffer(ptyId, opts, generation)
acquisition = { generation, scrollbackRows, promise }
acquisition = { generation, scrollbackRows, promise, timedOut: false }
this.providerBufferAcquisitionsByPtyId.set(ptyId, acquisition)
void promise.finally(() => {
if (this.providerBufferAcquisitionsByPtyId.get(ptyId) === acquisition) {
@@ -10598,9 +10602,26 @@ export class OrcaRuntimeService {
}
})
}
return typeof wait.timeoutMs === 'number'
? withTimeout(acquisition.promise, wait.timeoutMs, null)
: acquisition.promise
if (acquisition.timedOut) {
return null
}
if (typeof wait.timeoutMs !== 'number') {
return acquisition.promise
}
const result = await withTimeout<
{ settled: true; value: PtyProviderBufferSnapshot | null } | { settled: false }
>(
acquisition.promise.then((value) => ({ settled: true as const, value })),
wait.timeoutMs,
{ settled: false as const }
)
if (!result.settled) {
if (wait.retireOnTimeout) {
acquisition.timedOut = true
}
return null
}
return result.value
}
private async captureProviderTerminalBuffer(
@@ -32504,6 +32525,7 @@ const MAX_TAIL_PENDING_ANSI_CHARS = 4096
const DEFAULT_TERMINAL_READ_LIMIT = 120
const MAX_TERMINAL_READ_LIMIT = 2000
const MAX_TERMINAL_PREVIEW_CHARS = 32 * 1024
export const AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS = 8_000
const VISIBLE_TERMINAL_SNAPSHOT_TIMEOUT_MS = 750
const VISIBLE_TERMINAL_SNAPSHOT_RETRY_MS = 1_000
const MAX_PREVIEW_LINES = 6
@@ -1508,6 +1508,8 @@ describe('createRemoteRuntimePtyTransport', () => {
})
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
const oldStreamId = latestSubscribePayload().streamId
emitSnapshot(oldStreamId, 'before restart')
expect(transport.isConnected()).toBe(true)
runtimeCall.mockImplementation(async (args: { method: string }) =>
args.method === 'session.tabs.list' ? new Promise(() => {}) : { ok: true, result: {} }
@@ -1516,10 +1518,17 @@ describe('createRemoteRuntimePtyTransport', () => {
ok: true,
result: { type: 'end', streamId: oldStreamId, code: 0 }
})
const replacementSnapshot = transport.serializeBuffer?.({ scrollbackRows: 5000 })
let snapshotSettled = false
void replacementSnapshot?.then(() => {
snapshotSettled = true
})
await Promise.resolve()
expect(onExit).not.toHaveBeenCalled()
expect(onPtyExit).not.toHaveBeenCalled()
expect(transport.getPtyId()).toBe('remote:hub-env@@terminal-1')
expect(snapshotSettled).toBe(false)
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
@@ -1557,6 +1566,39 @@ describe('createRemoteRuntimePtyTransport', () => {
expect(onPtySpawn).not.toHaveBeenCalled()
expect(onPtyExit).not.toHaveBeenCalled()
expect(onExit).not.toHaveBeenCalled()
emitSnapshot(latestSubscribePayload().streamId, 'replacement initial state')
await vi.waitFor(() =>
expect(latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)).toBeDefined()
)
const requestFrame = latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)
const request = requestFrame
? decodeTerminalStreamJson<{ requestId?: number }>(requestFrame.payload)
: null
emitSnapshotFrame(
latestSubscribePayload().streamId,
TerminalStreamOpcode.SnapshotStart,
encodeTerminalStreamJson({
kind: 'scrollback',
requestId: request?.requestId,
cols: 100,
rows: 30
})
)
emitSnapshotFrame(
latestSubscribePayload().streamId,
TerminalStreamOpcode.SnapshotChunk,
encodeTerminalStreamText('replacement authoritative state')
)
emitSnapshotFrame(
latestSubscribePayload().streamId,
TerminalStreamOpcode.SnapshotEnd,
new Uint8Array()
)
await expect(replacementSnapshot).resolves.toMatchObject({
data: 'replacement authoritative state',
cols: 100,
rows: 30
})
})
it('coalesces concurrent stale errors for the handle that was replaced', async () => {
@@ -169,8 +169,19 @@ export function createRemoteRuntimePtyTransport(
function setAttachmentReady(ready: boolean): void {
attachmentReady = ready
if (!ready) {
return
}
for (const resolve of attachmentReadyWaiters) {
resolve(ready)
resolve(true)
}
attachmentReadyWaiters.clear()
}
function setAttachmentUnavailable(): void {
attachmentReady = false
for (const resolve of attachmentReadyWaiters) {
resolve(false)
}
attachmentReadyWaiters.clear()
}
@@ -1095,6 +1106,7 @@ export function createRemoteRuntimePtyTransport(
handle = null
remotePtyId = null
closeMultiplexedStream()
setAttachmentUnavailable()
emitRecoveryState()
if (stalePtyId) {
onPtyExit?.(stalePtyId)
@@ -1419,6 +1431,7 @@ export function createRemoteRuntimePtyTransport(
}
outputProcessor.clearAccumulatedState()
if (tabId && isWebTerminalSurfaceTabId(tabId)) {
setAttachmentReady(false)
multiplexedStream = null
multiplexedStreamHandle = null
clearPendingViewportClaim()
@@ -1433,7 +1446,7 @@ export function createRemoteRuntimePtyTransport(
remotePtyId = null
multiplexedStream = null
multiplexedStreamHandle = null
setAttachmentReady(false)
setAttachmentUnavailable()
terminalEnded = true
clearPendingViewportClaim()
emitRecoveryState()
@@ -1481,6 +1494,7 @@ export function createRemoteRuntimePtyTransport(
} else {
connecting = false
recovery.cancel()
setAttachmentUnavailable()
emitRecoveryState()
}
}
@@ -1860,6 +1874,7 @@ export function createRemoteRuntimePtyTransport(
const id = remotePtyId
unregisterShutdownHandlers(id)
closeMultiplexedStream()
setAttachmentUnavailable()
handle = null
remotePtyId = null
emitRecoveryState()
@@ -1885,6 +1900,7 @@ export function createRemoteRuntimePtyTransport(
connecting = false
clearPendingViewportClaim()
closeMultiplexedStream()
setAttachmentUnavailable()
emitRecoveryState()
storedCallbacks = {}
},
@@ -2033,16 +2049,15 @@ export function createRemoteRuntimePtyTransport(
if (!connected || !handle) {
return null
}
const targetHandle = handle
if (!(await waitForAttachmentReady()) || handle !== targetHandle) {
if (!(await waitForAttachmentReady()) || !handle) {
return null
}
return getCurrentMultiplexedStream(targetHandle)?.serializeBuffer(opts) ?? null
return getCurrentMultiplexedStream(handle)?.serializeBuffer(opts) ?? null
},
destroy() {
destroyed = true
setAttachmentReady(false)
setAttachmentUnavailable()
this.disconnect()
recovery.dispose()
inputBatcher.clear()
@@ -0,0 +1,131 @@
import { isTerminalLeafId } from '../../../../shared/stable-pane-id'
import { isRemoteRuntimePtyId, sendRuntimePtyInput } from '@/runtime/runtime-terminal-inspection'
import { useAppStore } from '@/store'
import { closeTerminalTab } from '../terminal/terminal-tab-actions'
import { startParkedTerminalByteWatcher } from './parked-terminal-byte-watcher'
import { subscribeToPtyExit } from './pty-dispatcher'
import { discardPreHandlerPtyState } from './pty-pre-handler-buffer'
import { detachTerminalLayoutLeaf } from './terminal-layout-leaf-detach'
import {
isParkRestorableTerminalPty,
type TerminalParkRestorePolicy
} from './terminal-hidden-view-parking'
import type { ParkableTerminalTabModel } from './terminal-parked-watcher-reconciliation'
import {
resolveTabTitleAfterPaneClose,
shouldClearLaunchAgentForClosedPane
} from './terminal-pane-close-identity'
import {
capturedPanesByTabId,
parkedWatchersByTabId,
type ParkedTabWatcherEntry,
type ParkedTerminalPaneCapture
} from './terminal-parked-watcher-registry'
export function startParkedPtyWatcher(args: {
worktreeId: string
tab: ParkableTerminalTabModel
pane: ParkedTerminalPaneCapture
entry: ParkedTabWatcherEntry
restoreTitleOnRegister: boolean
restorePolicy: TerminalParkRestorePolicy
}): void {
const { worktreeId, tab, pane, entry, restoreTitleOnRegister, restorePolicy } = args
const state = useAppStore.getState()
const ptyId = pane.ptyId
// Why: the tab model can change after the park decision, and legacy leaf ids make pane keys throw.
if (
!ptyId ||
entry.disposersByPtyId.has(ptyId) ||
!isTerminalLeafId(pane.leafId) ||
!isParkRestorableTerminalPty(ptyId, worktreeId, restorePolicy)
) {
return
}
const handlePtyExit = (_code: number, { hadPrimary }: { hadPrimary: boolean }): void => {
useAppStore.getState().clearRuntimePaneTitle(tab.id, pane.paneId)
if (entry.disposersByPtyId.size > 1) {
discardPreHandlerPtyState(ptyId)
collapseParkedExitedLeaf(tab.id, ptyId)
entry.disposersByPtyId.get(ptyId)?.()
entry.disposersByPtyId.delete(ptyId)
return
}
if (hadPrimary) {
entry.disposersByPtyId.get(ptyId)?.()
entry.disposersByPtyId.delete(ptyId)
return
}
// Why: the empty entry prevents a pending pinned-close confirmation from restarting the dead PTY.
entry.disposersByPtyId.get(ptyId)?.()
entry.disposersByPtyId.delete(ptyId)
closeTerminalTab(tab.id, {
captureRecentlyClosed: false,
hostCloseReason: 'pty-exit',
lifecyclePtyId: ptyId,
onClosed: () => {
discardPreHandlerPtyState(ptyId)
if (parkedWatchersByTabId.get(tab.id) === entry) {
parkedWatchersByTabId.delete(tab.id)
}
},
onCancel: () => {}
})
}
const initialTitle = state.runtimePaneTitlesByTabId[tab.id]?.[pane.paneId]
const disposeWatcher = startParkedTerminalByteWatcher({
ptyId,
tabId: tab.id,
worktreeId,
leafId: pane.leafId,
paneId: pane.paneId,
drivesTabTitle: pane.drivesTabTitle,
...(initialTitle !== undefined ? { initialTitle } : {}),
...(restoreTitleOnRegister ? { restoreTitleOnRegister: true } : {}),
sendInput: (data) => {
sendRuntimePtyInput(useAppStore.getState().settings, ptyId, data)
}
})
const unsubscribeExit = isRemoteRuntimePtyId(ptyId)
? () => {}
: subscribeToPtyExit(ptyId, handlePtyExit)
entry.paneIdByPtyId.set(ptyId, pane.paneId)
entry.disposersByPtyId.set(ptyId, () => {
unsubscribeExit()
disposeWatcher()
})
}
export function collapseParkedExitedLeaf(tabId: string, ptyId: string): void {
const state = useAppStore.getState()
const layout = state.terminalLayoutsByTabId[tabId]
const leafId =
capturedPanesByTabId.get(tabId)?.panes.find((pane) => pane.ptyId === ptyId)?.leafId ??
Object.entries(layout?.ptyIdsByLeafId ?? {}).find(([, boundPtyId]) => boundPtyId === ptyId)?.[0]
if (!leafId) {
return
}
const detached = detachTerminalLayoutLeaf(layout, leafId)
if (!detached) {
return
}
const terminalTab = Object.values(state.tabsByWorktree)
.flat()
.find((candidate) => candidate.id === tabId)
if (shouldClearLaunchAgentForClosedPane(terminalTab, ptyId)) {
state.clearTabLaunchAgent(tabId)
}
state.setTabLayout(tabId, detached.sourceLayout)
const activeLeafId = detached.sourceLayout.activeLeafId
const activePtyId = activeLeafId
? detached.sourceLayout.ptyIdsByLeafId?.[activeLeafId]
: undefined
const activePaneId = activePtyId
? (parkedWatchersByTabId.get(tabId)?.paneIdByPtyId.get(activePtyId) ?? null)
: null
state.updateTabTitle(
tabId,
resolveTabTitleAfterPaneClose(state.runtimePaneTitlesByTabId[tabId] ?? {}, activePaneId)
)
}
@@ -86,6 +86,7 @@ type MockStoreState = {
>
clearTabLaunchAgent: ReturnType<typeof vi.fn>
clearRuntimePaneTitle: ReturnType<typeof vi.fn>
setRuntimePaneTitle: ReturnType<typeof vi.fn>
setTabLayout: ReturnType<typeof vi.fn>
updateTabTitle: ReturnType<typeof vi.fn>
}
@@ -148,6 +149,7 @@ describe('terminal-parked-tab-watchers', () => {
runtimeStatusByEnvironmentId: new Map(),
clearTabLaunchAgent: vi.fn(),
clearRuntimePaneTitle: vi.fn(),
setRuntimePaneTitle: vi.fn(),
setTabLayout: vi.fn(),
updateTabTitle: vi.fn()
}
@@ -241,7 +243,8 @@ describe('terminal-parked-tab-watchers', () => {
expect(startedWatchers[0].options).toMatchObject({
ptyId: 'remote:env-1@@terminal-1'
})
expect(startedWatchers[0].options).not.toHaveProperty('onPtyExit')
expect(subscribeToPtyExit).not.toHaveBeenCalled()
expect(exitSubscriptions).toEqual([])
})
it('starts watchers for SSH PTYs (C1 SSH parking, default on)', () => {
@@ -7,12 +7,7 @@
*/
import { isTerminalLeafId } from '../../../../shared/stable-pane-id'
import { useAppStore } from '@/store'
import { isRemoteRuntimePtyId, sendRuntimePtyInput } from '@/runtime/runtime-terminal-inspection'
import { closeTerminalTab } from '../terminal/terminal-tab-actions'
import { detachTerminalLayoutLeaf } from './terminal-layout-leaf-detach'
import { subscribeToPtyExit } from './pty-dispatcher'
import { discardPreHandlerPtyState } from './pty-pre-handler-buffer'
import { startParkedTerminalByteWatcher } from './parked-terminal-byte-watcher'
import {
isParkRestorableTerminalPty,
selectPairedRuntimeParkingEnvironmentIds,
@@ -23,14 +18,13 @@ import {
resolveParkedTerminalPaneCandidates,
type ParkableTerminalTabModel
} from './terminal-parked-watcher-reconciliation'
import {
resolveTabTitleAfterPaneClose,
shouldClearLaunchAgentForClosedPane
} from './terminal-pane-close-identity'
import { collapseParkedExitedLeaf, startParkedPtyWatcher } from './terminal-parked-pty-watcher'
import {
capturedPanesByTabId,
disposeParkedTabWatchers,
parkedWatchersByTabId
parkedWatchersByTabId,
type ParkedTabWatcherEntry,
type ParkedTerminalPaneCapture
} from './terminal-parked-watcher-registry'
// Why: re-export so callers keep one import surface; the registry split only breaks the store-slice import cycle.
@@ -102,94 +96,99 @@ function startParkedTabWatchers(
tab: ParkableTerminalTabModel,
restoreTitleOnRegister: boolean
): void {
const state = useAppStore.getState()
const panes = resolveParkedTerminalPaneCandidates(tab, state)
const restorePolicy = parkRestorePolicyFromState(state)
const disposersByPtyId = new Map<string, () => void>()
const paneIdByPtyId = new Map<string, number>()
for (const pane of panes) {
const ptyId = pane.ptyId
// Why: re-guard — the tab model can change after the park decision, and legacy non-UUID leaf ids make makePaneKey throw.
if (
!ptyId ||
disposersByPtyId.has(ptyId) ||
!isTerminalLeafId(pane.leafId) ||
!isParkRestorableTerminalPty(ptyId, worktreeId, restorePolicy)
) {
continue
}
const handlePtyExit = (_code: number, { hadPrimary }: { hadPrimary: boolean }): void => {
// Why: while parked this sidecar is the only exit observer, so teardown must run here or dead leaves resurrect on reveal.
useAppStore.getState().clearRuntimePaneTitle(tab.id, pane.paneId)
if (disposersByPtyId.size > 1) {
// Why: a parked PaneManager is gone, so its retained primary cannot remove a dead split leaf from persisted layout.
discardPreHandlerPtyState(ptyId)
collapseParkedExitedLeaf(tab.id, ptyId)
disposersByPtyId.get(ptyId)?.()
disposersByPtyId.delete(ptyId)
return
}
if (hadPrimary) {
// Why: the sole pane's primary owner closes its tab; retire the sidecar to avoid duplicate confirmation.
disposersByPtyId.get(ptyId)?.()
disposersByPtyId.delete(ptyId)
return
}
// Why: keep the empty entry so a pending pinned-close confirm can't let parking restart a watcher on the dead PTY.
disposersByPtyId.get(ptyId)?.()
disposersByPtyId.delete(ptyId)
closeTerminalTab(tab.id, {
// Why: autonomous PTY exit still needs pinned-tab confirmation but must not enter reopen history.
captureRecentlyClosed: false,
// Why: same lifecycle echo as the mounted pty-exit handlers — tag the
// wire so the host can refuse it while its PTY is live, without
// `reason: 'pty-exit'` skipping the pinned confirmation above.
hostCloseReason: 'pty-exit',
lifecyclePtyId: ptyId,
onClosed: () => {
discardPreHandlerPtyState(ptyId)
const entry = parkedWatchersByTabId.get(tab.id)
if (entry?.disposersByPtyId === disposersByPtyId) {
parkedWatchersByTabId.delete(tab.id)
}
},
// Why: cancellation keeps the buffered final frame/exit for the reveal-mounted pane.
onCancel: () => {}
})
}
const initialTitle = state.runtimePaneTitlesByTabId[tab.id]?.[pane.paneId]
const disposeWatcher = startParkedTerminalByteWatcher({
ptyId,
tabId: tab.id,
worktreeId,
leafId: pane.leafId,
paneId: pane.paneId,
drivesTabTitle: pane.drivesTabTitle,
...(initialTitle !== undefined ? { initialTitle } : {}),
...(restoreTitleOnRegister ? { restoreTitleOnRegister: true } : {}),
sendInput: (data) => {
sendRuntimePtyInput(useAppStore.getState().settings, ptyId, data)
}
})
// Why: the paired host retires exited surfaces authoritatively; local/SSH
// exits still arrive through the renderer's singleton main channel.
const unsubscribeExit = isRemoteRuntimePtyId(ptyId)
? () => {}
: subscribeToPtyExit(ptyId, handlePtyExit)
paneIdByPtyId.set(ptyId, pane.paneId)
disposersByPtyId.set(ptyId, () => {
unsubscribeExit()
disposeWatcher()
})
}
// Why: track even with zero watchers so window.__terminalParkingDebug reflects every parked tab.
parkedWatchersByTabId.set(tab.id, {
const entry: ParkedTabWatcherEntry = {
worktreeId,
tabPtyId: tab.ptyId,
paneIdByPtyId,
disposersByPtyId
paneIdByPtyId: new Map(),
disposersByPtyId: new Map()
}
parkedWatchersByTabId.set(tab.id, entry)
const restorePolicy = parkRestorePolicyFromState(useAppStore.getState())
for (const pane of resolveParkedTerminalPaneCandidates(tab, useAppStore.getState())) {
startParkedPtyWatcher({
worktreeId,
tab,
pane,
entry,
restoreTitleOnRegister,
restorePolicy
})
}
}
function reconcileParkedTabWatchers(
worktreeId: string,
tab: ParkableTerminalTabModel,
entry: ParkedTabWatcherEntry,
restoreTitleOnRegister: boolean
): void {
const state = useAppStore.getState()
const expectedPanes = watchablePanes(worktreeId, tab)
const expectedPtyIds = new Set(expectedPanes.keys())
const reconciliation = reconcileParkedWatcherPtyIds({
currentTabPtyId: tab.ptyId,
entryTabPtyId: entry.tabPtyId,
paneIdByPtyId: entry.paneIdByPtyId,
expectedPtyIds
})
if (reconciliation.restartAll) {
const retainedTitles = reconciliation.retainedPtyIds.flatMap((ptyId) => {
const paneId = entry.paneIdByPtyId.get(ptyId)
const title =
paneId === undefined ? undefined : state.runtimePaneTitlesByTabId[tab.id]?.[paneId]
return paneId !== undefined && title !== undefined ? [{ paneId, title }] : []
})
for (const paneId of reconciliation.retiredPaneIds) {
state.clearRuntimePaneTitle(tab.id, paneId)
}
disposeParkedTabWatchers(tab.id)
for (const { paneId, title } of retainedTitles) {
useAppStore.getState().setRuntimePaneTitle(tab.id, paneId, title)
}
startParkedTabWatchers(worktreeId, tab, restoreTitleOnRegister)
return
}
for (const [ptyId, paneId] of Array.from(entry.paneIdByPtyId)) {
if (expectedPtyIds.has(ptyId)) {
continue
}
entry.paneIdByPtyId.delete(ptyId)
const dispose = entry.disposersByPtyId.get(ptyId)
entry.disposersByPtyId.delete(ptyId)
dispose?.()
state.clearRuntimePaneTitle(tab.id, paneId)
}
const restorePolicy = parkRestorePolicyFromState(useAppStore.getState())
for (const ptyId of reconciliation.addedPtyIds) {
const pane = expectedPanes.get(ptyId)
if (pane) {
startParkedPtyWatcher({
worktreeId,
tab,
pane,
entry,
restoreTitleOnRegister,
restorePolicy
})
}
}
}
function watchablePanes(
worktreeId: string,
tab: ParkableTerminalTabModel
): Map<string, ParkedTerminalPaneCapture> {
const state = useAppStore.getState()
const restorePolicy = parkRestorePolicyFromState(state)
return new Map(
resolveParkedTerminalPaneCandidates(tab, state).flatMap((pane) =>
pane.ptyId &&
isTerminalLeafId(pane.leafId) &&
isParkRestorableTerminalPty(pane.ptyId, worktreeId, restorePolicy)
? [[pane.ptyId, pane] as const]
: []
)
)
}
/**
@@ -225,39 +224,6 @@ export function shouldDeferParkedPtyExitTabClose(tabId: string, ptyId: string):
return defer
}
// Why: collapse the leaf from the stored layout so reveal can't reattach and resurrect the exited shell.
function collapseParkedExitedLeaf(tabId: string, ptyId: string): void {
const state = useAppStore.getState()
const layout = state.terminalLayoutsByTabId[tabId]
const leafId =
capturedPanesByTabId.get(tabId)?.panes.find((pane) => pane.ptyId === ptyId)?.leafId ??
Object.entries(layout?.ptyIdsByLeafId ?? {}).find(([, boundPtyId]) => boundPtyId === ptyId)?.[0]
if (!leafId) {
return
}
const detached = detachTerminalLayoutLeaf(layout, leafId)
if (detached) {
const terminalTab = Object.values(state.tabsByWorktree)
.flat()
.find((candidate) => candidate.id === tabId)
if (shouldClearLaunchAgentForClosedPane(terminalTab, ptyId)) {
state.clearTabLaunchAgent(tabId)
}
state.setTabLayout(tabId, detached.sourceLayout)
const activeLeafId = detached.sourceLayout.activeLeafId
const activePtyId = activeLeafId
? detached.sourceLayout.ptyIdsByLeafId?.[activeLeafId]
: undefined
const activePaneId = activePtyId
? (parkedWatchersByTabId.get(tabId)?.paneIdByPtyId.get(activePtyId) ?? null)
: null
state.updateTabTitle(
tabId,
resolveTabTitleAfterPaneClose(state.runtimePaneTitlesByTabId[tabId] ?? {}, activePaneId)
)
}
}
function disposeClosedParkedTabWatchers(
tabId: string,
entry: { paneIdByPtyId: ReadonlyMap<string, number> }
@@ -272,20 +238,6 @@ function disposeClosedParkedTabWatchers(
disposeParkedTabWatchers(tabId)
}
function watchablePtyIds(worktreeId: string, tab: ParkableTerminalTabModel): Set<string> {
const state = useAppStore.getState()
const restorePolicy = parkRestorePolicyFromState(state)
return new Set(
resolveParkedTerminalPaneCandidates(tab, state).flatMap((pane) =>
pane.ptyId &&
isTerminalLeafId(pane.leafId) &&
isParkRestorableTerminalPty(pane.ptyId, worktreeId, restorePolicy)
? [pane.ptyId]
: []
)
)
}
/**
* Reconciles watchers for one worktree against its rendered parked set.
* Run from an effect keyed on committed render state so disposal shares the
@@ -322,27 +274,11 @@ export function syncParkedTerminalTabWatchers(args: {
continue
}
const entry = parkedWatchersByTabId.get(tab.id)
const expectedPtyIds = watchablePtyIds(args.worktreeId, tab)
const reconciliation = entry
? reconcileParkedWatcherPtyIds({
currentTabPtyId: tab.ptyId,
entryTabPtyId: entry.tabPtyId,
paneIdByPtyId: entry.paneIdByPtyId,
expectedPtyIds
})
: null
if (entry && reconciliation?.restart) {
for (const paneId of reconciliation.retiredPaneIds) {
useAppStore.getState().clearRuntimePaneTitle(tab.id, paneId)
}
disposeParkedTabWatchers(tab.id)
}
if (!parkedWatchersByTabId.has(tab.id)) {
startParkedTabWatchers(
args.worktreeId,
tab,
args.restoreTitleOnStartTabIds?.has(tab.id) === true
)
const restoreTitleOnRegister = args.restoreTitleOnStartTabIds?.has(tab.id) === true
if (entry) {
reconcileParkedTabWatchers(args.worktreeId, tab, entry, restoreTitleOnRegister)
} else {
startParkedTabWatchers(args.worktreeId, tab, restoreTitleOnRegister)
}
}
}
@@ -0,0 +1,111 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import type {
ParkedTabWatcherEntry,
ParkedTerminalPaneCapture
} from './terminal-parked-watcher-registry'
const WORKTREE_ID = 'repo::/worktree'
const TAB_ID = 'tab-1'
const FIRST_PTY_ID = `${WORKTREE_ID}@@session-1`
const OLD_SECOND_PTY_ID = `${WORKTREE_ID}@@session-2`
const NEW_SECOND_PTY_ID = `${WORKTREE_ID}@@session-3`
const FIRST_LEAF_ID = '11111111-1111-4111-8111-111111111111'
const SECOND_LEAF_ID = '22222222-2222-4222-8222-222222222222'
const startedWatchers: {
pane: ParkedTerminalPaneCapture
dispose: ReturnType<typeof vi.fn>
}[] = []
vi.mock('./terminal-parked-pty-watcher', () => ({
collapseParkedExitedLeaf: vi.fn(),
startParkedPtyWatcher: (args: {
pane: ParkedTerminalPaneCapture
entry: ParkedTabWatcherEntry
}) => {
if (!args.pane.ptyId) {
return
}
const dispose = vi.fn()
startedWatchers.push({ pane: args.pane, dispose })
args.entry.paneIdByPtyId.set(args.pane.ptyId, args.pane.paneId)
args.entry.disposersByPtyId.set(args.pane.ptyId, dispose)
}
}))
vi.mock('./pty-pre-handler-buffer', () => ({
discardPreHandlerPtyState: vi.fn()
}))
const state = {
tabsByWorktree: {},
terminalLayoutsByTabId: {} as Record<string, object>,
runtimePaneTitlesByTabId: {} as Record<string, Record<number, string>>,
settings: null,
runtimeStatusByEnvironmentId: new Map(),
clearRuntimePaneTitle: vi.fn(),
setRuntimePaneTitle: vi.fn()
}
vi.mock('@/store', () => ({
useAppStore: { getState: () => state }
}))
import {
captureParkedTerminalPaneCandidates,
pruneParkedTerminalWatchers,
syncParkedTerminalTabWatchers
} from './terminal-parked-tab-watchers'
function sync(): void {
syncParkedTerminalTabWatchers({
worktreeId: WORKTREE_ID,
tabs: [{ id: TAB_ID, ptyId: FIRST_PTY_ID }],
parkedTabIds: new Set([TAB_ID])
})
}
beforeEach(() => {
state.terminalLayoutsByTabId = {}
state.runtimePaneTitlesByTabId = { [TAB_ID]: { 1: '⠋ Continuing agent', 2: 'Retired shell' } }
startedWatchers.length = 0
vi.clearAllMocks()
})
afterEach(() => {
pruneParkedTerminalWatchers(new Set())
})
it('retains a continuing watcher and title while reconciling a reminted split leaf', () => {
captureParkedTerminalPaneCandidates(TAB_ID, WORKTREE_ID, [
{ ptyId: FIRST_PTY_ID, paneId: 1, leafId: FIRST_LEAF_ID, drivesTabTitle: true },
{ ptyId: OLD_SECOND_PTY_ID, paneId: 2, leafId: SECOND_LEAF_ID, drivesTabTitle: false }
])
sync()
state.terminalLayoutsByTabId[TAB_ID] = {
root: {
type: 'split',
direction: 'vertical',
first: { type: 'leaf', leafId: FIRST_LEAF_ID },
second: { type: 'leaf', leafId: SECOND_LEAF_ID }
},
activeLeafId: FIRST_LEAF_ID,
expandedLeafId: null,
ptyIdsByLeafId: {
[FIRST_LEAF_ID]: FIRST_PTY_ID,
[SECOND_LEAF_ID]: NEW_SECOND_PTY_ID
}
}
sync()
expect(startedWatchers[0].dispose).not.toHaveBeenCalled()
expect(startedWatchers[1].dispose).toHaveBeenCalledOnce()
expect(startedWatchers[2].pane).toMatchObject({
ptyId: NEW_SECOND_PTY_ID,
paneId: 2,
drivesTabTitle: false
})
expect(state.clearRuntimePaneTitle).toHaveBeenCalledWith(TAB_ID, 2)
expect(state.clearRuntimePaneTitle).not.toHaveBeenCalledWith(TAB_ID, 1)
})
@@ -66,7 +66,7 @@ describe('paired parked-watcher reconciliation', () => {
])
})
it('restarts the watcher set and identifies the retired title slot', () => {
it('surgically reconciles a reminted split leaf without restarting its sibling', () => {
expect(
reconcileParkedWatcherPtyIds({
currentTabPtyId: FIRST_PTY_ID,
@@ -77,6 +77,11 @@ describe('paired parked-watcher reconciliation', () => {
]),
expectedPtyIds: new Set([FIRST_PTY_ID, NEW_SECOND_PTY_ID])
})
).toEqual({ restart: true, retiredPaneIds: [2] })
).toEqual({
restartAll: false,
addedPtyIds: [NEW_SECOND_PTY_ID],
retainedPtyIds: [FIRST_PTY_ID],
retiredPaneIds: [2]
})
})
})
@@ -71,12 +71,19 @@ export function reconcileParkedWatcherPtyIds(args: {
entryTabPtyId: string | null
paneIdByPtyId: ReadonlyMap<string, number>
expectedPtyIds: ReadonlySet<string>
}): { restart: boolean; retiredPaneIds: number[] } {
const samePtyIds =
args.paneIdByPtyId.size === args.expectedPtyIds.size &&
Array.from(args.paneIdByPtyId.keys()).every((ptyId) => args.expectedPtyIds.has(ptyId))
}): {
restartAll: boolean
addedPtyIds: string[]
retainedPtyIds: string[]
retiredPaneIds: number[]
} {
const retainedPtyIds = Array.from(args.paneIdByPtyId.keys()).filter((ptyId) =>
args.expectedPtyIds.has(ptyId)
)
return {
restart: args.entryTabPtyId !== args.currentTabPtyId || !samePtyIds,
restartAll: args.entryTabPtyId !== args.currentTabPtyId,
addedPtyIds: Array.from(args.expectedPtyIds).filter((ptyId) => !args.paneIdByPtyId.has(ptyId)),
retainedPtyIds,
retiredPaneIds: Array.from(args.paneIdByPtyId)
.filter(([ptyId]) => !args.expectedPtyIds.has(ptyId))
.map(([, paneId]) => paneId)
@@ -150,7 +150,7 @@ type RemoteRuntimeSnapshotRequest = {
const CONTROL_STREAM_ID = 0
const MAX_REMOTE_TERMINAL_SNAPSHOT_BYTES = 2 * 1024 * 1024
const REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS = 10_000
export const REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS = 10_000
const REMOTE_TERMINAL_RESYNC_TIMEOUT_MS = 10_000
// Why: a truncated recovery means the server is too flooded to serialize;
// retrying once per incoming chunk would stampede it, so back off instead.
@@ -297,19 +297,25 @@ class RemoteRuntimeTerminalMultiplexer {
resyncAttempts: 0,
capacityRejected: false,
watchdog: createRemoteTerminalStreamWatchdog((stall) => {
if (!e2eDisableRemoteTerminalStallRecovery) {
recordRendererCrashBreadcrumb('remote_terminal_stream_stall_recovery', {
environmentId: this.environmentId,
expectedSeq: state.expectedSeq ?? null,
inactiveForMs: stall.inactiveForMs,
outstandingDeliveryBytes: stall.outstandingDeliveryBytes,
pendingAckBytes: state.pendingAckBytes,
reason: stall.reason,
resyncAttempts: state.resyncAttempts,
snapshotPending: state.pendingSnapshotRequest !== null,
streamId: state.streamId,
terminal: state.terminal
})
if (e2eDisableRemoteTerminalStallRecovery) {
state.watchdog.completeCommandResponseProbe()
return
}
recordRendererCrashBreadcrumb('remote_terminal_stream_stall_recovery', {
environmentId: this.environmentId,
expectedSeq: state.expectedSeq ?? null,
inactiveForMs: stall.inactiveForMs,
outstandingDeliveryBytes: stall.outstandingDeliveryBytes,
pendingAckBytes: state.pendingAckBytes,
reason: stall.reason,
resyncAttempts: state.resyncAttempts,
snapshotPending: state.pendingSnapshotRequest !== null,
streamId: state.streamId,
terminal: state.terminal
})
if (stall.reason === 'command-response-timeout') {
this.probeCommandResponse(state)
} else {
this.recoverStalledStream(state)
}
})
@@ -988,6 +994,28 @@ class RemoteRuntimeTerminalMultiplexer {
return sent
}
private probeCommandResponse(stream: RemoteRuntimeMultiplexedTerminalState): void {
void this.requestSnapshot(stream).then(
() => {
if (this.streams.get(stream.streamId) !== stream) {
return
}
stream.watchdog.completeCommandResponseProbe()
recordRendererCrashBreadcrumb('remote_terminal_stream_stall_probe_succeeded', {
environmentId: this.environmentId,
streamId: stream.streamId,
terminal: stream.terminal
})
},
() => {
// Snapshot timeout owns recovery; an explicit host error already proves liveness.
if (this.streams.get(stream.streamId) === stream) {
stream.watchdog.completeCommandResponseProbe()
}
}
)
}
private recoverStalledStream(stream: RemoteRuntimeMultiplexedTerminalState): void {
if (this.streams.get(stream.streamId) !== stream) {
return
@@ -2,7 +2,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
TerminalStreamOpcode,
decodeTerminalStreamFrame,
decodeTerminalStreamJson,
encodeTerminalStreamFrame,
encodeTerminalStreamJson,
encodeTerminalStreamText
} from '../../../shared/terminal-stream-protocol'
import {
@@ -99,8 +101,8 @@ describe('remote terminal stalled stream recovery', () => {
healthy.close()
})
it('restarts a stream when an entered command receives no frames', async () => {
const { getRemoteRuntimeTerminalMultiplexer } =
it('probes then restarts a stream when an entered command receives no frames', async () => {
const { getRemoteRuntimeTerminalMultiplexer, REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS } =
await import('./remote-runtime-terminal-multiplexer')
const onTransportClose = vi.fn()
const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({
@@ -113,6 +115,10 @@ describe('remote terminal stalled stream recovery', () => {
expect(stream.sendInput('ls\r')).toBe(true)
await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS)
expect(onTransportClose).not.toHaveBeenCalled()
expect(sentFrames(TerminalStreamOpcode.SnapshotRequest)).toHaveLength(1)
await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS)
expect(onTransportClose).toHaveBeenCalledWith({ recoverable: true })
expect(sentUnsubscribeStreamIds()).toEqual([stream.streamId])
expect(recordBreadcrumb).toHaveBeenCalledWith({
@@ -127,6 +133,36 @@ describe('remote terminal stalled stream recovery', () => {
})
})
it('keeps a silent responsive stream after its authoritative snapshot probe', async () => {
const { getRemoteRuntimeTerminalMultiplexer } =
await import('./remote-runtime-terminal-multiplexer')
const onTransportClose = vi.fn()
const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({
terminal: 'term-password',
client: { id: 'mac-viewer', type: 'desktop' },
callbacks: { onData: vi.fn(), onSnapshot: vi.fn(), onTransportClose }
})
sendBinary.mockClear()
expect(stream.sendInput('secret\r')).toBe(true)
await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS)
const firstRequest = sentFrames(TerminalStreamOpcode.SnapshotRequest)[0]
const firstRequestPayload = firstRequest
? decodeTerminalStreamJson<{ requestId: number }>(firstRequest.payload)
: null
expect(firstRequestPayload?.requestId).toBeTypeOf('number')
emitRequestedSnapshot(stream.streamId, firstRequestPayload?.requestId ?? 0)
await vi.advanceTimersByTimeAsync(0)
expect(onTransportClose).not.toHaveBeenCalled()
expect(sentUnsubscribeStreamIds()).toEqual([])
expect(stream.sendInput('silent-command\r')).toBe(true)
await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS)
expect(sentFrames(TerminalStreamOpcode.SnapshotRequest)).toHaveLength(2)
stream.close()
})
it('keeps a command stream when any host frame proves it is responsive', async () => {
const { getRemoteRuntimeTerminalMultiplexer } =
await import('./remote-runtime-terminal-multiplexer')
@@ -204,10 +240,38 @@ describe('remote terminal stalled stream recovery', () => {
)
}
function sentUnsubscribeStreamIds(): number[] {
function emitRequestedSnapshot(streamId: number, requestId: number): void {
callbacks?.onBinary(
encodeTerminalStreamFrame({
opcode: TerminalStreamOpcode.SnapshotStart,
streamId,
seq: 1,
payload: encodeTerminalStreamJson({
kind: 'scrollback',
requestId,
cols: 80,
rows: 24
})
})
)
callbacks?.onBinary(
encodeTerminalStreamFrame({
opcode: TerminalStreamOpcode.SnapshotEnd,
streamId,
seq: 2,
payload: new Uint8Array()
})
)
}
function sentFrames(opcode: TerminalStreamOpcode) {
return sendBinary.mock.calls.flatMap(([bytes]) => {
const frame = decodeTerminalStreamFrame(bytes)
return frame?.opcode === TerminalStreamOpcode.Unsubscribe ? [frame.streamId] : []
return frame?.opcode === opcode ? [frame] : []
})
}
function sentUnsubscribeStreamIds(): number[] {
return sentFrames(TerminalStreamOpcode.Unsubscribe).map((frame) => frame.streamId)
}
})
@@ -9,6 +9,7 @@ export type RemoteTerminalStreamStall = {
export type RemoteTerminalStreamWatchdog = {
beginOutputDelivery: (bytes: number) => () => void
completeCommandResponseProbe: () => void
recordCommandInput: (text: string) => void
recordInbound: () => void
dispose: () => void
@@ -21,6 +22,7 @@ export function createRemoteTerminalStreamWatchdog(
let deliveryTimer: ReturnType<typeof setTimeout> | null = null
let outstandingDeliveryBytes = 0
let lastInboundAtMs = Date.now()
let commandResponseProbePending = false
let disposed = false
const clearResponseTimer = (): void => {
@@ -39,9 +41,13 @@ export function createRemoteTerminalStreamWatchdog(
if (disposed) {
return
}
disposed = true
clearResponseTimer()
clearDeliveryTimer()
if (reason === 'command-response-timeout') {
commandResponseProbePending = true
} else {
disposed = true
clearDeliveryTimer()
}
onStall({
inactiveForMs: Math.max(0, Date.now() - lastInboundAtMs),
outstandingDeliveryBytes,
@@ -75,8 +81,11 @@ export function createRemoteTerminalStreamWatchdog(
armDeliveryTimer()
}
},
completeCommandResponseProbe() {
commandResponseProbePending = false
},
recordCommandInput(text) {
if (disposed || responseTimer || !/[\r\n]/u.test(text)) {
if (disposed || commandResponseProbePending || responseTimer || !/[\r\n]/u.test(text)) {
return
}
responseTimer = setTimeout(
@@ -90,6 +99,7 @@ export function createRemoteTerminalStreamWatchdog(
},
dispose() {
disposed = true
commandResponseProbePending = false
clearResponseTimer()
clearDeliveryTimer()
outstandingDeliveryBytes = 0
+2 -2
View File
@@ -522,12 +522,12 @@ describe('web runtime environment identity', () => {
})
}
await expect(Promise.all(activeCalls)).resolves.toEqual(
expect.arrayContaining([
Array.from({ length: 8 }, () =>
expect.objectContaining({
ok: false,
error: expect.objectContaining({ code: 'runtime_manually_disconnected' })
})
])
)
)
}
)
@@ -132,6 +132,7 @@ export async function launchPairedWebClient(
}
return { page, dispose: () => page?.close() ?? Promise.resolve() }
} catch (error) {
void pagePromise.catch(() => undefined)
await page?.close().catch(() => undefined)
throw error
}
@@ -151,45 +151,54 @@ export async function runPairedTerminalParkingOracle(
expect(baseline.bufferCells).toBeGreaterThan(MIN_STAGED_BUFFER_CELLS)
const lagProbe = await startRendererLagProbe(page)
await page.evaluate(async () => {
await window.__store?.getState().updateSettings({ terminalHiddenViewParking: true })
})
await expect
.poll(
() =>
page.evaluate(
({ tabIds, worktreeIds }) => {
const verdicts = window.__terminalParkingDebug?.worktreeVerdicts() ?? []
return {
forceParked: worktreeIds.map(
(id) => verdicts.find((verdict) => verdict.worktreeId === id)?.forceParked
),
mounted: tabIds.filter((id) => window.__paneManagers?.has(id)).length,
ordinaryParkingCovers: worktreeIds.map(
(id) =>
verdicts.find((verdict) => verdict.worktreeId === id)?.ordinaryParkingCovers
),
parked: window.__terminalParkingDebug?.parkedTabIds().length,
retentionBudgetEnabled:
window.__store?.getState().settings?.terminalHiddenWorktreeRetentionBudget
}
},
{
tabIds: remoteTabs.map((tab) => tab.tabId),
worktreeIds: remoteTabs.map((tab) => tab.worktreeId)
}
),
{ timeout: 10_000 }
)
.toEqual({
forceParked: Array(TARGET_WORKTREE_COUNT).fill(false),
mounted: 1,
ordinaryParkingCovers: Array(TARGET_WORKTREE_COUNT).fill(true),
parked: TARGET_WORKTREE_COUNT - 1,
retentionBudgetEnabled: false
let maxLagMs = Number.POSITIVE_INFINITY
let lagProbeStopped = false
try {
await page.evaluate(async () => {
await window.__store?.getState().updateSettings({ terminalHiddenViewParking: true })
})
const maxLagMs = await lagProbe.evaluate((probe) => probe.stop())
await lagProbe.dispose()
await expect
.poll(
() =>
page.evaluate(
({ tabIds, worktreeIds }) => {
const verdicts = window.__terminalParkingDebug?.worktreeVerdicts() ?? []
return {
forceParked: worktreeIds.map(
(id) => verdicts.find((verdict) => verdict.worktreeId === id)?.forceParked
),
mounted: tabIds.filter((id) => window.__paneManagers?.has(id)).length,
ordinaryParkingCovers: worktreeIds.map(
(id) =>
verdicts.find((verdict) => verdict.worktreeId === id)?.ordinaryParkingCovers
),
parked: window.__terminalParkingDebug?.parkedTabIds().length,
retentionBudgetEnabled:
window.__store?.getState().settings?.terminalHiddenWorktreeRetentionBudget
}
},
{
tabIds: remoteTabs.map((tab) => tab.tabId),
worktreeIds: remoteTabs.map((tab) => tab.worktreeId)
}
),
{ timeout: 10_000 }
)
.toEqual({
forceParked: Array(TARGET_WORKTREE_COUNT).fill(false),
mounted: 1,
ordinaryParkingCovers: Array(TARGET_WORKTREE_COUNT).fill(true),
parked: TARGET_WORKTREE_COUNT - 1,
retentionBudgetEnabled: false
})
maxLagMs = await lagProbe.evaluate((probe) => probe.stop())
lagProbeStopped = true
} finally {
if (!lagProbeStopped) {
await lagProbe.evaluate((probe) => probe.stop()).catch(() => undefined)
}
await lagProbe.dispose()
}
const after = await readPairedRetentionSample(
page,
remoteTabs.map((tab) => tab.tabId)