Stop terminal recovery when a remote host disconnects

This commit is contained in:
Brennan Benson
2026-11-09 02:30:18 -08:00
parent 2ad86f5aac
commit 99e891fc0d
15 changed files with 808 additions and 53 deletions
@@ -51,6 +51,7 @@ export function RemoteHostStatusRow({
const pending = useRef<{ token: symbol; kind: typeof actionKind } | null>(null)
const [busy, setBusy] = useState<typeof actionKind | null>(null)
const [open, setOpen] = useState(false)
const card = useRef<HTMLDivElement>(null)
const reconnectCard =
tone === 'busy' && recovery && (recovery.attempt >= 3 || now - recovery.since >= 30_000)
const hasCard = Boolean(errorSentence || reconnectCard)
@@ -141,10 +142,22 @@ export function RemoteHostStatusRow({
<div className="group flex items-center rounded-md hover:bg-accent focus-within:bg-accent">
{hasCard ? (
<DropdownMenuSub open={open} onOpenChange={setOpen}>
<DropdownMenuSubTrigger className={triggerClass} hideChevron aria-label={rowName}>
<DropdownMenuSubTrigger
className={triggerClass}
hideChevron
aria-label={rowName}
onPointerLeave={(event) => {
if (
event.relatedTarget instanceof Node &&
card.current?.contains(event.relatedTarget)
) {
event.preventDefault()
}
}}
>
{details}
</DropdownMenuSubTrigger>
<DropdownMenuSubContent className="w-[min(18rem,calc(100vw-1rem))] p-1.5">
<DropdownMenuSubContent ref={card} className="w-[min(18rem,calc(100vw-1rem))] p-1.5">
<p className="px-1.5 py-1 text-xs text-muted-foreground">
{errorSentence ??
t(
@@ -247,3 +247,46 @@ it('clears the old unavailable card while Retry is connecting, then shows only e
expect(screen.getByRole('menuitem', { name: 'Retry now' })).toBeTruthy()
expect(screen.queryByRole('menuitem', { name: 'Copy error' })).toBeNull()
})
it('keeps its reconnect submenu actionable when the pointer transfers from the row', async () => {
const retry = vi.fn(async () => undefined)
render(
<TooltipProvider>
<DropdownMenu defaultOpen modal={false}>
<DropdownMenuTrigger>Hosts</DropdownMenuTrigger>
<DropdownMenuContent>
<RemoteHostStatusRow
label="Host"
tone="busy"
status="Reconnecting"
actionLabel="Stop"
actionKind="disconnect"
onAction={vi.fn()}
recovery={{ since: 0, attempt: 3 }}
now={60_000}
onRetry={retry}
/>
</DropdownMenuContent>
</DropdownMenu>
</TooltipProvider>
)
const row = screen.getByRole('menuitem', { name: 'Host, Reconnecting' })
fireEvent.pointerMove(row, { pointerType: 'mouse', clientX: 20, clientY: 20 })
const action = await screen.findByRole('menuitem', { name: 'Retry now' })
act(() => row.focus())
fireEvent.pointerOut(row, {
pointerType: 'mouse',
relatedTarget: action,
clientX: 30,
clientY: 20
})
fireEvent.pointerOver(action, {
pointerType: 'mouse',
relatedTarget: row,
clientX: 31,
clientY: 20
})
await act(async () => fireEvent.click(action))
expect(retry).toHaveBeenCalledTimes(1)
expect(screen.getByRole('menuitem', { name: 'Stop Host' })).toBeTruthy()
})
@@ -7,6 +7,7 @@ import {
CALM_STATUS_MIN_VISIBLE_MS,
CALM_STATUS_REVEAL_DELAY_MS
} from '../../../../../../../shared/calm-status-reveal'
import type { PtyTransport } from '../../pty-transport-types'
import type { TerminalPaneController } from '../../../controller/use-terminal-pane-controller'
import { resetRemoteServerNoticeClaimsForTests } from '../../recovery/server-outage/remote-server-notice-claims'
@@ -18,6 +19,13 @@ vi.mock('@/i18n/i18n', () => ({
fallback.replace(/\{\{(\w+)\}\}/g, (_match, name: string) => options?.[name] ?? ''),
getIntlLocale: () => 'en-US'
}))
const hostLifetime = vi.hoisted(() => ({ retired: false }))
vi.mock('@/store/slices/runtime-status/views/use-runtime-status-view', () => ({
useRuntimeHostRetired: (id: string | null) => Boolean(id && hostLifetime.retired)
}))
vi.mock('@/store/slices/runtime-status/views/runtime-status-subscription-store', () => ({
runtimeStatusSubscriptions: { getHostSnapshot: () => ({ retired: hostLifetime.retired }) }
}))
vi.mock('@/store/index', () => ({
useAppStore: (selector: (state: unknown) => unknown) => selector({ wslHostStatuses })
}))
@@ -39,6 +47,7 @@ vi.mock('../../recovery/server-outage/remote-server-outage', () => ({
}))
afterEach(() => {
hostLifetime.retired = false
outage.environmentId = 'env-1'
delete wslHostStatuses['env-1']
cleanup()
@@ -224,3 +233,75 @@ describe('TerminalPaneRecoveryPortals', () => {
expect(notices()).toHaveLength(1)
})
})
it('releases a held shared notice immediately for the stopped pane action', async () => {
vi.useFakeTimers()
const { TerminalPaneRecoveryPortals } = await import('./TerminalPaneRecoveryPortals')
const pane = document.createElement('div')
document.body.append(pane)
const controller = controllerFor([pane])
const view = render(<TerminalPaneRecoveryPortals controller={controller} />)
act(() => vi.advanceTimersByTime(CALM_STATUS_REVEAL_DELAY_MS * 2))
expect(view.container.querySelector('[data-remote-server-outage-notice]')).not.toBeNull()
outage.environmentId = null
hostLifetime.retired = true
view.rerender(
<TerminalPaneRecoveryPortals
controller={{
...controller,
ptyRecoveryStatesByPaneId: { 1: { phase: 'disconnected', epoch: 2, attempt: 1 } }
}}
/>
)
expect(view.container.querySelector('[data-remote-server-outage-notice]')).toBeNull()
expect(
pane.querySelector('[data-terminal-remote-runtime-reconnect-banner="disconnected"] button')
).not.toBeNull()
})
it.each(['success', 'failure', 'later-stop', 'rebind', 'already-revived'] as const)(
'lifts the main hold before pane retry, with %s settlement',
async (settlement) => {
const { createPaneRecoveryReconnect } = await import('./TerminalPaneRecoveryPortals')
const retry = vi.fn()
let epoch = 3
const stub = {
getRuntimeEnvironmentId: () => 'env-1',
getRecoveryState: () => ({ epoch }),
retryRecovery: retry
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this action reads only environment, epoch and retry from the transport.
const transport = stub as unknown as PtyTransport
let current = transport
let finish: (response: { ok: boolean }) => void = () => {}
const connect = vi.fn(
() =>
new Promise((resolve) => {
finish = resolve
})
)
Object.defineProperty(window, 'api', {
configurable: true,
value: { runtimeEnvironments: { connect } }
})
hostLifetime.retired = true
createPaneRecoveryReconnect(() => current, null)()
expect(connect).toHaveBeenCalledWith({ selector: 'env-1' })
expect(retry).not.toHaveBeenCalled()
if (settlement !== 'failure') {
hostLifetime.retired = false
}
if (settlement === 'later-stop') {
epoch += 1
hostLifetime.retired = true
}
if (settlement === 'already-revived') {
epoch += 1
}
if (settlement === 'rebind') {
current = { ...transport }
}
await act(async () => finish({ ok: settlement !== 'failure' }))
expect(retry).toHaveBeenCalledTimes(settlement === 'success' ? 1 : 0)
}
)
@@ -18,6 +18,8 @@ import { WslHostStoppedCard } from '../../recovery/wsl/WslHostStoppedCard'
import { useRetryPanesWhenWslHostReturns } from '../../recovery/wsl/use-wsl-host-return-retry'
import { useCalmStatusReveal } from '@/hooks/use-calm-status-reveal'
import { useAppStore } from '@/store/index'
import { useRuntimeHostRetired } from '@/store/slices/runtime-status/views/use-runtime-status-view'
import { runtimeStatusSubscriptions } from '@/store/slices/runtime-status/views/runtime-status-subscription-store'
import { paneShowsWslHostStoppedCard } from '@/lib/wsl-host-stopped-status'
import { withoutWslHostStoppedLines } from '../../../../../../../shared/wsl-host-stopped-message'
import { mapPaneTerminalErrors } from '../../../errors/terminal-error-accumulation'
@@ -30,6 +32,26 @@ export function createPaneRecoveryReconnect(
awaitingSignIn = false
): () => void {
return () => {
const transport = getTransport()
const environmentId = transport?.getRuntimeEnvironmentId?.()
if (environmentId && runtimeStatusSubscriptions.getHostSnapshot(environmentId)?.retired) {
const epoch = transport?.getRecoveryState?.().epoch
void window.api.runtimeEnvironments
.connect({ selector: environmentId })
.then((response) => {
if (
response.ok &&
getTransport() === transport &&
transport?.getRuntimeEnvironmentId?.() === environmentId &&
!runtimeStatusSubscriptions.getHostSnapshot(environmentId)?.retired &&
transport.getRecoveryState?.().epoch === epoch
) {
transport.retryRecovery?.()
}
})
.catch(() => undefined)
return
}
const retry = (): void => {
getTransport()?.retryRecovery?.()
}
@@ -82,6 +104,7 @@ export function TerminalPaneRecoveryPortals({
state.openSettingsPage()
}, [pairingRefusedEnvironmentId])
const ownerEnvironmentId = useWorkspaceRuntimeEnvironmentId(controller.worktreeId)
const ownerRetired = useRuntimeHostRetired(ownerEnvironmentId)
// One subscription for both the card and the retry: one listener per mounted surface.
const ownerWslHost = useAppStore((state) =>
ownerEnvironmentId ? state.wslHostStatuses[ownerEnvironmentId] : undefined
@@ -114,6 +137,7 @@ export function TerminalPaneRecoveryPortals({
// Why not isolated panes: an Activity thread shows one pane away from this tab's surface.
const canSpeak =
controller.isVisible &&
!ownerRetired &&
!isolated &&
!showSshReconnectOverlay &&
!awaitingSignInHere &&
@@ -122,8 +146,13 @@ export function TerminalPaneRecoveryPortals({
stoppedWslDistro === null &&
pairingRefusedEnvironmentId === null
const serverDown = retryingOutageEnvironmentId !== null
const notice = useRemoteServerNoticeClaim(ownerEnvironmentId, serverDown, canSpeak)
const notice = useRemoteServerNoticeClaim(
ownerRetired ? null : ownerEnvironmentId,
serverDown,
canSpeak
)
const [noticeEnvironmentId, setNoticeEnvironmentId] = useState<string | null>(null)
const noticeRetired = useRuntimeHostRetired(noticeEnvironmentId)
if (notice.speaks && ownerEnvironmentId !== noticeEnvironmentId) {
setNoticeEnvironmentId(ownerEnvironmentId)
}
@@ -136,7 +165,7 @@ export function TerminalPaneRecoveryPortals({
setHandedOff(true)
}
const noticeReveal =
canSpeak && !handedOff && (notice.speaks || !serverDown)
!noticeRetired && canSpeak && !handedOff && (notice.speaks || !serverDown)
? revealed
: { visible: false, downSince: null }
usePublishRemoteServerNoticeShowing(noticeEnvironmentId, noticeReveal.visible)
@@ -196,6 +225,7 @@ export function TerminalPaneRecoveryPortals({
// Why stopped panes too: a card flashing over the notice's hold is the overlap this avoids.
heldByServer={
!isolated &&
!ownerRetired &&
!awaitingSignIn &&
!wslHostUnavailable &&
pairingRefusedEnvironmentId === null &&
@@ -19,6 +19,13 @@ vi.mock('@/i18n/i18n', () => ({
getIntlLocale: () => 'en-US'
}))
const hostLifetime = vi.hoisted(() => ({ retired: false }))
vi.mock('@/store/slices/runtime-status/views/use-runtime-status-view', () => ({
useRuntimeHostRetired: (id: string | null) => Boolean(id && hostLifetime.retired)
}))
vi.mock('@/store/slices/runtime-status/views/runtime-status-subscription-store', () => ({
runtimeStatusSubscriptions: { getHostSnapshot: () => ({ retired: hostLifetime.retired }) }
}))
vi.mock('@/store/index', () => ({
useAppStore: (selector: (state: unknown) => unknown) =>
selector({
@@ -35,6 +42,7 @@ vi.mock('@/store/index', () => ({
}))
afterEach(() => {
hostLifetime.retired = false
cleanup()
vi.useRealTimers()
})
@@ -282,3 +290,42 @@ describe('CalmRemoteRuntimeReconnectBanner', () => {
expect(screen.getByText('Terminal stopped responding')).toBeInTheDocument()
})
})
it('overrides held pane recovery with its actual stopped outcome at once', () => {
vi.useFakeTimers()
const view = render(
<CalmRemoteRuntimeReconnectBanner
recovery={{ phase: 'backoff', cause: 'connection' }}
heldByServer={false}
environmentId="env-1"
onReconnect={vi.fn()}
/>
)
act(() => vi.advanceTimersByTime(CALM_STATUS_REVEAL_DELAY_MS * 2))
expect(
view.container.querySelector('[data-terminal-remote-runtime-reconnect-banner]')
).not.toBeNull()
hostLifetime.retired = true
view.rerender(
<CalmRemoteRuntimeReconnectBanner
recovery={null}
heldByServer={false}
environmentId="env-1"
onReconnect={vi.fn()}
/>
)
expect(view.container.querySelector('[data-terminal-remote-runtime-reconnect-banner]')).toBeNull()
view.rerender(
<CalmRemoteRuntimeReconnectBanner
recovery={{ phase: 'disconnected' }}
heldByServer={false}
environmentId="env-1"
onReconnect={vi.fn()}
/>
)
expect(
view.container.querySelector(
'[data-terminal-remote-runtime-reconnect-banner="disconnected"] button'
)
).not.toBeNull()
})
@@ -16,6 +16,7 @@ import {
selectRuntimeServerTailscaleHost
} from '../../recovery/server-outage/remote-server-outage'
import { useAppStore } from '@/store/index'
import { useRuntimeHostRetired } from '@/store/slices/runtime-status/views/use-runtime-status-view'
type BannerProps = {
recovery: Pick<VisiblePtyRecoveryState, 'phase' | 'cause' | 'waiting'>
@@ -173,6 +174,7 @@ export function CalmRemoteRuntimeReconnectBanner({
heldByServer: boolean
environmentId: string | null
}): React.JSX.Element | null {
const hostRetired = useRuntimeHostRetired(environmentId)
const serverName = useAppStore((state) =>
environmentId ? selectRuntimeServerName(state, environmentId) : fallbackRemoteServerName()
)
@@ -202,7 +204,13 @@ export function CalmRemoteRuntimeReconnectBanner({
const reveal = useCalmStatusReveal(active && !heldByServer, needsAction, recoveringSince)
const shown = recovery ?? lastRecovery
// Why a resolved action card goes at once: its copy would describe a state that no longer holds.
if (heldByServer || !reveal.visible || !shown || (!active && lastNeededAction)) {
if (
(hostRetired && recovery?.phase !== 'disconnected') ||
heldByServer ||
!reveal.visible ||
!shown ||
(!active && lastNeededAction)
) {
return null
}
return (
@@ -263,6 +263,18 @@ export class RemoteRuntimePtyRecoveryState {
this.onChange?.()
}
/** Host retirement ends this epoch; only a current successor or explicit Connect can resume it. */
suspend(): void {
if (this.phase === 'disposed') {
return
}
this.clearOwnerRefresh()
this.epoch += 1
this.clearTimers()
this.phase = 'disconnected'
this.onChange?.()
}
cancel(): void {
if (this.phase === 'disposed') {
return
@@ -0,0 +1,223 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import type { RuntimeHostStatusSnapshot } from '../../../../../../shared/remote-runtime/host-status/runtime-host-status'
import { TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY } from '../../../../../../shared/terminal/runtime-policy/terminal-runtime-capabilities'
import {
TerminalStreamOpcode,
decodeTerminalStreamFrame,
decodeTerminalStreamJson
} from '../../../../../../shared/terminal/stream/terminal-stream-protocol'
import type { PtyTransport } from '../pty-transport-types'
import { createRemoteRuntimeTransportMocks } from '../remote/remote-runtime-pty-transport-test-harness'
import type { MultiplexSubscriptionCallbacks } from '../remote/remote-runtime-pty-transport-stream-fixtures'
vi.mock('@/store/index', () => ({
useAppStore: {
getState: () => {
throw new Error('Unexpected ownership read')
}
}
}))
let callbacks: MultiplexSubscriptionCallbacks = null
let resolvedHandle = 'terminal-1'
const fixture = createRemoteRuntimeTransportMocks({
getCallbacks: () => callbacks,
setCallbacks: (next) => {
callbacks = next
},
getResolvedPaneHandle: () => resolvedHandle,
setResolvedPaneHandle: (next) => {
resolvedHandle = next
}
})
const transports: PtyTransport[] = []
function publish(retired = false, pairingRevision = 1) {
const snapshot: RuntimeHostStatusSnapshot = {
environmentId: 'env-1',
pairingRevision,
sequence: pairingRevision,
checkedAt: Date.now(),
status: null,
transport: retired ? 'disconnected' : 'ready',
verification: retired ? 'blocked' : 'verified',
...(retired ? { retired: true } : {})
}
fixture.publishHostSnapshot(snapshot)
}
async function createTransport() {
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
transports.push(transport)
return transport
}
function pendingReply() {
let reject: (error: Error) => void = () => {}
let resolve: (value: unknown) => void = () => {}
const promise = new Promise((settle, refuse) => {
reject = refuse
resolve = settle
})
return { promise, resolve, reject }
}
beforeEach(() => {
fixture.resetRemoteRuntimeTransport()
fixture.subscriptionSendBinary.mockImplementation((bytes) => {
const frame = decodeTerminalStreamFrame(bytes)
if (frame?.opcode !== TerminalStreamOpcode.Subscribe) {
return
}
const payload = decodeTerminalStreamJson<{ streamId: number }>(frame.payload)!
queueMicrotask(() => {
callbacks?.onResponse({
ok: true,
result: { type: 'subscribed', streamId: payload.streamId }
})
fixture.emitSnapshot(payload.streamId, 'retained prompt$ ')
})
})
})
afterEach(() => {
for (const transport of transports.splice(0)) {
transport.destroy?.()
}
vi.useRealTimers()
})
it.each(['success', 'refusal'] as const)(
'retires an awaited recovery iteration and fences its late %s before a verified legacy host revives it',
async (settlement) => {
publish()
const transport = await createTransport()
const onExit = vi.fn()
const onDisconnect = vi.fn()
await transport.connect({ url: '', callbacks: { onExit, onDisconnect } })
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
const retainedId = transport.getPtyId()
const request = pendingReply()
const dispatch = fixture.runtimeCall.getMockImplementation()!
fixture.runtimeCall.mockImplementation((args) =>
args.method === 'session.tabs.activate' ? request.promise : dispatch(args)
)
callbacks?.onClose?.()
await vi.waitFor(() =>
expect(
fixture.runtimeCall.mock.calls.some(([args]) => args.method === 'session.tabs.activate')
).toBe(true)
)
publish(true)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(transport.getPtyId()).toBe(retainedId)
expect(onExit).not.toHaveBeenCalled()
expect(onDisconnect).not.toHaveBeenCalled()
const callsAtStop = fixture.runtimeCall.mock.calls.length
request.resolve(
settlement === 'success'
? await dispatch({ method: 'session.tabs.activate', params: { tabId: 'tab-1' } })
: { ok: false, error: { code: 'runtime_manually_disconnected', message: 'Stopped' } }
)
await new Promise((resolve) => setTimeout(resolve, 20))
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./recovery/remote-runtime-pty-recovery-state')
retryAllRemoteRuntimePtyRecoveriesNow()
expect(transport.retryRecovery?.()).toBe(false)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(fixture.runtimeCall.mock.calls.length).toBe(callsAtStop)
fixture.runtimeCall.mockImplementation(dispatch)
publish(false, 2)
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
expect(transport.getPtyId()).toBe(retainedId)
callbacks?.onClose?.()
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
}
)
it('terminates a current manual-disconnect refusal inside the production activate/list poll', async () => {
const transport = await createTransport()
await transport.connect({ url: '', callbacks: {} })
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
const dispatch = fixture.runtimeCall.getMockImplementation()!
fixture.runtimeCall.mockResolvedValue({
ok: false,
error: { code: 'runtime_manually_disconnected', message: 'Stopped' }
})
fixture.runtimeCall.mockClear()
callbacks?.onClose?.()
await vi.waitFor(() => expect(transport.getRecoveryState?.().phase).toBe('disconnected'))
await new Promise((resolve) => setTimeout(resolve, 60))
expect(fixture.runtimeCall.mock.calls.map(([args]) => args.method)).toEqual([
'session.tabs.activate'
])
fixture.runtimeCall.mockImplementation(dispatch)
publish()
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
})
it.each(['connect', 'attach'] as const)(
'observes an already-retired host before first %s and replays only after current contact',
async (entry) => {
publish(true)
const transport = await createTransport()
if (entry === 'connect') {
await transport.connect({ url: '', callbacks: {} })
} else {
transport.attach({ existingPtyId: 'remote:env-1@@terminal-1', callbacks: {} })
}
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(fixture.runtimeCall).not.toHaveBeenCalled()
publish(false, 2)
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
}
)
it.each(['success', 'rejection'] as const)(
'fences a pending first-create %s through Stop and reconciles on current contact',
async (settlement) => {
const transport = await createTransport()
const reply = pendingReply()
const dispatch = fixture.runtimeCall.getMockImplementation()!
fixture.runtimeCall.mockImplementation((args) =>
args.method === 'terminal.create' ? reply.promise : dispatch(args)
)
const connect = transport.connect({ url: '', callbacks: {} })
await vi.waitFor(() =>
expect(
fixture.runtimeCall.mock.calls.some(([args]) => args.method === 'terminal.create')
).toBe(true)
)
publish(true)
const stoppedEpoch = transport.getRecoveryState?.().epoch
if (settlement === 'rejection') {
reply.reject(new Error('Remote Orca runtime connection closed.'))
} else {
reply.resolve({ ok: true, result: { terminal: { handle: 'terminal-1' } } })
}
await connect
expect(transport.getRecoveryState?.().epoch).toBe(stoppedEpoch)
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./recovery/remote-runtime-pty-recovery-state')
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
expect(transport.getPtyId()).toBeNull()
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
fixture.runtimeCall.mockImplementation((args) =>
args.method === 'status.get'
? { ok: true, result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] } }
: dispatch(args)
)
publish(false, 2)
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
const creates = fixture.runtimeCall.mock.calls.filter(
([args]) => args.method === 'terminal.create'
)
expect(creates).toHaveLength(2)
expect(creates[1][0].params.reconcileExisting).toBe(true)
expect(creates[1][0].params.clientMutationId).toBe(creates[0][0].params.clientMutationId)
}
)
@@ -132,6 +132,7 @@ import { subscribeAcceptedWebSessionTerminalHandle } from '@/runtime/web-session
import { hostSnapshotAffirmsWorktreeContents } from '@/runtime/web-session-tabs-sync/hydration/host-session-snapshot-authority'
import { runRemoteAgentSessionLaunch } from '@/runtime/hosted-terminals/creation/agent-launch/remote-agent-session-launch'
import { useAppStore } from '@/store/index'
import { runtimeStatusSubscriptions } from '@/store/slices/runtime-status/views/runtime-status-subscription-store'
import { recordWebAgentSessionHandoff } from '@/runtime/hosted-terminals/creation/agent-launch/web-agent-session-handoff'
import { refreshWebRuntimeSessionTabsSnapshot } from '@/runtime/web-session-tabs-sync/inventory/web-runtime-session-snapshot'
import {
@@ -387,6 +388,58 @@ export function createRemoteRuntimePtyTransport(
const disconnectedInputGrace = createRemoteRuntimeDisconnectedInputGrace()
let previousRecoveryPhase: RemoteRuntimePtyRecoveryPhase = 'idle'
let unregisterControlReadyReviver: (() => void) | null = null
let unregisterHostLifetime: (() => void) | null = null
const hostRetired = (): boolean =>
runtimeStatusSubscriptions.getHostSnapshot(currentRuntimeEnvironmentId)?.retired === true
function suspendHostLifetime(): void {
lifecycleEpoch += 1
attachGeneration += 1
subscriptionGeneration += 1
connecting = false
awaitingSshSignIn = false
settleSignInConnectWait()
cancelTerminalCreateRetryWait()
clearPublishedHandleWait()
viewportBatcher.clear()
clearPendingViewportClaim()
setAttachmentUnavailable()
recovery.suspend()
holdUnsentPendingInput()
}
function observeHostLifetime(): boolean {
if (!unregisterHostLifetime) {
let previous = runtimeStatusSubscriptions.getHostSnapshot(currentRuntimeEnvironmentId)
unregisterHostLifetime = runtimeStatusSubscriptions.subscribeHostLifetime(
currentRuntimeEnvironmentId,
() => {
const snapshot = runtimeStatusSubscriptions.getHostSnapshot(currentRuntimeEnvironmentId)
const canRevive =
snapshot &&
!snapshot.retired &&
(previous?.retired ||
snapshot.pairingRevision !== previous?.pairingRevision ||
snapshot.verification === 'verified')
const newlyRetired =
snapshot?.retired &&
(!previous?.retired || snapshot.pairingRevision !== previous.pairingRevision)
previous = snapshot
if (destroyed || terminalEnded) {
return
}
if (newlyRetired) {
suspendHostLifetime()
} else if (canRevive && recovery.currentPhase === 'disconnected') {
transport.retryRecovery?.()
}
}
)
}
if (!hostRetired()) {
return true
}
suspendHostLifetime()
return false
}
// Why: the pane's retry is waiting on the user's SSH sign-in, not on a lost link.
let awaitingSshSignIn = false
// Keep connect pending through sign-in recovery so launch finalization receives the recovered terminal binding.
@@ -410,7 +463,7 @@ export function createRemoteRuntimePtyTransport(
unregisterControlReadyReviver = registerRemoteRuntimePtyControlReadyReviver(
runtimeEnvironmentId,
() => {
if (destroyed) {
if (destroyed || hostRetired()) {
return false
}
// Why: a ready control socket proves sign-in finished, whatever the replay meets next.
@@ -1166,6 +1219,9 @@ export function createRemoteRuntimePtyTransport(
nextRequest = 'list'
}
} catch (error) {
if (toRemoteRuntimeClientErrorLike(error).code === 'runtime_manually_disconnected') {
throw error
}
// Why: the inventory can race the reconnect that invalidated the handle; unknown liveness must not retire the pane.
lastRequestError = error
if (request === 'activate') {
@@ -1214,7 +1270,7 @@ export function createRemoteRuntimePtyTransport(
// Schedule bounded auto-retries and retain a parked retry after deadline so external triggers always have work to revive.
function scheduleConnectRetryAfterRecoverableFailure(): void {
if (destroyed) {
if (destroyed || hostRetired()) {
return
}
// Ambiguous creates require explicit reconciliation-gated retry; automatic replay could repeatedly probe an incapable runtime.
@@ -1376,16 +1432,34 @@ export function createRemoteRuntimePtyTransport(
params?: unknown,
timeoutMs = 15_000
): Promise<TResult> {
const response = await window.api.runtimeEnvironments.call({
selector: environmentId,
method,
params,
timeoutMs,
expectedEnvironmentPairingRevision: currentPaneRevision()
})
const epoch = lifecycleEpoch
if (hostRetired()) {
throw Object.assign(new Error('Runtime environment was disconnected.'), {
code: 'runtime_manually_disconnected'
})
}
try {
const response = await window.api.runtimeEnvironments.call({
selector: environmentId,
method,
params,
timeoutMs,
expectedEnvironmentPairingRevision: currentPaneRevision()
})
if (epoch !== lifecycleEpoch) {
throw new Error(ownerChanged)
}
return unwrapRuntimeRpcResult(response as RuntimeRpcResponse<TResult>)
} catch (error) {
if (epoch !== lifecycleEpoch) {
throw new Error(ownerChanged)
}
if (
epoch === lifecycleEpoch &&
toRemoteRuntimeClientErrorLike(error).code === 'runtime_manually_disconnected'
) {
suspendHostLifetime()
}
// Why await: the caller's retry must read the new revision, not race the catalog re-read.
await refreshRuntimeEnvironmentsAfterPairingChange(error)
throw error
@@ -1521,9 +1595,35 @@ export function createRemoteRuntimePtyTransport(
if (createRemainingMs !== null && createRemainingMs <= 0) {
break
}
// A retired in-flight create is still an unknown outcome until its stable operation is reconciled.
const alreadyNeedsReconciliation =
kind === 'agent-session'
? agentSessionRequiresHostAuthorityReplay
: terminalCreateNeedsReconciliation
const needsReconciliation = (value: boolean): void => {
if (kind === 'agent-session') {
agentSessionRequiresHostAuthorityReplay = value
} else {
terminalCreateNeedsReconciliation = value
}
}
needsReconciliation(true)
try {
return await invoke(Math.min(15_000, createRemainingMs ?? 15_000), reconcileExisting)
const result = await invoke(
Math.min(15_000, createRemainingMs ?? 15_000),
reconcileExisting
)
if (lifecycleEpoch === expectedLifecycleEpoch) {
needsReconciliation(alreadyNeedsReconciliation)
}
return result
} catch (error) {
if (
lifecycleEpoch === expectedLifecycleEpoch &&
!isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))
) {
needsReconciliation(alreadyNeedsReconciliation)
}
lastError = error
const clientError = toRemoteRuntimeClientErrorLike(error)
if (!isRecoverableRemoteRuntimeConnectionError(clientError)) {
@@ -2903,7 +3003,7 @@ export function createRemoteRuntimePtyTransport(
replacementPolicy: HostHandleReplacementPolicy = 'reuse',
requestedRecoveryEpoch?: number
): void {
if (destroyed || !connected || !handle) {
if (destroyed || hostRetired() || !connected || !handle) {
return
}
const recoveryWasActive = recovery.isActive
@@ -3414,6 +3514,9 @@ export function createRemoteRuntimePtyTransport(
return
}
const persistedHandle = nextHandle
if (!observeHostLifetime()) {
return
}
void (async () => {
const adoptPersistedHandle = () =>
adoptResolvedHostPane(
@@ -3599,6 +3702,9 @@ export function createRemoteRuntimePtyTransport(
if (destroyed || !worktreeId) {
return
}
if (!observeHostLifetime()) {
return
}
let releasePrewarmedStream: (() => void) | null = null
try {
@@ -4047,6 +4153,8 @@ export function createRemoteRuntimePtyTransport(
},
disconnect() {
unregisterHostLifetime?.()
unregisterHostLifetime = null
settleSignInConnectWait()
lifecycleEpoch += 1
attachGeneration += 1
@@ -4082,6 +4190,8 @@ export function createRemoteRuntimePtyTransport(
},
detach() {
unregisterHostLifetime?.()
unregisterHostLifetime = null
// Drop the processor gauge before detach callbacks/batcher flush can throw; the successor already owns the PTY.
outputProcessor.disposePendingSideEffectGauge()
settleSignInConnectWait()
@@ -4233,37 +4343,22 @@ export function createRemoteRuntimePtyTransport(
},
retryRecovery() {
if (hostRetired()) {
return false
}
if (recovery.hasOwnerRefresh || surfacedErrorMessages.has(ownerChanged)) {
return recovery.retryNow()
}
if (
!destroyed &&
!terminalEnded &&
!connected &&
isWebTerminalSurfaceTabId(tabId ?? '') &&
recovery.currentPhase === 'disconnected'
) {
// Give replayed attach a fresh deadline; idle cancellation would reset grace and let old held input release arbitrarily late.
autoRecoveryWindowSpent = false
if (!destroyed && !terminalEnded && !connected && recovery.currentPhase === 'disconnected') {
if (isWebTerminalSurfaceTabId(tabId ?? '')) {
autoRecoveryWindowSpent = false
}
recovery.begin()
if (replayLastTransportEntryPoint()) {
return true
}
recovery.markDisconnected()
}
if (
!destroyed &&
!terminalEnded &&
!connected &&
!handle &&
(terminalCreateNeedsReconciliation || agentSessionRequiresHostAuthorityReplay) &&
lastConnectOptions &&
recovery.currentPhase === 'disconnected'
) {
recovery.begin()
void connectHoldingRestoredInput(lastConnectOptions, true)
return true
}
// Why: online/resume fires a parked retry; the button must not be weaker than an event (#12684).
if (!destroyed && !terminalEnded && recovery.currentPhase === 'disconnected') {
if (recovery.retryNow()) {
@@ -1,5 +1,11 @@
import { vi } from 'vitest'
import type { Mock } from 'vitest'
import { createStore } from 'zustand/vanilla'
import { createRuntimeStatusSubscriptions } from '@/store/slices/runtime-status/views/runtime-status-subscriptions'
import type {
RuntimeHostStatusSnapshot,
RuntimeEnvironmentStatus
} from '../../../../../../shared/remote-runtime/host-status/runtime-host-status'
import {
createTerminalStreamFixtures,
type MultiplexSubscriptionCallbacks
@@ -46,6 +52,7 @@ export function readyHostSessionInventoryResponse(
}
export type RemoteRuntimeTransportMocks = {
publishHostSnapshot: (snapshot: RuntimeHostStatusSnapshot) => void
// Why: specs re-implement these with their own request/callback shapes, so keep the loose spy type.
runtimeCall: Mock
runtimeSubscribe: Mock
@@ -60,6 +67,21 @@ export function createRemoteRuntimeTransportMocks(
bindings: RemoteRuntimeTransportBindings
): RemoteRuntimeTransportMocks {
const runtimeCall = vi.fn()
let hostStatuses = createStore(() => ({
runtimeStatusByEnvironmentId: new Map<string, RuntimeEnvironmentStatus>()
}))
let runtimeStatusSubscriptions = createRuntimeStatusSubscriptions(hostStatuses)
const publishHostSnapshot = (snapshot: RuntimeHostStatusSnapshot): void => {
hostStatuses.setState({
runtimeStatusByEnvironmentId: new Map(
hostStatuses.getState().runtimeStatusByEnvironmentId
).set(snapshot.environmentId, {
snapshot,
status: snapshot.retired ? null : snapshot.status,
checkedAt: snapshot.checkedAt
})
})
}
const runtimeSubscribe = vi.fn()
const refreshSessionTabsSnapshot = vi.fn(async () => {})
const subscriptionSendBinary = vi.fn()
@@ -70,6 +92,13 @@ export function createRemoteRuntimeTransportMocks(
function resetRemoteRuntimeTransport(): void {
vi.resetModules()
hostStatuses = createStore(() => ({
runtimeStatusByEnvironmentId: new Map<string, RuntimeEnvironmentStatus>()
}))
runtimeStatusSubscriptions = createRuntimeStatusSubscriptions(hostStatuses)
vi.doMock('@/store/slices/runtime-status/views/runtime-status-subscription-store', () => ({
runtimeStatusSubscriptions
}))
vi.doUnmock(
'../../../../runtime/hosted-terminals/streams/registry/remote-runtime-terminal-multiplexer-registry'
)
@@ -145,6 +174,7 @@ export function createRemoteRuntimeTransportMocks(
}
return {
publishHostSnapshot,
runtimeCall,
runtimeSubscribe,
refreshSessionTabsSnapshot,
@@ -0,0 +1,14 @@
import { useAppStore } from '../../../index'
import { createRuntimeStatusSubscriptions } from './runtime-status-subscriptions'
let subscriptions: ReturnType<typeof createRuntimeStatusSubscriptions> | undefined
const current = () => (subscriptions ??= createRuntimeStatusSubscriptions(useAppStore))
export const runtimeStatusSubscriptions: ReturnType<typeof createRuntimeStatusSubscriptions> = {
getConnectionSnapshot: (id) => current().getConnectionSnapshot(id),
subscribeConnection: (id, listener) => current().subscribeConnection(id, listener),
getHostSnapshot: (id) => current().getHostSnapshot(id),
subscribeHostLifetime: (id, listener) => current().subscribeHostLifetime(id, listener),
getParkingSnapshot: () => current().getParkingSnapshot(),
subscribeParking: (listener) => current().subscribeParking(listener)
}
@@ -138,3 +138,51 @@ describe('runtime status notification isolation', () => {
expect(unsubscribe).toHaveBeenCalledTimes(2)
})
})
it.each([100, 1000])('limits lifetime delivery to affected owners among %i hosts', (count) => {
const { store, subscriptions, publish } = setup()
store.setState({
runtimeStatusByEnvironmentId: new Map(
Array.from({ length: count }, (_, i) => [String(i), entry({ environmentId: String(i) })])
)
})
const subscribe = vi.spyOn(store, 'subscribe')
const affected = vi.fn()
const unaffected = vi.fn()
const release = Array.from({ length: count }, (_, i) =>
subscriptions.subscribeHostLifetime(String(i), i === 0 ? affected : unaffected)
)
expect(subscribe).toHaveBeenCalledTimes(1)
store.setState({ unrelated: 3 })
publish('0', entry({ environmentId: '0', sequence: 4, checkedAt: 4 }))
expect(affected).not.toHaveBeenCalled()
expect(unaffected).not.toHaveBeenCalled()
publish('0', entry({ environmentId: '0', retired: true }))
expect(affected).toHaveBeenCalledTimes(1)
expect(unaffected).not.toHaveBeenCalled()
expect(subscriptions.getHostSnapshot('0')?.retired).toBe(true)
release.forEach((stop) => stop())
publish('0', entry({ environmentId: '0', pairingRevision: 2 }))
expect(affected).toHaveBeenCalledTimes(1)
})
it('keeps lifetime baselines current through nested successor publication and a late subscriber', () => {
const { subscriptions, publish } = setup()
const seen: boolean[] = []
const release = subscriptions.subscribeHostLifetime('a', () => {
seen.push(Boolean(subscriptions.getHostSnapshot('a')?.retired))
if (seen.length === 1) {
publish('a', entry({ pairingRevision: 2 }))
}
})
publish('a', entry({ retired: true }))
expect(seen).toEqual([true, false])
release()
publish('a', entry({ retired: true, pairingRevision: 3 }))
expect(subscriptions.getHostSnapshot('a')?.retired).toBe(true)
const late = vi.fn()
const stop = subscriptions.subscribeHostLifetime('a', late)
publish('a', entry({ pairingRevision: 4 }))
expect(late).toHaveBeenCalledTimes(1)
stop()
})
@@ -1,4 +1,7 @@
import type { RuntimeEnvironmentStatus } from '../../../../../../shared/remote-runtime/host-status/runtime-host-status'
import type {
RuntimeEnvironmentStatus,
RuntimeHostStatusSnapshot
} from '../../../../../../shared/remote-runtime/host-status/runtime-host-status'
import { runtimeHostConnectionStateForEntry } from '@/runtime/hosts/connections/runtime-host-connection-state'
import type { RuntimeHostConnectionState } from '../../../../../../shared/remote-runtime/host-status/runtime-host-connection-state'
import { createPairedRuntimeParkingEnvironmentIdsSelector } from '@/components/terminal-pane/parking/retention/paired-runtime-parking-capabilities'
@@ -14,6 +17,8 @@ type Listener = () => void
type ConnectionSubscription = {
snapshot: RuntimeHostConnectionState
listeners: Set<Listener>
host: RuntimeHostStatusSnapshot | undefined
lifetimeListeners: Set<Listener>
}
export function createRuntimeStatusSubscriptions(store: RuntimeStatusStore) {
@@ -24,6 +29,12 @@ export function createRuntimeStatusSubscriptions(store: RuntimeStatusStore) {
let previousStatuses: RuntimeStatusState['runtimeStatusByEnvironmentId'] | null = null
let unsubscribeStore: (() => void) | null = null
function getHostSnapshot(environmentId: string | null): RuntimeHostStatusSnapshot | undefined {
return environmentId === null
? undefined
: store.getState().runtimeStatusByEnvironmentId.get(environmentId)?.snapshot
}
function getConnectionSnapshot(environmentId: string | null): RuntimeHostConnectionState {
return runtimeHostConnectionStateForEntry(
environmentId === null
@@ -54,12 +65,23 @@ export function createRuntimeStatusSubscriptions(store: RuntimeStatusStore) {
continue
}
const snapshot = runtimeHostConnectionStateForEntry(statuses.get(environmentId))
if (snapshot === subscription.snapshot) {
continue
if (snapshot !== subscription.snapshot) {
subscription.snapshot = snapshot
for (const listener of subscription.listeners) {
pending.push({ listeners: subscription.listeners, listener })
}
}
subscription.snapshot = snapshot
for (const listener of subscription.listeners) {
pending.push({ listeners: subscription.listeners, listener })
const host = statuses.get(environmentId)?.snapshot
const previousHost = subscription.host
subscription.host = host
if (
host?.pairingRevision !== previousHost?.pairingRevision ||
Boolean(host?.retired) !== Boolean(previousHost?.retired) ||
(host?.verification === 'verified') !== (previousHost?.verification === 'verified')
) {
for (const listener of subscription.lifetimeListeners) {
pending.push({ listeners: subscription.lifetimeListeners, listener })
}
}
}
if (parkingListeners.size > 0) {
@@ -91,32 +113,47 @@ export function createRuntimeStatusSubscriptions(store: RuntimeStatusStore) {
selectParking = createPairedRuntimeParkingEnvironmentIdsSelector()
}
function subscribeConnection(environmentId: string | null, listener: Listener): () => void {
function subscribeHost(
environmentId: string | null,
listener: Listener,
lifetime: boolean
): () => void {
if (environmentId === null) {
return () => {}
}
ensureSubscription()
let subscription = connections.get(environmentId)
if (!subscription) {
subscription = { snapshot: getConnectionSnapshot(environmentId), listeners: new Set() }
subscription = {
snapshot: getConnectionSnapshot(environmentId),
listeners: new Set(),
host: getHostSnapshot(environmentId),
lifetimeListeners: new Set()
}
connections.set(environmentId, subscription)
}
const registered = () => listener()
subscription.listeners.add(registered)
const listeners = lifetime ? subscription.lifetimeListeners : subscription.listeners
listeners.add(registered)
let released = false
return () => {
if (released) {
return
}
released = true
subscription.listeners.delete(registered)
if (subscription.listeners.size === 0) {
listeners.delete(registered)
if (subscription.listeners.size === 0 && subscription.lifetimeListeners.size === 0) {
connections.delete(environmentId)
}
releaseIfUnused()
}
}
const subscribeConnection = (environmentId: string | null, listener: Listener) =>
subscribeHost(environmentId, listener, false)
const subscribeHostLifetime = (environmentId: string | null, listener: Listener) =>
subscribeHost(environmentId, listener, true)
function subscribeParking(listener: Listener): () => void {
ensureSubscription()
if (parkingListeners.size === 0) {
@@ -135,5 +172,12 @@ export function createRuntimeStatusSubscriptions(store: RuntimeStatusStore) {
}
}
return { getConnectionSnapshot, subscribeConnection, getParkingSnapshot, subscribeParking }
return {
getConnectionSnapshot,
subscribeConnection,
getHostSnapshot,
subscribeHostLifetime,
getParkingSnapshot,
subscribeParking
}
}
@@ -5,6 +5,7 @@ import { expect, it } from 'vitest'
import { useAppStore } from '../../../index'
import {
usePairedRuntimeParkingEnvironmentIds,
useRuntimeHostRetired,
useRuntimeHostDisconnected
} from './use-runtime-status-view'
@@ -106,3 +107,59 @@ it('observes a commit between render and subscription without waiting for anothe
useAppStore.setState(original, true)
}
})
it('renders only the retired host and rebinds to the current host without waiting for another event', () => {
const original = useAppStore.getState()
const container = document.createElement('div')
const root = createRoot(container)
const renders = { a: 0, b: 0 }
function Probe({ id }: { id: 'a' | 'b' }) {
renders[id] += 1
return <output>{String(useRuntimeHostRetired(id))}</output>
}
try {
useAppStore.setState({ runtimeStatusByEnvironmentId: new Map() })
act(() =>
root.render(
<>
<Probe id="a" />
<Probe id="b" />
</>
)
)
renders.a = renders.b = 0
act(() =>
useAppStore.setState({
runtimeStatusByEnvironmentId: new Map([
[
'a',
{
status: null,
checkedAt: 1,
snapshot: {
environmentId: 'a',
pairingRevision: 1,
sequence: 1,
checkedAt: 1,
status: null,
verification: 'blocked',
transport: 'disconnected',
retired: true
}
}
]
])
})
)
expect(container.textContent).toBe('truefalse')
expect(renders.a).toBeGreaterThan(0)
expect(renders.b).toBe(0)
act(() => root.render(<Probe id="a" />))
expect(container.textContent).toBe('true')
act(() => root.render(<Probe id="b" />))
expect(container.textContent).toBe('false')
} finally {
act(() => root.unmount())
useAppStore.setState(original, true)
}
})
@@ -1,9 +1,19 @@
import { useCallback, useSyncExternalStore } from 'react'
import { useAppStore } from '../../../index'
import { createRuntimeStatusSubscriptions } from './runtime-status-subscriptions'
import { runtimeStatusSubscriptions } from './runtime-status-subscription-store'
import { isDisconnectedRuntimeHostState } from '@/runtime/hosts/connections/runtime-host-connection-state'
const runtimeStatusSubscriptions = createRuntimeStatusSubscriptions(useAppStore)
export function useRuntimeHostRetired(environmentId: string | null): boolean {
const subscribe = useCallback(
(listener: () => void) =>
runtimeStatusSubscriptions.subscribeHostLifetime(environmentId, listener),
[environmentId]
)
const getSnapshot = useCallback(
() => runtimeStatusSubscriptions.getHostSnapshot(environmentId)?.retired === true,
[environmentId]
)
return useSyncExternalStore(subscribe, getSnapshot, getSnapshot)
}
export function useRuntimeHostDisconnected(environmentId: string | null) {
const subscribe = useCallback(