fix(remote-runtime): make every advertised recovery attempt reachable, and stop two recovery latches (#17822)

* fix(remote-runtime): derive the recovery budget and stop faking a spent window

#11305: RECOVERY_DELAYS_MS summed to 60,750ms against a hand-written
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS of 60,000ms, so the ladder's tail was
unreachable. Derive the deadline from the schedule plus one RPC timeout per step
so a half-open link can actually reach every backoff step, and pin the relation
with a test that fails if the sum ever outgrows the budget.

#12683: markDisconnected() is a UI latch, not proof the auto-recovery window ran
out. Track deadline expiry on the recovery state and only let that license the
same-handle reattach that bypasses require-replacement fencing.

#12684: a recoverable connect() failure latched 'disconnected' with no armed
retry, no parked retry and a Reconnect button that returned false. Schedule a
bounded retry (which the deadline parks for online/resume) and let the button
fire a parked retry.

* fix(remote-runtime): stop a post-latch connect failure from re-arming the recovery window

The last attempt's RPC budget expires at the same instant as the deadline, so a
silently dropped link rejects after phase latched to 'disconnected'. begin() then
started a fresh full-length window, so the budget never actually expired. Park the
retry under the latched epoch instead, which keeps online/resume/Reconnect armed
even when the deadline lands mid-attempt with nothing scheduled.

Also fences the same-handle end-reuse window on its own 60s constant so the derived
recovery budget no longer silently triples an unrelated stale-handle check.

* fix(i18n): restore the activity-options key the rebase dropped

* fix(i18n): union en.json with main so the rebase cannot drop keys
This commit is contained in:
Neil
2026-09-02 15:14:40 -07:00
committed by GitHub
parent 7458c39181
commit 033a2a64e1
18 changed files with 426 additions and 44 deletions
@@ -17,7 +17,7 @@ describe('TerminalRemoteRuntimeReconnectBanner', () => {
render(<TerminalRemoteRuntimeReconnectBanner phase="backoff" onReconnect={vi.fn()} />)
expect(screen.getByText('Reconnecting to remote runtime')).toBeInTheDocument()
expect(screen.getByText(/retry for up to one minute/)).toBeInTheDocument()
expect(screen.getByText(/retrying automatically/)).toBeInTheDocument()
expect(screen.queryByRole('button')).not.toBeInTheDocument()
})
@@ -50,7 +50,7 @@ export function TerminalRemoteRuntimeReconnectBanner({
{retrying
? translate(
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.retryingBody',
'Orca will retry for up to one minute. This terminal will resume if the connection returns.'
'Orca is retrying automatically. This terminal will resume if the connection returns.'
)
: translate(
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.disconnectedBody',
@@ -0,0 +1,160 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
createRemoteRuntimeTransportMocks,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
import {
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
} from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
const { runtimeCall, resetRemoteRuntimeTransport } = createRemoteRuntimeTransportMocks({
getCallbacks: () => subscriptionCallbacks,
setCallbacks: (callbacks) => {
subscriptionCallbacks = callbacks
},
getResolvedPaneHandle: () => resolvedPaneHandle,
setResolvedPaneHandle: (handle) => {
resolvedPaneHandle = handle
}
})
// #12684: connect() classified these failures as recoverable and then latched 'disconnected' with
// nothing armed — no backoff timer, no parked retry, and a Reconnect button that returned false.
describe('recoverable connect failures on a remote runtime pane', () => {
let resolvePaneCalls = 0
function installUnreachableRuntime(): void {
resolvePaneCalls = 0
runtimeCall.mockImplementation(async (args: { method: string }) => {
if (args.method === 'terminal.resolvePane') {
resolvePaneCalls += 1
}
throw Object.assign(new Error('Remote Orca runtime closed the connection.'), {
code: 'remote_runtime_unavailable'
})
})
}
// Why: installUnreachableRuntime() rejects synchronously, so every failure lands during a backoff
// wait. A silently dropped link instead burns the whole RPC budget, so the rejection arrives while
// the attempt is still in flight — including after the auto-recovery deadline has already latched.
function installSilentlyDroppedRuntime(): void {
resolvePaneCalls = 0
runtimeCall.mockImplementation(async (args: { method: string }) => {
if (args.method === 'terminal.resolvePane') {
resolvePaneCalls += 1
}
await new Promise((resolve) => {
setTimeout(resolve, REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS)
})
throw Object.assign(new Error('Remote Orca runtime closed the connection.'), {
code: 'remote_runtime_unavailable'
})
})
}
beforeEach(() => {
resetRemoteRuntimeTransport()
})
it('keeps retrying a recoverable connect failure instead of latching immediately', async () => {
vi.useFakeTimers()
try {
installUnreachableRuntime()
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
await transport.connect({
url: '',
sessionId: 'remote:env-1@@',
callbacks: { onError }
})
// Loss of contact is unverifiable, not a dead terminal: automatic recovery must still be running.
expect(resolvePaneCalls).toBe(1)
expect(transport.getRecoveryState?.().phase).toBe('backoff')
expect(onError).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1_000)
expect(resolvePaneCalls).toBeGreaterThan(1)
transport.destroy?.()
} finally {
vi.useRealTimers()
}
})
it('leaves both revival paths armed once the recovery window is spent', async () => {
vi.useFakeTimers()
try {
installUnreachableRuntime()
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
// Why dynamic: resetRemoteRuntimeTransport() re-registers the module graph, and the retry
// registry only sees panes from the same instance the transport was loaded from.
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
await transport.connect({ url: '', sessionId: 'remote:env-1@@', callbacks: {} })
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const callsAtCutoff = resolvePaneCalls
// The cutoff stops self-initiated retries only; online/resume must still find a parked retry.
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
await vi.advanceTimersByTimeAsync(1_000)
expect(resolvePaneCalls).toBeGreaterThan(callsAtCutoff)
// ...and so must the Reconnect button, which returned false before #12684.
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(transport.retryRecovery?.()).toBe(true)
transport.destroy?.()
} finally {
vi.useRealTimers()
}
})
it('keeps the window bounded when a silent drop fails after the deadline latched', async () => {
vi.useFakeTimers()
try {
installSilentlyDroppedRuntime()
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
void transport.connect({ url: '', sessionId: 'remote:env-1@@', callbacks: {} })
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS * 2)
// The in-flight rejection must not begin a new epoch; that re-arms a full-length window forever.
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const callsAtCutoff = resolvePaneCalls
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS * 2)
expect(resolvePaneCalls).toBe(callsAtCutoff)
// A deadline that lands mid-attempt parks nothing, so the latch must still stay revivable.
expect(transport.retryRecovery?.()).toBe(true)
transport.destroy?.()
} finally {
vi.useRealTimers()
}
})
})
@@ -26,6 +26,7 @@ import {
import { RuntimeRpcCallQueueOverloadError } from '../../../../shared/runtime-rpc-call-queue'
import { withRemoteRuntimeTailscaleHint } from '../../../../shared/remote-runtime-tailscale-hint'
import type { PtyTransportRecoveryState } from './pty-transport-types'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
const ELECTRON_IPC_PREFIX = "Error invoking remote method 'runtimeEnvironments:call': "
@@ -409,7 +410,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () =
await vi.advanceTimersByTimeAsync(16_000)
// Auto-recovery deadline latches the pane 'disconnected'.
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
// Connectivity restored; 'online'/system-resume trigger fires.
@@ -417,7 +418,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () =
await vi.advanceTimersByTimeAsync(16_000)
// Latch again, then the user clicks the Reconnect banner.
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
transport.retryRecovery?.()
await vi.advanceTimersByTimeAsync(16_000)
@@ -8,6 +8,7 @@ import {
encodeTerminalStreamText
} from '../../../../shared/terminal-stream-protocol'
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
describe('remote runtime pty reattach after the bounded recovery window', () => {
const runtimeCall = vi.fn()
@@ -198,7 +199,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
expect(transport.getRecoveryState?.().phase).not.toBe('disconnected')
await vi.advanceTimersByTimeAsync(50_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
// The cutoff must not tear down the accepted-snapshot listener; it is the only path back.
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
@@ -227,7 +228,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
const { transport, onError } = await attachStalePane()
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const listCallsAtCutoff = hostListCalls
@@ -277,7 +278,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
const { transport, onError } = await attachStalePane()
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
const listCallsAtCutoff = hostListCalls
@@ -310,7 +311,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const listCallsAtCutoff = hostListCalls
@@ -365,7 +366,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
callbacks: { onError: vi.fn() }
})
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const callsBeforeRetry = runtimeCall.mock.calls.length
@@ -378,4 +379,60 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
vi.useRealTimers()
}
})
// #12683: a fatal resubscribe latches the banner via markDisconnected(), but that latch is not
// evidence the auto-recovery window ran out, so it must not license reattaching a fenced handle.
it('does not reattach a fenced same handle when only a UI latch closed the window', async () => {
vi.useFakeTimers()
try {
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'web-terminal-tab-1',
leafId: 'pane:1',
onPtyExit: vi.fn(),
onPtyRebind: vi.fn()
})
transport.attach({
existingPtyId: 'remote:env-1@@terminal-stale',
cols: 80,
rows: 24,
callbacks: { onError: vi.fn() }
})
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
emitSnapshot(latestSubscribePayload().streamId, 'live before the drop')
// The host keeps publishing the same handle, so no replacement can ever arrive.
runtimeCall.mockImplementation(async (args: { method: string }) => {
if (args.method !== 'session.tabs.list') {
return { ok: true, result: {} }
}
hostListCalls += 1
return { ok: true, result: hostSnapshot('terminal-stale', hostListCalls + 1, 'epoch-1') }
})
// The stream drops and the resubscribe fails fatally with a stale handle: markDisconnected()
// latches the banner, then stale routing fences the handle with require-replacement.
// Only that one attempt fails, so a later reattach would succeed and be observable.
runtimeSubscribe.mockImplementationOnce(async () => {
throw new Error('terminal_handle_stale')
})
subscriptionCallbacks?.onClose?.()
await vi.advanceTimersByTimeAsync(16_000)
expect(transport.getRecoveryState?.().phase).not.toBe('idle')
const subscribesBeforeRepublish = subscribedTerminalHandles().length
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
hostSnapshot('terminal-stale', 9, 'epoch-2'),
'env-1'
)
await vi.advanceTimersByTimeAsync(1_000)
// The recovery deadline never fired, so the fenced handle stays fenced.
expect(subscribedTerminalHandles()).toHaveLength(subscribesBeforeRepublish)
transport.destroy?.()
} finally {
vi.useRealTimers()
}
})
})
@@ -5,6 +5,7 @@ import {
decodeTerminalStreamJson
} from '../../../../shared/terminal-stream-protocol'
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
// Why: the recovery cutoff no longer tears down the retry registry entry or the accepted-snapshot
// listener, so those two module-global collections are the only places a latched pane can accumulate.
@@ -184,7 +185,7 @@ describe('remote runtime pty latched-pane retention', () => {
for (let cycle = 0; cycle < 20; cycle += 1) {
const transport = await attachStalePane(cycle)
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
latched.push(await registries())
transport.destroy?.()
@@ -208,7 +209,7 @@ describe('remote runtime pty latched-pane retention', () => {
const settled: { subscribers: number; scheduled: number }[] = []
for (let cycle = 0; cycle < 20; cycle += 1) {
const transport = await attachStalePane(cycle)
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
transport.detach?.()
await vi.advanceTimersByTimeAsync(1_000)
@@ -227,7 +228,7 @@ describe('remote runtime pty latched-pane retention', () => {
const transports: Awaited<ReturnType<typeof attachStalePane>>[] = []
for (let pane = 0; pane < 8; pane += 1) {
transports.push(await attachStalePane(pane))
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
}
// Retention is per live pane, not per timeout: eight latched panes hold eight of each.
expect(await registries()).toEqual({ subscribers: 8, scheduled: 8 })
@@ -247,7 +248,7 @@ describe('remote runtime pty latched-pane retention', () => {
vi.useFakeTimers()
try {
const transport = await attachStalePane(0)
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const baseline = await registries()
@@ -277,7 +278,7 @@ describe('remote runtime pty latched-pane retention', () => {
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
const transport = await attachStalePane(0)
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const baseline = await registries()
@@ -295,7 +296,7 @@ describe('remote runtime pty latched-pane retention', () => {
// A second trigger in the same window must find nothing to advance, so an online/resume
// storm cannot stack fresh recovery epochs on one pane.
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
observed.push({ ...(await registries()), timers: vi.getTimerCount(), revived })
}
@@ -325,7 +326,7 @@ describe('remote runtime pty latched-pane retention', () => {
try {
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
const transport = await attachStalePane(0)
await vi.advanceTimersByTimeAsync(66_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const baseline = await registries()
@@ -1,6 +1,8 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS,
REMOTE_RUNTIME_RECOVERY_DELAYS_MS,
RemoteRuntimePtyRecoveryState,
retryAllRemoteRuntimePtyRecoveriesNow
} from './remote-runtime-pty-recovery-state'
@@ -63,7 +65,7 @@ describe('RemoteRuntimePtyRecoveryState', () => {
const epoch = state.begin()
state.schedule(epoch, retry)
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(state.currentPhase).toBe('disconnected')
expect(state.isActive).toBe(false)
@@ -112,7 +114,7 @@ describe('RemoteRuntimePtyRecoveryState', () => {
const state = new RemoteRuntimePtyRecoveryState()
const firstEpoch = state.begin()
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const manualEpoch = state.begin()
expect(manualEpoch).toBe(firstEpoch + 1)
@@ -227,4 +229,68 @@ describe('RemoteRuntimePtyRecoveryState', () => {
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
state.dispose()
})
// #11305: the schedule and the deadline lived as two independent literals and drifted apart.
it('keeps the backoff schedule inside the auto-recovery budget it arms', () => {
const scheduleSumMs = REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce(
(total, delayMs) => total + delayMs,
0
)
expect(scheduleSumMs).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
// Every step also needs room for the attempt it leads into, or the tail is dead code
// whenever a half-open link makes each attempt burn its full RPC timeout.
expect(
scheduleSumMs +
REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
})
it('reaches every backoff step when each attempt burns a full RPC timeout', async () => {
vi.useFakeTimers()
const state = new RemoteRuntimePtyRecoveryState()
const attemptStartsMs: number[] = []
const epoch = state.begin()
const startedAt = Date.now()
const failSlowly = (currentEpoch: number): void => {
attemptStartsMs.push(Date.now() - startedAt)
// Silent-drop reconnects do not fail instantly; they time out.
setTimeout(() => {
if (state.isCurrent(currentEpoch)) {
state.schedule(currentEpoch, failSlowly)
}
}, REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS)
}
state.schedule(epoch, failSlowly)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(attemptStartsMs.length).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length)
expect(state.currentPhase).toBe('disconnected')
state.dispose()
})
// #12683: markDisconnected() is a UI latch, not proof the window ran out.
it('only reports the auto-recovery window spent when the deadline actually fired', async () => {
vi.useFakeTimers()
const state = new RemoteRuntimePtyRecoveryState()
const epoch = state.begin()
state.schedule(epoch, vi.fn())
state.markDisconnected()
expect(state.currentPhase).toBe('disconnected')
expect(state.autoRecoveryDeadlineExpired).toBe(false)
const secondEpoch = state.begin()
state.schedule(secondEpoch, vi.fn())
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(state.currentPhase).toBe('disconnected')
expect(state.autoRecoveryDeadlineExpired).toBe(true)
state.markHealthy()
expect(state.autoRecoveryDeadlineExpired).toBe(false)
state.dispose()
})
})
@@ -1,5 +1,17 @@
const RECOVERY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS = 60_000
export const REMOTE_RUNTIME_RECOVERY_DELAYS_MS = [
250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000
] as const
// Why: mirrors DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS in the main-process runtime router; a silently
// dropped link burns the whole RPC timeout on the attempt each backoff step leads into.
export const REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS = 15_000
// Why derived, not hand-tuned: a literal deadline drifted below the ladder it arms, making the last
// backoff steps unreachable dead code (#11305). Loss of contact is never evidence of exit, so the
// window must outlast the schedule it advertises rather than the schedule being trimmed to fit.
export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS =
REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce((total, delayMs) => total + delayMs, 0) +
REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
export type RemoteRuntimePtyRecoveryPhase =
| 'idle'
@@ -34,6 +46,9 @@ export class RemoteRuntimePtyRecoveryState {
private deadlineTimer: ReturnType<typeof setTimeout> | null = null
private pendingRetry: ((epoch: number) => void) | null = null
private pendingEpoch: number | null = null
// Why: only the wall-clock deadline proves the auto-recovery window was actually spent; a UI latch
// via markDisconnected() must not forge that evidence (#12683).
private deadlineExpired = false
constructor(private readonly onChange?: () => void) {}
@@ -53,6 +68,10 @@ export class RemoteRuntimePtyRecoveryState {
return this.attempt
}
get autoRecoveryDeadlineExpired(): boolean {
return this.deadlineExpired
}
begin(): number {
if (this.phase === 'disposed') {
return this.epoch
@@ -82,7 +101,10 @@ export class RemoteRuntimePtyRecoveryState {
}
this.clearRetryTimer()
this.phase = 'backoff'
const delayMs = RECOVERY_DELAYS_MS[Math.min(this.attempt, RECOVERY_DELAYS_MS.length - 1)]
const delayMs =
REMOTE_RUNTIME_RECOVERY_DELAYS_MS[
Math.min(this.attempt, REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length - 1)
]
this.attempt += 1
this.pendingRetry = retry
this.pendingEpoch = epoch
@@ -107,11 +129,22 @@ export class RemoteRuntimePtyRecoveryState {
// Why: a wait that ends with no liveness evidence arms no timer, so park a retry or online/resume/reconnect find nothing to revive.
parkRetryForExternalTrigger(epoch: number, retry: (epoch: number) => void): boolean {
if (!this.isCurrent(epoch) || this.pendingRetry !== null) {
return this.isCurrent(epoch) && this.parkRetry(retry)
}
// Why: the deadline can latch while an attempt is still in flight, before schedule() parked anything,
// so the late failure has no live epoch to join and must not begin a new one — that would re-arm a
// full-length window and the budget would never actually expire.
parkRetryAfterDeadline(retry: (epoch: number) => void): boolean {
return this.phase === 'disconnected' && this.parkRetry(retry)
}
private parkRetry(retry: (epoch: number) => void): boolean {
if (this.pendingRetry !== null) {
return false
}
this.pendingRetry = retry
this.pendingEpoch = epoch
this.pendingEpoch = this.epoch
scheduledRecoveries.add(this)
return true
}
@@ -152,6 +185,7 @@ export class RemoteRuntimePtyRecoveryState {
if (this.phase === 'disposed') {
return
}
this.deadlineExpired = false
this.clearTimers()
this.phase = 'idle'
this.attempt = 0
@@ -175,6 +209,7 @@ export class RemoteRuntimePtyRecoveryState {
if (this.phase === 'disposed') {
return
}
this.deadlineExpired = false
this.epoch += 1
this.clearTimers()
this.phase = 'idle'
@@ -191,11 +226,13 @@ export class RemoteRuntimePtyRecoveryState {
private armDeadline(epoch: number): void {
this.clearDeadlineTimer()
this.deadlineExpired = false
const timer = setTimeout(() => {
if (this.deadlineTimer !== timer || !this.isCurrent(epoch)) {
return
}
this.deadlineTimer = null
this.deadlineExpired = true
// Why: the cutoff stops self-initiated retries but must keep the pane revivable by online/resume/reconnect.
this.stopRetryTimer()
this.phase = 'disconnected'
@@ -4,6 +4,7 @@ import {
createRemoteRuntimeTransportMocks,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -81,7 +82,7 @@ describe('createRemoteRuntimePtyTransport', () => {
let createCalls = 0
runtimeCall.mockImplementation(async (args: { method: string }) => {
if (args.method === 'status.get') {
vi.setSystemTime(startedAt + 59_000)
vi.setSystemTime(startedAt + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS - 1_000)
return {
ok: true,
result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] }
@@ -163,7 +164,7 @@ describe('createRemoteRuntimePtyTransport', () => {
transport.destroy?.()
})
it('stops unknown terminal-create recovery after one minute and remains manually retryable', async () => {
it('stops unknown terminal-create recovery at the cutoff and remains manually retryable', async () => {
vi.useFakeTimers()
try {
let reachable = false
@@ -205,7 +206,7 @@ describe('createRemoteRuntimePtyTransport', () => {
onRecoveryStateChange: (state) => recoveryStates.push(state.phase)
}
})
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
await connect
const callsAtCutoff = runtimeCall.mock.calls.length
@@ -218,7 +219,7 @@ describe('createRemoteRuntimePtyTransport', () => {
statusTimesOut = true
expect(transport.retryRecovery?.()).toBe(true)
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const callsAtManualCutoff = runtimeCall.mock.calls.length
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
await vi.advanceTimersByTimeAsync(5 * 60_000)
@@ -278,7 +279,7 @@ describe('createRemoteRuntimePtyTransport', () => {
})
const connect = transport.connect({ url: '', callbacks: {} })
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
await connect
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
@@ -4,6 +4,7 @@ import {
readyHostSessionInventoryResponse,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -72,7 +73,7 @@ describe('createRemoteRuntimePtyTransport', () => {
expect(hostListCalls).toBe(callsAfterTwoWindows)
expect(transport.getRecoveryState?.().phase).toBe('recovering')
await vi.advanceTimersByTimeAsync(9_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(subscribedTerminalHandles()).toEqual(['terminal-stable'])
transport.destroy?.()
@@ -180,7 +181,7 @@ describe('createRemoteRuntimePtyTransport', () => {
'terminal-flapping'
])
expect(transport.isConnected()).toBe(false)
await vi.advanceTimersByTimeAsync(45_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
transport.destroy?.()
} finally {
@@ -9,6 +9,7 @@ import {
readyHostSessionInventoryResponse,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -160,7 +161,7 @@ describe('createRemoteRuntimePtyTransport', () => {
expect(transport.isConnected()).toBe(false)
expect(onPtyExit).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(44_001)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(subscribedTerminalHandles()).toHaveLength(3)
} finally {
@@ -9,6 +9,7 @@ import {
createRemoteRuntimeTransportMocks,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -534,7 +535,7 @@ describe('createRemoteRuntimePtyTransport', () => {
partitioned = true
callbacksByConnection[0].onClose?.()
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const disconnectedState = transport.getRecoveryState?.()
const callsAtCutoff = runtimeSubscribe.mock.calls.length
@@ -3,6 +3,10 @@ import {
createRemoteRuntimeTransportMocks,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
import {
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
REMOTE_RUNTIME_RECOVERY_DELAYS_MS
} from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -140,10 +144,11 @@ describe('createRemoteRuntimePtyTransport', () => {
existingPtyId: 'remote:env-1@@stale-client-handle',
callbacks: {}
})
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const attemptsAtCutoff = runtimeSubscribe.mock.calls.length
expect(attemptsAtCutoff).toBe(8)
// Why: every backoff step must be reachable inside the window it arms (#11305).
expect(attemptsAtCutoff).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
await vi.advanceTimersByTimeAsync(5 * 60_000)
@@ -235,7 +240,7 @@ describe('createRemoteRuntimePtyTransport', () => {
await vi.advanceTimersByTimeAsync(250)
expect(activateAttempts).toBe(2)
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
rejectInFlight(
@@ -292,7 +297,7 @@ describe('createRemoteRuntimePtyTransport', () => {
await vi.advanceTimersByTimeAsync(250)
expect(runtimeSubscribe).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
rejectSubscription(
@@ -349,7 +354,7 @@ describe('createRemoteRuntimePtyTransport', () => {
expect.objectContaining({ method: 'terminal.resolvePane' })
)
await vi.advanceTimersByTimeAsync(60_000)
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
resolveMetadata({
@@ -90,6 +90,10 @@ const HOST_SESSION_POLL_MAX_MS = 1_000
const HOST_SESSION_ATTACH_TIMEOUT_MS = 15_000
const HOST_SESSION_INVENTORY_MAX_WINDOWS_PER_RECOVERY = 2
const HOST_SESSION_SAME_HANDLE_END_REUSE_LIMIT = 2
// Why its own constant: this fences how long an end-then-reattach on the same handle still counts as
// one recovery, which is unrelated to how long auto-recovery keeps retrying. It read the recovery
// budget before that budget became a derived value, and must not drift with it.
const HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS = 60_000
const MAX_SURFACED_TERMINAL_ERRORS = 8
const TERMINAL_CREATE_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
@@ -247,7 +251,9 @@ export function createRemoteRuntimePtyTransport(
clearPublishedHandleWait()
}
if (recovery.currentPhase === 'disconnected') {
autoRecoveryWindowSpent = true
// Why: only the wall-clock deadline is evidence the window was spent; a UI latch from a fatal
// resubscribe must not license reattaching a fenced same handle (#12683).
autoRecoveryWindowSpent ||= recovery.autoRecoveryDeadlineExpired
// Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic.
subscriptionGeneration += 1
closeMultiplexedStream()
@@ -326,7 +332,7 @@ export function createRemoteRuntimePtyTransport(
if (
sameHandleEndReuseHandle !== targetHandle ||
sameHandleEndReuseAttachedAt === null ||
Date.now() - sameHandleEndReuseAttachedAt >= REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
Date.now() - sameHandleEndReuseAttachedAt >= HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS
) {
resetSameHandleEndReuse()
return 'prefer-replacement'
@@ -870,6 +876,43 @@ export function createRemoteRuntimePtyTransport(
return false
}
// Why: a recoverable connect failure is unverifiable contact loss, not a dead terminal, so retry
// whichever path can still reach the pane instead of latching with nothing armed (#12684).
function retryAfterRecoverableConnectFailure(nextEpoch: number): void {
if (destroyed || terminalEnded) {
return
}
if (connected && handle) {
scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(handle), nextEpoch)
return
}
replayLastTransportEntryPoint()
}
// Why: schedule() both auto-retries inside the window and leaves the retry parked when the deadline
// latches, so online/resume and the Reconnect button always find something to fire.
function scheduleConnectRetryAfterRecoverableFailure(): void {
if (destroyed) {
return
}
// Why: an ambiguous create already owns a reconciliation-gated retry that only Reconnect may
// re-enter; auto-replaying here would just re-probe a runtime that cannot reconcile.
if (terminalCreateNeedsReconciliation || agentSessionRequiresHostAuthorityReplay) {
recovery.markDisconnected()
return
}
// Why: the last attempt's RPC budget expires at the same instant as the deadline, so a silent drop
// rejects after the latch. Beginning a new epoch there re-arms the whole window, so park instead.
if (recovery.currentPhase === 'disconnected') {
recovery.parkRetryAfterDeadline(retryAfterRecoverableConnectFailure)
return
}
const recoveryEpoch = recovery.isActive ? recovery.currentEpoch : recovery.begin()
if (!recovery.schedule(recoveryEpoch, retryAfterRecoverableConnectFailure)) {
recovery.markDisconnected()
}
}
async function attachHostSessionMirror(
options: { cols?: number; rows?: number },
notifySpawn = true,
@@ -1033,6 +1076,8 @@ export function createRemoteRuntimePtyTransport(
kind === 'agent-session'
? agentSessionRequiresHostAuthorityReplay
: terminalCreateNeedsReconciliation
// Why the same budget: this loop calls recovery.begin(), so a shorter local deadline would abandon
// the create while the recovery state still reports 'recovering' with nothing in flight.
let recoveryDeadlineAt: number | null = recovery.isActive
? Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
: null
@@ -2314,7 +2359,7 @@ export function createRemoteRuntimePtyTransport(
} else if (
isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))
) {
recovery.markDisconnected()
scheduleConnectRetryAfterRecoverableFailure()
} else {
recovery.cancel()
emitRecoveryState()
@@ -2616,6 +2661,12 @@ export function createRemoteRuntimePtyTransport(
void transport.connect(lastConnectOptions)
return true
}
// Why: online/resume fires a parked retry; the button must not be weaker than an event (#12684).
if (!destroyed && !terminalEnded && recovery.currentPhase === 'disconnected') {
if (recovery.retryNow()) {
return true
}
}
if (
destroyed ||
terminalEnded ||
+1 -1
View File
@@ -2881,7 +2881,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "Reconnecting to remote runtime",
"disconnectedTitle": "Remote runtime disconnected",
"retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
"retryingBody": "Orca is retrying automatically. This terminal will resume if the connection returns.",
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
"reconnectButton": "Reconnect"
}
+1 -1
View File
@@ -2881,7 +2881,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "リモートランタイムに再接続中",
"disconnectedTitle": "リモートランタイムが切断されました",
"retryingBody": "Orca は最大1分間再試行します。接続が復元されると、このターミナルが再開されます。",
"retryingBody": "Orca は自動的に再試行します。接続が復元されると、このターミナルが再開されます。",
"disconnectedBody": "自動再試行が停止しました。このターミナルセッションを再開するには再接続してください。",
"reconnectButton": "再接続"
}
+1 -1
View File
@@ -2886,7 +2886,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "원격 런타임에 다시 연결 중",
"disconnectedTitle": "원격 런타임 연결 끊김",
"retryingBody": "Orca는 최대 1분간 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.",
"retryingBody": "Orca가 자동으로 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.",
"disconnectedBody": "자동 재시도가 중지되었습니다. 이 터미널 세션을 재개하려면 다시 연결하세요.",
"reconnectButton": "다시 연결"
}
+1 -1
View File
@@ -2896,7 +2896,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "正在重新连接到远程运行时",
"disconnectedTitle": "远程运行时已断开连接",
"retryingBody": "Orca 将重试最多一分钟。如果连接恢复,此终端将恢复。",
"retryingBody": "Orca 正在自动重试。如果连接恢复,此终端将恢复。",
"disconnectedBody": "自动重试已停止。重新连接以恢复此终端会话。",
"reconnectButton": "重新连接"
}