From 033a2a64e17f97d7a69bcaba7f1e21cf5fb82346 Mon Sep 17 00:00:00 2001
From: Neil <4138956+nwparker@users.noreply.github.com>
Date: Wed, 2 Sep 2026 15:14:40 -0700
Subject: [PATCH] fix(remote-runtime): make every advertised recovery attempt
reachable, and stop two recovery latches (#17822)
* fix(remote-runtime): derive the recovery budget and stop faking a spent window
#11305: RECOVERY_DELAYS_MS summed to 60,750ms against a hand-written
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS of 60,000ms, so the ladder's tail was
unreachable. Derive the deadline from the schedule plus one RPC timeout per step
so a half-open link can actually reach every backoff step, and pin the relation
with a test that fails if the sum ever outgrows the budget.
#12683: markDisconnected() is a UI latch, not proof the auto-recovery window ran
out. Track deadline expiry on the recovery state and only let that license the
same-handle reattach that bypasses require-replacement fencing.
#12684: a recoverable connect() failure latched 'disconnected' with no armed
retry, no parked retry and a Reconnect button that returned false. Schedule a
bounded retry (which the deadline parks for online/resume) and let the button
fire a parked retry.
* fix(remote-runtime): stop a post-latch connect failure from re-arming the recovery window
The last attempt's RPC budget expires at the same instant as the deadline, so a
silently dropped link rejects after phase latched to 'disconnected'. begin() then
started a fresh full-length window, so the budget never actually expired. Park the
retry under the latched epoch instead, which keeps online/resume/Reconnect armed
even when the deadline lands mid-attempt with nothing scheduled.
Also fences the same-handle end-reuse window on its own 60s constant so the derived
recovery budget no longer silently triples an unrelated stale-handle check.
* fix(i18n): restore the activity-options key the rebase dropped
* fix(i18n): union en.json with main so the rebase cannot drop keys
---
...minalRemoteRuntimeReconnectBanner.test.tsx | 2 +-
.../TerminalRemoteRuntimeReconnectBanner.tsx | 2 +-
...e-runtime-connect-failure-recovery.test.ts | 160 ++++++++++++++++++
...ge-toast-flood-and-stuck-reconnect.test.ts | 5 +-
...mote-runtime-pty-deadline-reattach.test.ts | 67 +++++++-
...runtime-pty-latched-pane-retention.test.ts | 15 +-
.../remote-runtime-pty-recovery-state.test.ts | 70 +++++++-
.../remote-runtime-pty-recovery-state.ts | 47 ++++-
...-transport-create-outcome-recovery.test.ts | 11 +-
...ty-transport-stale-handle-recovery.test.ts | 5 +-
...ransport-sticky-replacement-policy.test.ts | 3 +-
...ime-pty-transport-stream-reconnect.test.ts | 3 +-
...-pty-transport-web-mirror-recovery.test.ts | 15 +-
.../remote-runtime-pty-transport.ts | 57 ++++++-
src/renderer/src/i18n/locales/es.json | 2 +-
src/renderer/src/i18n/locales/ja.json | 2 +-
src/renderer/src/i18n/locales/ko.json | 2 +-
src/renderer/src/i18n/locales/zh.json | 2 +-
18 files changed, 426 insertions(+), 44 deletions(-)
create mode 100644 src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts
diff --git a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx
index 70341d7a4f6..824a6a80744 100644
--- a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx
+++ b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx
@@ -17,7 +17,7 @@ describe('TerminalRemoteRuntimeReconnectBanner', () => {
render()
expect(screen.getByText('Reconnecting to remote runtime')).toBeInTheDocument()
- expect(screen.getByText(/retry for up to one minute/)).toBeInTheDocument()
+ expect(screen.getByText(/retrying automatically/)).toBeInTheDocument()
expect(screen.queryByRole('button')).not.toBeInTheDocument()
})
diff --git a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx
index c629fa71a36..1cf0fee0903 100644
--- a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx
+++ b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx
@@ -50,7 +50,7 @@ export function TerminalRemoteRuntimeReconnectBanner({
{retrying
? translate(
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.retryingBody',
- 'Orca will retry for up to one minute. This terminal will resume if the connection returns.'
+ 'Orca is retrying automatically. This terminal will resume if the connection returns.'
)
: translate(
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.disconnectedBody',
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts
new file mode 100644
index 00000000000..6c12277f2c0
--- /dev/null
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts
@@ -0,0 +1,160 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+ createRemoteRuntimeTransportMocks,
+ type MultiplexSubscriptionCallbacks
+} from './remote-runtime-pty-transport-test-harness'
+import {
+ REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
+ REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
+} from './remote-runtime-pty-recovery-state'
+
+let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
+let resolvedPaneHandle = 'terminal-1'
+
+const { runtimeCall, resetRemoteRuntimeTransport } = createRemoteRuntimeTransportMocks({
+ getCallbacks: () => subscriptionCallbacks,
+ setCallbacks: (callbacks) => {
+ subscriptionCallbacks = callbacks
+ },
+ getResolvedPaneHandle: () => resolvedPaneHandle,
+ setResolvedPaneHandle: (handle) => {
+ resolvedPaneHandle = handle
+ }
+})
+
+// #12684: connect() classified these failures as recoverable and then latched 'disconnected' with
+// nothing armed — no backoff timer, no parked retry, and a Reconnect button that returned false.
+describe('recoverable connect failures on a remote runtime pane', () => {
+ let resolvePaneCalls = 0
+
+ function installUnreachableRuntime(): void {
+ resolvePaneCalls = 0
+ runtimeCall.mockImplementation(async (args: { method: string }) => {
+ if (args.method === 'terminal.resolvePane') {
+ resolvePaneCalls += 1
+ }
+ throw Object.assign(new Error('Remote Orca runtime closed the connection.'), {
+ code: 'remote_runtime_unavailable'
+ })
+ })
+ }
+
+ // Why: installUnreachableRuntime() rejects synchronously, so every failure lands during a backoff
+ // wait. A silently dropped link instead burns the whole RPC budget, so the rejection arrives while
+ // the attempt is still in flight — including after the auto-recovery deadline has already latched.
+ function installSilentlyDroppedRuntime(): void {
+ resolvePaneCalls = 0
+ runtimeCall.mockImplementation(async (args: { method: string }) => {
+ if (args.method === 'terminal.resolvePane') {
+ resolvePaneCalls += 1
+ }
+ await new Promise((resolve) => {
+ setTimeout(resolve, REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS)
+ })
+ throw Object.assign(new Error('Remote Orca runtime closed the connection.'), {
+ code: 'remote_runtime_unavailable'
+ })
+ })
+ }
+
+ beforeEach(() => {
+ resetRemoteRuntimeTransport()
+ })
+
+ it('keeps retrying a recoverable connect failure instead of latching immediately', async () => {
+ vi.useFakeTimers()
+ try {
+ installUnreachableRuntime()
+ const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
+ const onError = vi.fn()
+ const transport = createRemoteRuntimePtyTransport('env-1', {
+ worktreeId: 'wt-1',
+ tabId: 'tab-1',
+ leafId: 'pane:1'
+ })
+
+ await transport.connect({
+ url: '',
+ sessionId: 'remote:env-1@@',
+ callbacks: { onError }
+ })
+
+ // Loss of contact is unverifiable, not a dead terminal: automatic recovery must still be running.
+ expect(resolvePaneCalls).toBe(1)
+ expect(transport.getRecoveryState?.().phase).toBe('backoff')
+ expect(onError).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(1_000)
+ expect(resolvePaneCalls).toBeGreaterThan(1)
+
+ transport.destroy?.()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('leaves both revival paths armed once the recovery window is spent', async () => {
+ vi.useFakeTimers()
+ try {
+ installUnreachableRuntime()
+ const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
+ // Why dynamic: resetRemoteRuntimeTransport() re-registers the module graph, and the retry
+ // registry only sees panes from the same instance the transport was loaded from.
+ const { retryAllRemoteRuntimePtyRecoveriesNow } =
+ await import('./remote-runtime-pty-recovery-state')
+ const transport = createRemoteRuntimePtyTransport('env-1', {
+ worktreeId: 'wt-1',
+ tabId: 'tab-1',
+ leafId: 'pane:1'
+ })
+
+ await transport.connect({ url: '', sessionId: 'remote:env-1@@', callbacks: {} })
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000)
+
+ expect(transport.getRecoveryState?.().phase).toBe('disconnected')
+ const callsAtCutoff = resolvePaneCalls
+
+ // The cutoff stops self-initiated retries only; online/resume must still find a parked retry.
+ expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
+ await vi.advanceTimersByTimeAsync(1_000)
+ expect(resolvePaneCalls).toBeGreaterThan(callsAtCutoff)
+
+ // ...and so must the Reconnect button, which returned false before #12684.
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000)
+ expect(transport.getRecoveryState?.().phase).toBe('disconnected')
+ expect(transport.retryRecovery?.()).toBe(true)
+
+ transport.destroy?.()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+ it('keeps the window bounded when a silent drop fails after the deadline latched', async () => {
+ vi.useFakeTimers()
+ try {
+ installSilentlyDroppedRuntime()
+ const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
+ const transport = createRemoteRuntimePtyTransport('env-1', {
+ worktreeId: 'wt-1',
+ tabId: 'tab-1',
+ leafId: 'pane:1'
+ })
+
+ void transport.connect({ url: '', sessionId: 'remote:env-1@@', callbacks: {} })
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS * 2)
+
+ // The in-flight rejection must not begin a new epoch; that re-arms a full-length window forever.
+ expect(transport.getRecoveryState?.().phase).toBe('disconnected')
+ const callsAtCutoff = resolvePaneCalls
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS * 2)
+ expect(resolvePaneCalls).toBe(callsAtCutoff)
+
+ // A deadline that lands mid-attempt parks nothing, so the latch must still stay revivable.
+ expect(transport.retryRecovery?.()).toBe(true)
+
+ transport.destroy?.()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+})
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts
index b797a0f5877..9ac752b2b31 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts
@@ -26,6 +26,7 @@ import {
import { RuntimeRpcCallQueueOverloadError } from '../../../../shared/runtime-rpc-call-queue'
import { withRemoteRuntimeTailscaleHint } from '../../../../shared/remote-runtime-tailscale-hint'
import type { PtyTransportRecoveryState } from './pty-transport-types'
+import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
const ELECTRON_IPC_PREFIX = "Error invoking remote method 'runtimeEnvironments:call': "
@@ -409,7 +410,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () =
await vi.advanceTimersByTimeAsync(16_000)
// Auto-recovery deadline latches the pane 'disconnected'.
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
// Connectivity restored; 'online'/system-resume trigger fires.
@@ -417,7 +418,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () =
await vi.advanceTimersByTimeAsync(16_000)
// Latch again, then the user clicks the Reconnect banner.
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
transport.retryRecovery?.()
await vi.advanceTimersByTimeAsync(16_000)
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts
index 886516414ca..61758efb299 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts
@@ -8,6 +8,7 @@ import {
encodeTerminalStreamText
} from '../../../../shared/terminal-stream-protocol'
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
+import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
describe('remote runtime pty reattach after the bounded recovery window', () => {
const runtimeCall = vi.fn()
@@ -198,7 +199,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
expect(transport.getRecoveryState?.().phase).not.toBe('disconnected')
- await vi.advanceTimersByTimeAsync(50_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
// The cutoff must not tear down the accepted-snapshot listener; it is the only path back.
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
@@ -227,7 +228,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
const { transport, onError } = await attachStalePane()
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const listCallsAtCutoff = hostListCalls
@@ -277,7 +278,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
const { transport, onError } = await attachStalePane()
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
const listCallsAtCutoff = hostListCalls
@@ -310,7 +311,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const listCallsAtCutoff = hostListCalls
@@ -365,7 +366,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
callbacks: { onError: vi.fn() }
})
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const callsBeforeRetry = runtimeCall.mock.calls.length
@@ -378,4 +379,60 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
vi.useRealTimers()
}
})
+
+ // #12683: a fatal resubscribe latches the banner via markDisconnected(), but that latch is not
+ // evidence the auto-recovery window ran out, so it must not license reattaching a fenced handle.
+ it('does not reattach a fenced same handle when only a UI latch closed the window', async () => {
+ vi.useFakeTimers()
+ try {
+ const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
+ const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
+ const transport = createRemoteRuntimePtyTransport('env-1', {
+ worktreeId: 'wt-1',
+ tabId: 'web-terminal-tab-1',
+ leafId: 'pane:1',
+ onPtyExit: vi.fn(),
+ onPtyRebind: vi.fn()
+ })
+ transport.attach({
+ existingPtyId: 'remote:env-1@@terminal-stale',
+ cols: 80,
+ rows: 24,
+ callbacks: { onError: vi.fn() }
+ })
+ await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
+ emitSnapshot(latestSubscribePayload().streamId, 'live before the drop')
+
+ // The host keeps publishing the same handle, so no replacement can ever arrive.
+ runtimeCall.mockImplementation(async (args: { method: string }) => {
+ if (args.method !== 'session.tabs.list') {
+ return { ok: true, result: {} }
+ }
+ hostListCalls += 1
+ return { ok: true, result: hostSnapshot('terminal-stale', hostListCalls + 1, 'epoch-1') }
+ })
+ // The stream drops and the resubscribe fails fatally with a stale handle: markDisconnected()
+ // latches the banner, then stale routing fences the handle with require-replacement.
+ // Only that one attempt fails, so a later reattach would succeed and be observable.
+ runtimeSubscribe.mockImplementationOnce(async () => {
+ throw new Error('terminal_handle_stale')
+ })
+ subscriptionCallbacks?.onClose?.()
+ await vi.advanceTimersByTimeAsync(16_000)
+ expect(transport.getRecoveryState?.().phase).not.toBe('idle')
+
+ const subscribesBeforeRepublish = subscribedTerminalHandles().length
+ handleEvents.queueAcceptedWebSessionTerminalSnapshot(
+ hostSnapshot('terminal-stale', 9, 'epoch-2'),
+ 'env-1'
+ )
+ await vi.advanceTimersByTimeAsync(1_000)
+
+ // The recovery deadline never fired, so the fenced handle stays fenced.
+ expect(subscribedTerminalHandles()).toHaveLength(subscribesBeforeRepublish)
+ transport.destroy?.()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
})
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts
index 5892ee2c19d..3c1ec0421a9 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts
@@ -5,6 +5,7 @@ import {
decodeTerminalStreamJson
} from '../../../../shared/terminal-stream-protocol'
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
+import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
// Why: the recovery cutoff no longer tears down the retry registry entry or the accepted-snapshot
// listener, so those two module-global collections are the only places a latched pane can accumulate.
@@ -184,7 +185,7 @@ describe('remote runtime pty latched-pane retention', () => {
for (let cycle = 0; cycle < 20; cycle += 1) {
const transport = await attachStalePane(cycle)
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
latched.push(await registries())
transport.destroy?.()
@@ -208,7 +209,7 @@ describe('remote runtime pty latched-pane retention', () => {
const settled: { subscribers: number; scheduled: number }[] = []
for (let cycle = 0; cycle < 20; cycle += 1) {
const transport = await attachStalePane(cycle)
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
transport.detach?.()
await vi.advanceTimersByTimeAsync(1_000)
@@ -227,7 +228,7 @@ describe('remote runtime pty latched-pane retention', () => {
const transports: Awaited>[] = []
for (let pane = 0; pane < 8; pane += 1) {
transports.push(await attachStalePane(pane))
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
}
// Retention is per live pane, not per timeout: eight latched panes hold eight of each.
expect(await registries()).toEqual({ subscribers: 8, scheduled: 8 })
@@ -247,7 +248,7 @@ describe('remote runtime pty latched-pane retention', () => {
vi.useFakeTimers()
try {
const transport = await attachStalePane(0)
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const baseline = await registries()
@@ -277,7 +278,7 @@ describe('remote runtime pty latched-pane retention', () => {
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
const transport = await attachStalePane(0)
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const baseline = await registries()
@@ -295,7 +296,7 @@ describe('remote runtime pty latched-pane retention', () => {
// A second trigger in the same window must find nothing to advance, so an online/resume
// storm cannot stack fresh recovery epochs on one pane.
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
observed.push({ ...(await registries()), timers: vi.getTimerCount(), revived })
}
@@ -325,7 +326,7 @@ describe('remote runtime pty latched-pane retention', () => {
try {
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
const transport = await attachStalePane(0)
- await vi.advanceTimersByTimeAsync(66_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const baseline = await registries()
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts
index d71af0b9afb..c245c184436 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts
@@ -1,6 +1,8 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
+ REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS,
+ REMOTE_RUNTIME_RECOVERY_DELAYS_MS,
RemoteRuntimePtyRecoveryState,
retryAllRemoteRuntimePtyRecoveriesNow
} from './remote-runtime-pty-recovery-state'
@@ -63,7 +65,7 @@ describe('RemoteRuntimePtyRecoveryState', () => {
const epoch = state.begin()
state.schedule(epoch, retry)
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(state.currentPhase).toBe('disconnected')
expect(state.isActive).toBe(false)
@@ -112,7 +114,7 @@ describe('RemoteRuntimePtyRecoveryState', () => {
const state = new RemoteRuntimePtyRecoveryState()
const firstEpoch = state.begin()
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const manualEpoch = state.begin()
expect(manualEpoch).toBe(firstEpoch + 1)
@@ -227,4 +229,68 @@ describe('RemoteRuntimePtyRecoveryState', () => {
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
state.dispose()
})
+
+ // #11305: the schedule and the deadline lived as two independent literals and drifted apart.
+ it('keeps the backoff schedule inside the auto-recovery budget it arms', () => {
+ const scheduleSumMs = REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce(
+ (total, delayMs) => total + delayMs,
+ 0
+ )
+
+ expect(scheduleSumMs).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
+ // Every step also needs room for the attempt it leads into, or the tail is dead code
+ // whenever a half-open link makes each attempt burn its full RPC timeout.
+ expect(
+ scheduleSumMs +
+ REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
+ ).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
+ })
+
+ it('reaches every backoff step when each attempt burns a full RPC timeout', async () => {
+ vi.useFakeTimers()
+ const state = new RemoteRuntimePtyRecoveryState()
+ const attemptStartsMs: number[] = []
+ const epoch = state.begin()
+ const startedAt = Date.now()
+
+ const failSlowly = (currentEpoch: number): void => {
+ attemptStartsMs.push(Date.now() - startedAt)
+ // Silent-drop reconnects do not fail instantly; they time out.
+ setTimeout(() => {
+ if (state.isCurrent(currentEpoch)) {
+ state.schedule(currentEpoch, failSlowly)
+ }
+ }, REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS)
+ }
+ state.schedule(epoch, failSlowly)
+
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
+
+ expect(attemptStartsMs.length).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length)
+ expect(state.currentPhase).toBe('disconnected')
+ state.dispose()
+ })
+
+ // #12683: markDisconnected() is a UI latch, not proof the window ran out.
+ it('only reports the auto-recovery window spent when the deadline actually fired', async () => {
+ vi.useFakeTimers()
+ const state = new RemoteRuntimePtyRecoveryState()
+ const epoch = state.begin()
+ state.schedule(epoch, vi.fn())
+
+ state.markDisconnected()
+ expect(state.currentPhase).toBe('disconnected')
+ expect(state.autoRecoveryDeadlineExpired).toBe(false)
+
+ const secondEpoch = state.begin()
+ state.schedule(secondEpoch, vi.fn())
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
+
+ expect(state.currentPhase).toBe('disconnected')
+ expect(state.autoRecoveryDeadlineExpired).toBe(true)
+
+ state.markHealthy()
+ expect(state.autoRecoveryDeadlineExpired).toBe(false)
+ state.dispose()
+ })
})
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts
index fbdd50e443f..002e1320398 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts
@@ -1,5 +1,17 @@
-const RECOVERY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
-export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS = 60_000
+export const REMOTE_RUNTIME_RECOVERY_DELAYS_MS = [
+ 250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000
+] as const
+
+// Why: mirrors DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS in the main-process runtime router; a silently
+// dropped link burns the whole RPC timeout on the attempt each backoff step leads into.
+export const REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS = 15_000
+
+// Why derived, not hand-tuned: a literal deadline drifted below the ladder it arms, making the last
+// backoff steps unreachable dead code (#11305). Loss of contact is never evidence of exit, so the
+// window must outlast the schedule it advertises rather than the schedule being trimmed to fit.
+export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS =
+ REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce((total, delayMs) => total + delayMs, 0) +
+ REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
export type RemoteRuntimePtyRecoveryPhase =
| 'idle'
@@ -34,6 +46,9 @@ export class RemoteRuntimePtyRecoveryState {
private deadlineTimer: ReturnType | null = null
private pendingRetry: ((epoch: number) => void) | null = null
private pendingEpoch: number | null = null
+ // Why: only the wall-clock deadline proves the auto-recovery window was actually spent; a UI latch
+ // via markDisconnected() must not forge that evidence (#12683).
+ private deadlineExpired = false
constructor(private readonly onChange?: () => void) {}
@@ -53,6 +68,10 @@ export class RemoteRuntimePtyRecoveryState {
return this.attempt
}
+ get autoRecoveryDeadlineExpired(): boolean {
+ return this.deadlineExpired
+ }
+
begin(): number {
if (this.phase === 'disposed') {
return this.epoch
@@ -82,7 +101,10 @@ export class RemoteRuntimePtyRecoveryState {
}
this.clearRetryTimer()
this.phase = 'backoff'
- const delayMs = RECOVERY_DELAYS_MS[Math.min(this.attempt, RECOVERY_DELAYS_MS.length - 1)]
+ const delayMs =
+ REMOTE_RUNTIME_RECOVERY_DELAYS_MS[
+ Math.min(this.attempt, REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length - 1)
+ ]
this.attempt += 1
this.pendingRetry = retry
this.pendingEpoch = epoch
@@ -107,11 +129,22 @@ export class RemoteRuntimePtyRecoveryState {
// Why: a wait that ends with no liveness evidence arms no timer, so park a retry or online/resume/reconnect find nothing to revive.
parkRetryForExternalTrigger(epoch: number, retry: (epoch: number) => void): boolean {
- if (!this.isCurrent(epoch) || this.pendingRetry !== null) {
+ return this.isCurrent(epoch) && this.parkRetry(retry)
+ }
+
+ // Why: the deadline can latch while an attempt is still in flight, before schedule() parked anything,
+ // so the late failure has no live epoch to join and must not begin a new one — that would re-arm a
+ // full-length window and the budget would never actually expire.
+ parkRetryAfterDeadline(retry: (epoch: number) => void): boolean {
+ return this.phase === 'disconnected' && this.parkRetry(retry)
+ }
+
+ private parkRetry(retry: (epoch: number) => void): boolean {
+ if (this.pendingRetry !== null) {
return false
}
this.pendingRetry = retry
- this.pendingEpoch = epoch
+ this.pendingEpoch = this.epoch
scheduledRecoveries.add(this)
return true
}
@@ -152,6 +185,7 @@ export class RemoteRuntimePtyRecoveryState {
if (this.phase === 'disposed') {
return
}
+ this.deadlineExpired = false
this.clearTimers()
this.phase = 'idle'
this.attempt = 0
@@ -175,6 +209,7 @@ export class RemoteRuntimePtyRecoveryState {
if (this.phase === 'disposed') {
return
}
+ this.deadlineExpired = false
this.epoch += 1
this.clearTimers()
this.phase = 'idle'
@@ -191,11 +226,13 @@ export class RemoteRuntimePtyRecoveryState {
private armDeadline(epoch: number): void {
this.clearDeadlineTimer()
+ this.deadlineExpired = false
const timer = setTimeout(() => {
if (this.deadlineTimer !== timer || !this.isCurrent(epoch)) {
return
}
this.deadlineTimer = null
+ this.deadlineExpired = true
// Why: the cutoff stops self-initiated retries but must keep the pane revivable by online/resume/reconnect.
this.stopRetryTimer()
this.phase = 'disconnected'
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts
index b34ec4f3fea..59dc3612ed1 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts
@@ -4,6 +4,7 @@ import {
createRemoteRuntimeTransportMocks,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
+import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -81,7 +82,7 @@ describe('createRemoteRuntimePtyTransport', () => {
let createCalls = 0
runtimeCall.mockImplementation(async (args: { method: string }) => {
if (args.method === 'status.get') {
- vi.setSystemTime(startedAt + 59_000)
+ vi.setSystemTime(startedAt + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS - 1_000)
return {
ok: true,
result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] }
@@ -163,7 +164,7 @@ describe('createRemoteRuntimePtyTransport', () => {
transport.destroy?.()
})
- it('stops unknown terminal-create recovery after one minute and remains manually retryable', async () => {
+ it('stops unknown terminal-create recovery at the cutoff and remains manually retryable', async () => {
vi.useFakeTimers()
try {
let reachable = false
@@ -205,7 +206,7 @@ describe('createRemoteRuntimePtyTransport', () => {
onRecoveryStateChange: (state) => recoveryStates.push(state.phase)
}
})
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
await connect
const callsAtCutoff = runtimeCall.mock.calls.length
@@ -218,7 +219,7 @@ describe('createRemoteRuntimePtyTransport', () => {
statusTimesOut = true
expect(transport.retryRecovery?.()).toBe(true)
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const callsAtManualCutoff = runtimeCall.mock.calls.length
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
await vi.advanceTimersByTimeAsync(5 * 60_000)
@@ -278,7 +279,7 @@ describe('createRemoteRuntimePtyTransport', () => {
})
const connect = transport.connect({ url: '', callbacks: {} })
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
await connect
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts
index 8817d5c23e4..53c1921fe74 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts
@@ -4,6 +4,7 @@ import {
readyHostSessionInventoryResponse,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
+import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -72,7 +73,7 @@ describe('createRemoteRuntimePtyTransport', () => {
expect(hostListCalls).toBe(callsAfterTwoWindows)
expect(transport.getRecoveryState?.().phase).toBe('recovering')
- await vi.advanceTimersByTimeAsync(9_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(subscribedTerminalHandles()).toEqual(['terminal-stable'])
transport.destroy?.()
@@ -180,7 +181,7 @@ describe('createRemoteRuntimePtyTransport', () => {
'terminal-flapping'
])
expect(transport.isConnected()).toBe(false)
- await vi.advanceTimersByTimeAsync(45_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
transport.destroy?.()
} finally {
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts
index 48a24a2c084..5a61a5587e6 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts
@@ -9,6 +9,7 @@ import {
readyHostSessionInventoryResponse,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
+import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -160,7 +161,7 @@ describe('createRemoteRuntimePtyTransport', () => {
expect(transport.isConnected()).toBe(false)
expect(onPtyExit).not.toHaveBeenCalled()
- await vi.advanceTimersByTimeAsync(44_001)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(subscribedTerminalHandles()).toHaveLength(3)
} finally {
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts
index 4ef1075e3f7..e0b85414e3f 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts
@@ -9,6 +9,7 @@ import {
createRemoteRuntimeTransportMocks,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
+import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -534,7 +535,7 @@ describe('createRemoteRuntimePtyTransport', () => {
partitioned = true
callbacksByConnection[0].onClose?.()
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const disconnectedState = transport.getRecoveryState?.()
const callsAtCutoff = runtimeSubscribe.mock.calls.length
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts
index f8c07ea045d..784ed9d8768 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts
@@ -3,6 +3,10 @@ import {
createRemoteRuntimeTransportMocks,
type MultiplexSubscriptionCallbacks
} from './remote-runtime-pty-transport-test-harness'
+import {
+ REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
+ REMOTE_RUNTIME_RECOVERY_DELAYS_MS
+} from './remote-runtime-pty-recovery-state'
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
let resolvedPaneHandle = 'terminal-1'
@@ -140,10 +144,11 @@ describe('createRemoteRuntimePtyTransport', () => {
existingPtyId: 'remote:env-1@@stale-client-handle',
callbacks: {}
})
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
const attemptsAtCutoff = runtimeSubscribe.mock.calls.length
- expect(attemptsAtCutoff).toBe(8)
+ // Why: every backoff step must be reachable inside the window it arms (#11305).
+ expect(attemptsAtCutoff).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
await vi.advanceTimersByTimeAsync(5 * 60_000)
@@ -235,7 +240,7 @@ describe('createRemoteRuntimePtyTransport', () => {
await vi.advanceTimersByTimeAsync(250)
expect(activateAttempts).toBe(2)
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
rejectInFlight(
@@ -292,7 +297,7 @@ describe('createRemoteRuntimePtyTransport', () => {
await vi.advanceTimersByTimeAsync(250)
expect(runtimeSubscribe).toHaveBeenCalledTimes(1)
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
rejectSubscription(
@@ -349,7 +354,7 @@ describe('createRemoteRuntimePtyTransport', () => {
expect.objectContaining({ method: 'terminal.resolvePane' })
)
- await vi.advanceTimersByTimeAsync(60_000)
+ await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
resolveMetadata({
diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts
index 1e5782fe496..b1bd7336c86 100644
--- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts
+++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts
@@ -90,6 +90,10 @@ const HOST_SESSION_POLL_MAX_MS = 1_000
const HOST_SESSION_ATTACH_TIMEOUT_MS = 15_000
const HOST_SESSION_INVENTORY_MAX_WINDOWS_PER_RECOVERY = 2
const HOST_SESSION_SAME_HANDLE_END_REUSE_LIMIT = 2
+// Why its own constant: this fences how long an end-then-reattach on the same handle still counts as
+// one recovery, which is unrelated to how long auto-recovery keeps retrying. It read the recovery
+// budget before that budget became a derived value, and must not drift with it.
+const HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS = 60_000
const MAX_SURFACED_TERMINAL_ERRORS = 8
const TERMINAL_CREATE_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
@@ -247,7 +251,9 @@ export function createRemoteRuntimePtyTransport(
clearPublishedHandleWait()
}
if (recovery.currentPhase === 'disconnected') {
- autoRecoveryWindowSpent = true
+ // Why: only the wall-clock deadline is evidence the window was spent; a UI latch from a fatal
+ // resubscribe must not license reattaching a fenced same handle (#12683).
+ autoRecoveryWindowSpent ||= recovery.autoRecoveryDeadlineExpired
// Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic.
subscriptionGeneration += 1
closeMultiplexedStream()
@@ -326,7 +332,7 @@ export function createRemoteRuntimePtyTransport(
if (
sameHandleEndReuseHandle !== targetHandle ||
sameHandleEndReuseAttachedAt === null ||
- Date.now() - sameHandleEndReuseAttachedAt >= REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
+ Date.now() - sameHandleEndReuseAttachedAt >= HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS
) {
resetSameHandleEndReuse()
return 'prefer-replacement'
@@ -870,6 +876,43 @@ export function createRemoteRuntimePtyTransport(
return false
}
+ // Why: a recoverable connect failure is unverifiable contact loss, not a dead terminal, so retry
+ // whichever path can still reach the pane instead of latching with nothing armed (#12684).
+ function retryAfterRecoverableConnectFailure(nextEpoch: number): void {
+ if (destroyed || terminalEnded) {
+ return
+ }
+ if (connected && handle) {
+ scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(handle), nextEpoch)
+ return
+ }
+ replayLastTransportEntryPoint()
+ }
+
+ // Why: schedule() both auto-retries inside the window and leaves the retry parked when the deadline
+ // latches, so online/resume and the Reconnect button always find something to fire.
+ function scheduleConnectRetryAfterRecoverableFailure(): void {
+ if (destroyed) {
+ return
+ }
+ // Why: an ambiguous create already owns a reconciliation-gated retry that only Reconnect may
+ // re-enter; auto-replaying here would just re-probe a runtime that cannot reconcile.
+ if (terminalCreateNeedsReconciliation || agentSessionRequiresHostAuthorityReplay) {
+ recovery.markDisconnected()
+ return
+ }
+ // Why: the last attempt's RPC budget expires at the same instant as the deadline, so a silent drop
+ // rejects after the latch. Beginning a new epoch there re-arms the whole window, so park instead.
+ if (recovery.currentPhase === 'disconnected') {
+ recovery.parkRetryAfterDeadline(retryAfterRecoverableConnectFailure)
+ return
+ }
+ const recoveryEpoch = recovery.isActive ? recovery.currentEpoch : recovery.begin()
+ if (!recovery.schedule(recoveryEpoch, retryAfterRecoverableConnectFailure)) {
+ recovery.markDisconnected()
+ }
+ }
+
async function attachHostSessionMirror(
options: { cols?: number; rows?: number },
notifySpawn = true,
@@ -1033,6 +1076,8 @@ export function createRemoteRuntimePtyTransport(
kind === 'agent-session'
? agentSessionRequiresHostAuthorityReplay
: terminalCreateNeedsReconciliation
+ // Why the same budget: this loop calls recovery.begin(), so a shorter local deadline would abandon
+ // the create while the recovery state still reports 'recovering' with nothing in flight.
let recoveryDeadlineAt: number | null = recovery.isActive
? Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
: null
@@ -2314,7 +2359,7 @@ export function createRemoteRuntimePtyTransport(
} else if (
isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))
) {
- recovery.markDisconnected()
+ scheduleConnectRetryAfterRecoverableFailure()
} else {
recovery.cancel()
emitRecoveryState()
@@ -2616,6 +2661,12 @@ export function createRemoteRuntimePtyTransport(
void transport.connect(lastConnectOptions)
return true
}
+ // Why: online/resume fires a parked retry; the button must not be weaker than an event (#12684).
+ if (!destroyed && !terminalEnded && recovery.currentPhase === 'disconnected') {
+ if (recovery.retryNow()) {
+ return true
+ }
+ }
if (
destroyed ||
terminalEnded ||
diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json
index f4a4fd931de..0ac5d624ede 100644
--- a/src/renderer/src/i18n/locales/es.json
+++ b/src/renderer/src/i18n/locales/es.json
@@ -2881,7 +2881,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "Reconnecting to remote runtime",
"disconnectedTitle": "Remote runtime disconnected",
- "retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
+ "retryingBody": "Orca is retrying automatically. This terminal will resume if the connection returns.",
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
"reconnectButton": "Reconnect"
}
diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json
index ee6e4440f79..d3af25ad697 100644
--- a/src/renderer/src/i18n/locales/ja.json
+++ b/src/renderer/src/i18n/locales/ja.json
@@ -2881,7 +2881,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "リモートランタイムに再接続中",
"disconnectedTitle": "リモートランタイムが切断されました",
- "retryingBody": "Orca は最大1分間再試行します。接続が復元されると、このターミナルが再開されます。",
+ "retryingBody": "Orca は自動的に再試行します。接続が復元されると、このターミナルが再開されます。",
"disconnectedBody": "自動再試行が停止しました。このターミナルセッションを再開するには再接続してください。",
"reconnectButton": "再接続"
}
diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json
index 76ed151ca2c..47d9e48fd26 100644
--- a/src/renderer/src/i18n/locales/ko.json
+++ b/src/renderer/src/i18n/locales/ko.json
@@ -2886,7 +2886,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "원격 런타임에 다시 연결 중",
"disconnectedTitle": "원격 런타임 연결 끊김",
- "retryingBody": "Orca는 최대 1분간 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.",
+ "retryingBody": "Orca가 자동으로 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.",
"disconnectedBody": "자동 재시도가 중지되었습니다. 이 터미널 세션을 재개하려면 다시 연결하세요.",
"reconnectButton": "다시 연결"
}
diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json
index 5c1324afa12..b097b789571 100644
--- a/src/renderer/src/i18n/locales/zh.json
+++ b/src/renderer/src/i18n/locales/zh.json
@@ -2896,7 +2896,7 @@
"TerminalRemoteRuntimeReconnectBanner": {
"retryingTitle": "正在重新连接到远程运行时",
"disconnectedTitle": "远程运行时已断开连接",
- "retryingBody": "Orca 将重试最多一分钟。如果连接恢复，此终端将恢复。",
+ "retryingBody": "Orca 正在自动重试。如果连接恢复，此终端将恢复。",
"disconnectedBody": "自动重试已停止。重新连接以恢复此终端会话。",
"reconnectButton": "重新连接"
}