From dea5e69db848b4c64534c67564ee58c8cb5cbada Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:06:15 -0700 Subject: [PATCH] fix(remote-runtime): derive the recovery budget and stop faking a spent window #11305: RECOVERY_DELAYS_MS summed to 60,750ms against a hand-written REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS of 60,000ms, so the ladder's tail was unreachable. Derive the deadline from the schedule plus one RPC timeout per step so a half-open link can actually reach every backoff step, and pin the relation with a test that fails if the sum ever outgrows the budget. #12683: markDisconnected() is a UI latch, not proof the auto-recovery window ran out. Track deadline expiry on the recovery state and only let that license the same-handle reattach that bypasses require-replacement fencing. #12684: a recoverable connect() failure latched 'disconnected' with no armed retry, no parked retry and a Reconnect button that returned false. Schedule a bounded retry (which the deadline parks for online/resume) and let the button fire a parked retry. --- ...minalRemoteRuntimeReconnectBanner.test.tsx | 2 +- .../TerminalRemoteRuntimeReconnectBanner.tsx | 2 +- ...e-runtime-connect-failure-recovery.test.ts | 111 ++++++++++++++++++ ...ge-toast-flood-and-stuck-reconnect.test.ts | 5 +- ...mote-runtime-pty-deadline-reattach.test.ts | 67 ++++++++++- ...runtime-pty-latched-pane-retention.test.ts | 15 +-- .../remote-runtime-pty-recovery-state.test.ts | 70 ++++++++++- .../remote-runtime-pty-recovery-state.ts | 32 ++++- ...-transport-create-outcome-recovery.test.ts | 11 +- ...ty-transport-stale-handle-recovery.test.ts | 5 +- ...ransport-sticky-replacement-policy.test.ts | 3 +- ...ime-pty-transport-stream-reconnect.test.ts | 3 +- ...-pty-transport-web-mirror-recovery.test.ts | 15 ++- .../remote-runtime-pty-transport.ts | 43 ++++++- src/renderer/src/i18n/locales/en.json | 2 +- src/renderer/src/i18n/locales/es.json | 2 +- src/renderer/src/i18n/locales/ja.json | 2 +- src/renderer/src/i18n/locales/ko.json | 2 +- src/renderer/src/i18n/locales/zh.json | 2 +- 19 files changed, 352 insertions(+), 42 deletions(-) create mode 100644 src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts diff --git a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx index 70341d7a4f6..824a6a80744 100644 --- a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx +++ b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx @@ -17,7 +17,7 @@ describe('TerminalRemoteRuntimeReconnectBanner', () => { render() expect(screen.getByText('Reconnecting to remote runtime')).toBeInTheDocument() - expect(screen.getByText(/retry for up to one minute/)).toBeInTheDocument() + expect(screen.getByText(/retrying automatically/)).toBeInTheDocument() expect(screen.queryByRole('button')).not.toBeInTheDocument() }) diff --git a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx index c629fa71a36..1cf0fee0903 100644 --- a/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx +++ b/src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx @@ -50,7 +50,7 @@ export function TerminalRemoteRuntimeReconnectBanner({ {retrying ? translate( 'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.retryingBody', - 'Orca will retry for up to one minute. This terminal will resume if the connection returns.' + 'Orca is retrying automatically. This terminal will resume if the connection returns.' ) : translate( 'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.disconnectedBody', diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts new file mode 100644 index 00000000000..93a84f57647 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-runtime-connect-failure-recovery.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + createRemoteRuntimeTransportMocks, + type MultiplexSubscriptionCallbacks +} from './remote-runtime-pty-transport-test-harness' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' + +let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null +let resolvedPaneHandle = 'terminal-1' + +const { runtimeCall, resetRemoteRuntimeTransport } = createRemoteRuntimeTransportMocks({ + getCallbacks: () => subscriptionCallbacks, + setCallbacks: (callbacks) => { + subscriptionCallbacks = callbacks + }, + getResolvedPaneHandle: () => resolvedPaneHandle, + setResolvedPaneHandle: (handle) => { + resolvedPaneHandle = handle + } +}) + +// #12684: connect() classified these failures as recoverable and then latched 'disconnected' with +// nothing armed — no backoff timer, no parked retry, and a Reconnect button that returned false. +describe('recoverable connect failures on a remote runtime pane', () => { + let resolvePaneCalls = 0 + + function installUnreachableRuntime(): void { + resolvePaneCalls = 0 + runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method === 'terminal.resolvePane') { + resolvePaneCalls += 1 + } + throw Object.assign(new Error('Remote Orca runtime closed the connection.'), { + code: 'remote_runtime_unavailable' + }) + }) + } + + beforeEach(() => { + resetRemoteRuntimeTransport() + }) + + it('keeps retrying a recoverable connect failure instead of latching immediately', async () => { + vi.useFakeTimers() + try { + installUnreachableRuntime() + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + + await transport.connect({ + url: '', + sessionId: 'remote:env-1@@', + callbacks: { onError } + }) + + // Loss of contact is unverifiable, not a dead terminal: automatic recovery must still be running. + expect(resolvePaneCalls).toBe(1) + expect(transport.getRecoveryState?.().phase).toBe('backoff') + expect(onError).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(1_000) + expect(resolvePaneCalls).toBeGreaterThan(1) + + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('leaves both revival paths armed once the recovery window is spent', async () => { + vi.useFakeTimers() + try { + installUnreachableRuntime() + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + // Why dynamic: resetRemoteRuntimeTransport() re-registers the module graph, and the retry + // registry only sees panes from the same instance the transport was loaded from. + const { retryAllRemoteRuntimePtyRecoveriesNow } = + await import('./remote-runtime-pty-recovery-state') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + + await transport.connect({ url: '', sessionId: 'remote:env-1@@', callbacks: {} }) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000) + + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + const callsAtCutoff = resolvePaneCalls + + // The cutoff stops self-initiated retries only; online/resume must still find a parked retry. + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1) + await vi.advanceTimersByTimeAsync(1_000) + expect(resolvePaneCalls).toBeGreaterThan(callsAtCutoff) + + // ...and so must the Reconnect button, which returned false before #12684. + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + expect(transport.retryRecovery?.()).toBe(true) + + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts index b797a0f5877..9ac752b2b31 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-outage-toast-flood-and-stuck-reconnect.test.ts @@ -26,6 +26,7 @@ import { import { RuntimeRpcCallQueueOverloadError } from '../../../../shared/runtime-rpc-call-queue' import { withRemoteRuntimeTailscaleHint } from '../../../../shared/remote-runtime-tailscale-hint' import type { PtyTransportRecoveryState } from './pty-transport-types' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' const ELECTRON_IPC_PREFIX = "Error invoking remote method 'runtimeEnvironments:call': " @@ -409,7 +410,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () = await vi.advanceTimersByTimeAsync(16_000) // Auto-recovery deadline latches the pane 'disconnected'. - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') // Connectivity restored; 'online'/system-resume trigger fires. @@ -417,7 +418,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () = await vi.advanceTimersByTimeAsync(16_000) // Latch again, then the user clicks the Reconnect banner. - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) transport.retryRecovery?.() await vi.advanceTimersByTimeAsync(16_000) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts index 886516414ca..61758efb299 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts @@ -8,6 +8,7 @@ import { encodeTerminalStreamText } from '../../../../shared/terminal-stream-protocol' import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' describe('remote runtime pty reattach after the bounded recovery window', () => { const runtimeCall = vi.fn() @@ -198,7 +199,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () => expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1) expect(transport.getRecoveryState?.().phase).not.toBe('disconnected') - await vi.advanceTimersByTimeAsync(50_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') // The cutoff must not tear down the accepted-snapshot listener; it is the only path back. expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1) @@ -227,7 +228,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () => const { transport, onError } = await attachStalePane() const handleEvents = await import('../../runtime/web-session-terminal-handle-events') - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') const listCallsAtCutoff = hostListCalls @@ -277,7 +278,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () => const { transport, onError } = await attachStalePane() const handleEvents = await import('../../runtime/web-session-terminal-handle-events') - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1) const listCallsAtCutoff = hostListCalls @@ -310,7 +311,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () => const { retryAllRemoteRuntimePtyRecoveriesNow } = await import('./remote-runtime-pty-recovery-state') - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') const listCallsAtCutoff = hostListCalls @@ -365,7 +366,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () => callbacks: { onError: vi.fn() } }) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') const callsBeforeRetry = runtimeCall.mock.calls.length @@ -378,4 +379,60 @@ describe('remote runtime pty reattach after the bounded recovery window', () => vi.useRealTimers() } }) + + // #12683: a fatal resubscribe latches the banner via markDisconnected(), but that latch is not + // evidence the auto-recovery window ran out, so it must not license reattaching a fenced handle. + it('does not reattach a fenced same handle when only a UI latch closed the window', async () => { + vi.useFakeTimers() + try { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'web-terminal-tab-1', + leafId: 'pane:1', + onPtyExit: vi.fn(), + onPtyRebind: vi.fn() + }) + transport.attach({ + existingPtyId: 'remote:env-1@@terminal-stale', + cols: 80, + rows: 24, + callbacks: { onError: vi.fn() } + }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + emitSnapshot(latestSubscribePayload().streamId, 'live before the drop') + + // The host keeps publishing the same handle, so no replacement can ever arrive. + runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method !== 'session.tabs.list') { + return { ok: true, result: {} } + } + hostListCalls += 1 + return { ok: true, result: hostSnapshot('terminal-stale', hostListCalls + 1, 'epoch-1') } + }) + // The stream drops and the resubscribe fails fatally with a stale handle: markDisconnected() + // latches the banner, then stale routing fences the handle with require-replacement. + // Only that one attempt fails, so a later reattach would succeed and be observable. + runtimeSubscribe.mockImplementationOnce(async () => { + throw new Error('terminal_handle_stale') + }) + subscriptionCallbacks?.onClose?.() + await vi.advanceTimersByTimeAsync(16_000) + expect(transport.getRecoveryState?.().phase).not.toBe('idle') + + const subscribesBeforeRepublish = subscribedTerminalHandles().length + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot('terminal-stale', 9, 'epoch-2'), + 'env-1' + ) + await vi.advanceTimersByTimeAsync(1_000) + + // The recovery deadline never fired, so the fenced handle stays fenced. + expect(subscribedTerminalHandles()).toHaveLength(subscribesBeforeRepublish) + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) }) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts index 5892ee2c19d..3c1ec0421a9 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts @@ -5,6 +5,7 @@ import { decodeTerminalStreamJson } from '../../../../shared/terminal-stream-protocol' import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' // Why: the recovery cutoff no longer tears down the retry registry entry or the accepted-snapshot // listener, so those two module-global collections are the only places a latched pane can accumulate. @@ -184,7 +185,7 @@ describe('remote runtime pty latched-pane retention', () => { for (let cycle = 0; cycle < 20; cycle += 1) { const transport = await attachStalePane(cycle) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') latched.push(await registries()) transport.destroy?.() @@ -208,7 +209,7 @@ describe('remote runtime pty latched-pane retention', () => { const settled: { subscribers: number; scheduled: number }[] = [] for (let cycle = 0; cycle < 20; cycle += 1) { const transport = await attachStalePane(cycle) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') transport.detach?.() await vi.advanceTimersByTimeAsync(1_000) @@ -227,7 +228,7 @@ describe('remote runtime pty latched-pane retention', () => { const transports: Awaited>[] = [] for (let pane = 0; pane < 8; pane += 1) { transports.push(await attachStalePane(pane)) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) } // Retention is per live pane, not per timeout: eight latched panes hold eight of each. expect(await registries()).toEqual({ subscribers: 8, scheduled: 8 }) @@ -247,7 +248,7 @@ describe('remote runtime pty latched-pane retention', () => { vi.useFakeTimers() try { const transport = await attachStalePane(0) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') const baseline = await registries() @@ -277,7 +278,7 @@ describe('remote runtime pty latched-pane retention', () => { const { retryAllRemoteRuntimePtyRecoveriesNow } = await import('./remote-runtime-pty-recovery-state') const transport = await attachStalePane(0) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') const baseline = await registries() @@ -295,7 +296,7 @@ describe('remote runtime pty latched-pane retention', () => { // A second trigger in the same window must find nothing to advance, so an online/resume // storm cannot stack fresh recovery epochs on one pane. expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') observed.push({ ...(await registries()), timers: vi.getTimerCount(), revived }) } @@ -325,7 +326,7 @@ describe('remote runtime pty latched-pane retention', () => { try { const handleEvents = await import('../../runtime/web-session-terminal-handle-events') const transport = await attachStalePane(0) - await vi.advanceTimersByTimeAsync(66_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) expect(transport.getRecoveryState?.().phase).toBe('disconnected') const baseline = await registries() diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts index d71af0b9afb..c245c184436 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts @@ -1,6 +1,8 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS, + REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS, + REMOTE_RUNTIME_RECOVERY_DELAYS_MS, RemoteRuntimePtyRecoveryState, retryAllRemoteRuntimePtyRecoveriesNow } from './remote-runtime-pty-recovery-state' @@ -63,7 +65,7 @@ describe('RemoteRuntimePtyRecoveryState', () => { const epoch = state.begin() state.schedule(epoch, retry) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(state.currentPhase).toBe('disconnected') expect(state.isActive).toBe(false) @@ -112,7 +114,7 @@ describe('RemoteRuntimePtyRecoveryState', () => { const state = new RemoteRuntimePtyRecoveryState() const firstEpoch = state.begin() - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) const manualEpoch = state.begin() expect(manualEpoch).toBe(firstEpoch + 1) @@ -227,4 +229,68 @@ describe('RemoteRuntimePtyRecoveryState', () => { expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0) state.dispose() }) + + // #11305: the schedule and the deadline lived as two independent literals and drifted apart. + it('keeps the backoff schedule inside the auto-recovery budget it arms', () => { + const scheduleSumMs = REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce( + (total, delayMs) => total + delayMs, + 0 + ) + + expect(scheduleSumMs).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) + // Every step also needs room for the attempt it leads into, or the tail is dead code + // whenever a half-open link makes each attempt burn its full RPC timeout. + expect( + scheduleSumMs + + REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS + ).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) + }) + + it('reaches every backoff step when each attempt burns a full RPC timeout', async () => { + vi.useFakeTimers() + const state = new RemoteRuntimePtyRecoveryState() + const attemptStartsMs: number[] = [] + const epoch = state.begin() + const startedAt = Date.now() + + const failSlowly = (currentEpoch: number): void => { + attemptStartsMs.push(Date.now() - startedAt) + // Silent-drop reconnects do not fail instantly; they time out. + setTimeout(() => { + if (state.isCurrent(currentEpoch)) { + state.schedule(currentEpoch, failSlowly) + } + }, REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS) + } + state.schedule(epoch, failSlowly) + + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) + + expect(attemptStartsMs.length).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length) + expect(state.currentPhase).toBe('disconnected') + state.dispose() + }) + + // #12683: markDisconnected() is a UI latch, not proof the window ran out. + it('only reports the auto-recovery window spent when the deadline actually fired', async () => { + vi.useFakeTimers() + const state = new RemoteRuntimePtyRecoveryState() + const epoch = state.begin() + state.schedule(epoch, vi.fn()) + + state.markDisconnected() + expect(state.currentPhase).toBe('disconnected') + expect(state.autoRecoveryDeadlineExpired).toBe(false) + + const secondEpoch = state.begin() + state.schedule(secondEpoch, vi.fn()) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) + + expect(state.currentPhase).toBe('disconnected') + expect(state.autoRecoveryDeadlineExpired).toBe(true) + + state.markHealthy() + expect(state.autoRecoveryDeadlineExpired).toBe(false) + state.dispose() + }) }) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts index fbdd50e443f..8a5db87a19d 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts @@ -1,5 +1,17 @@ -const RECOVERY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const -export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS = 60_000 +export const REMOTE_RUNTIME_RECOVERY_DELAYS_MS = [ + 250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000 +] as const + +// Why: mirrors DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS in the main-process runtime router; a silently +// dropped link burns the whole RPC timeout on the attempt each backoff step leads into. +export const REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS = 15_000 + +// Why derived, not hand-tuned: a literal deadline drifted below the ladder it arms, making the last +// backoff steps unreachable dead code (#11305). Loss of contact is never evidence of exit, so the +// window must outlast the schedule it advertises rather than the schedule being trimmed to fit. +export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS = + REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce((total, delayMs) => total + delayMs, 0) + + REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS export type RemoteRuntimePtyRecoveryPhase = | 'idle' @@ -34,6 +46,9 @@ export class RemoteRuntimePtyRecoveryState { private deadlineTimer: ReturnType | null = null private pendingRetry: ((epoch: number) => void) | null = null private pendingEpoch: number | null = null + // Why: only the wall-clock deadline proves the auto-recovery window was actually spent; a UI latch + // via markDisconnected() must not forge that evidence (#12683). + private deadlineExpired = false constructor(private readonly onChange?: () => void) {} @@ -53,6 +68,10 @@ export class RemoteRuntimePtyRecoveryState { return this.attempt } + get autoRecoveryDeadlineExpired(): boolean { + return this.deadlineExpired + } + begin(): number { if (this.phase === 'disposed') { return this.epoch @@ -82,7 +101,10 @@ export class RemoteRuntimePtyRecoveryState { } this.clearRetryTimer() this.phase = 'backoff' - const delayMs = RECOVERY_DELAYS_MS[Math.min(this.attempt, RECOVERY_DELAYS_MS.length - 1)] + const delayMs = + REMOTE_RUNTIME_RECOVERY_DELAYS_MS[ + Math.min(this.attempt, REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length - 1) + ] this.attempt += 1 this.pendingRetry = retry this.pendingEpoch = epoch @@ -152,6 +174,7 @@ export class RemoteRuntimePtyRecoveryState { if (this.phase === 'disposed') { return } + this.deadlineExpired = false this.clearTimers() this.phase = 'idle' this.attempt = 0 @@ -175,6 +198,7 @@ export class RemoteRuntimePtyRecoveryState { if (this.phase === 'disposed') { return } + this.deadlineExpired = false this.epoch += 1 this.clearTimers() this.phase = 'idle' @@ -191,11 +215,13 @@ export class RemoteRuntimePtyRecoveryState { private armDeadline(epoch: number): void { this.clearDeadlineTimer() + this.deadlineExpired = false const timer = setTimeout(() => { if (this.deadlineTimer !== timer || !this.isCurrent(epoch)) { return } this.deadlineTimer = null + this.deadlineExpired = true // Why: the cutoff stops self-initiated retries but must keep the pane revivable by online/resume/reconnect. this.stopRetryTimer() this.phase = 'disconnected' diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts index b34ec4f3fea..59dc3612ed1 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts @@ -4,6 +4,7 @@ import { createRemoteRuntimeTransportMocks, type MultiplexSubscriptionCallbacks } from './remote-runtime-pty-transport-test-harness' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null let resolvedPaneHandle = 'terminal-1' @@ -81,7 +82,7 @@ describe('createRemoteRuntimePtyTransport', () => { let createCalls = 0 runtimeCall.mockImplementation(async (args: { method: string }) => { if (args.method === 'status.get') { - vi.setSystemTime(startedAt + 59_000) + vi.setSystemTime(startedAt + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS - 1_000) return { ok: true, result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] } @@ -163,7 +164,7 @@ describe('createRemoteRuntimePtyTransport', () => { transport.destroy?.() }) - it('stops unknown terminal-create recovery after one minute and remains manually retryable', async () => { + it('stops unknown terminal-create recovery at the cutoff and remains manually retryable', async () => { vi.useFakeTimers() try { let reachable = false @@ -205,7 +206,7 @@ describe('createRemoteRuntimePtyTransport', () => { onRecoveryStateChange: (state) => recoveryStates.push(state.phase) } }) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) await connect const callsAtCutoff = runtimeCall.mock.calls.length @@ -218,7 +219,7 @@ describe('createRemoteRuntimePtyTransport', () => { statusTimesOut = true expect(transport.retryRecovery?.()).toBe(true) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) const callsAtManualCutoff = runtimeCall.mock.calls.length expect(transport.getRecoveryState?.().phase).toBe('disconnected') await vi.advanceTimersByTimeAsync(5 * 60_000) @@ -278,7 +279,7 @@ describe('createRemoteRuntimePtyTransport', () => { }) const connect = transport.connect({ url: '', callbacks: {} }) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) await connect expect(transport.getRecoveryState?.().phase).toBe('disconnected') diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts index 8817d5c23e4..53c1921fe74 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stale-handle-recovery.test.ts @@ -4,6 +4,7 @@ import { readyHostSessionInventoryResponse, type MultiplexSubscriptionCallbacks } from './remote-runtime-pty-transport-test-harness' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null let resolvedPaneHandle = 'terminal-1' @@ -72,7 +73,7 @@ describe('createRemoteRuntimePtyTransport', () => { expect(hostListCalls).toBe(callsAfterTwoWindows) expect(transport.getRecoveryState?.().phase).toBe('recovering') - await vi.advanceTimersByTimeAsync(9_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') expect(subscribedTerminalHandles()).toEqual(['terminal-stable']) transport.destroy?.() @@ -180,7 +181,7 @@ describe('createRemoteRuntimePtyTransport', () => { 'terminal-flapping' ]) expect(transport.isConnected()).toBe(false) - await vi.advanceTimersByTimeAsync(45_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') transport.destroy?.() } finally { diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts index 48a24a2c084..5a61a5587e6 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-sticky-replacement-policy.test.ts @@ -9,6 +9,7 @@ import { readyHostSessionInventoryResponse, type MultiplexSubscriptionCallbacks } from './remote-runtime-pty-transport-test-harness' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null let resolvedPaneHandle = 'terminal-1' @@ -160,7 +161,7 @@ describe('createRemoteRuntimePtyTransport', () => { expect(transport.isConnected()).toBe(false) expect(onPtyExit).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(44_001) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') expect(subscribedTerminalHandles()).toHaveLength(3) } finally { diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts index 4ef1075e3f7..e0b85414e3f 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts @@ -9,6 +9,7 @@ import { createRemoteRuntimeTransportMocks, type MultiplexSubscriptionCallbacks } from './remote-runtime-pty-transport-test-harness' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null let resolvedPaneHandle = 'terminal-1' @@ -534,7 +535,7 @@ describe('createRemoteRuntimePtyTransport', () => { partitioned = true callbacksByConnection[0].onClose?.() - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) const disconnectedState = transport.getRecoveryState?.() const callsAtCutoff = runtimeSubscribe.mock.calls.length diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts index f8c07ea045d..784ed9d8768 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-web-mirror-recovery.test.ts @@ -3,6 +3,10 @@ import { createRemoteRuntimeTransportMocks, type MultiplexSubscriptionCallbacks } from './remote-runtime-pty-transport-test-harness' +import { + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS, + REMOTE_RUNTIME_RECOVERY_DELAYS_MS +} from './remote-runtime-pty-recovery-state' let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null let resolvedPaneHandle = 'terminal-1' @@ -140,10 +144,11 @@ describe('createRemoteRuntimePtyTransport', () => { existingPtyId: 'remote:env-1@@stale-client-handle', callbacks: {} }) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) const attemptsAtCutoff = runtimeSubscribe.mock.calls.length - expect(attemptsAtCutoff).toBe(8) + // Why: every backoff step must be reachable inside the window it arms (#11305). + expect(attemptsAtCutoff).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length) expect(transport.getRecoveryState?.().phase).toBe('disconnected') await vi.advanceTimersByTimeAsync(5 * 60_000) @@ -235,7 +240,7 @@ describe('createRemoteRuntimePtyTransport', () => { await vi.advanceTimersByTimeAsync(250) expect(activateAttempts).toBe(2) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') rejectInFlight( @@ -292,7 +297,7 @@ describe('createRemoteRuntimePtyTransport', () => { await vi.advanceTimersByTimeAsync(250) expect(runtimeSubscribe).toHaveBeenCalledTimes(1) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') rejectSubscription( @@ -349,7 +354,7 @@ describe('createRemoteRuntimePtyTransport', () => { expect.objectContaining({ method: 'terminal.resolvePane' }) ) - await vi.advanceTimersByTimeAsync(60_000) + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) expect(transport.getRecoveryState?.().phase).toBe('disconnected') resolveMetadata({ diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index 1e5782fe496..b43dc3f8a51 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -247,7 +247,9 @@ export function createRemoteRuntimePtyTransport( clearPublishedHandleWait() } if (recovery.currentPhase === 'disconnected') { - autoRecoveryWindowSpent = true + // Why: only the wall-clock deadline is evidence the window was spent; a UI latch from a fatal + // resubscribe must not license reattaching a fenced same handle (#12683). + autoRecoveryWindowSpent ||= recovery.autoRecoveryDeadlineExpired // Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic. subscriptionGeneration += 1 closeMultiplexedStream() @@ -870,6 +872,37 @@ export function createRemoteRuntimePtyTransport( return false } + // Why: a recoverable connect failure is unverifiable contact loss, not a dead terminal, so retry + // whichever path can still reach the pane instead of latching with nothing armed (#12684). + function retryAfterRecoverableConnectFailure(nextEpoch: number): void { + if (destroyed || terminalEnded) { + return + } + if (connected && handle) { + scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(handle), nextEpoch) + return + } + replayLastTransportEntryPoint() + } + + // Why: schedule() both auto-retries inside the window and leaves the retry parked when the deadline + // latches, so online/resume and the Reconnect button always find something to fire. + function scheduleConnectRetryAfterRecoverableFailure(): void { + if (destroyed) { + return + } + // Why: an ambiguous create already owns a reconciliation-gated retry that only Reconnect may + // re-enter; auto-replaying here would just re-probe a runtime that cannot reconcile. + if (terminalCreateNeedsReconciliation || agentSessionRequiresHostAuthorityReplay) { + recovery.markDisconnected() + return + } + const recoveryEpoch = recovery.isActive ? recovery.currentEpoch : recovery.begin() + if (!recovery.schedule(recoveryEpoch, retryAfterRecoverableConnectFailure)) { + recovery.markDisconnected() + } + } + async function attachHostSessionMirror( options: { cols?: number; rows?: number }, notifySpawn = true, @@ -2314,7 +2347,7 @@ export function createRemoteRuntimePtyTransport( } else if ( isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error)) ) { - recovery.markDisconnected() + scheduleConnectRetryAfterRecoverableFailure() } else { recovery.cancel() emitRecoveryState() @@ -2616,6 +2649,12 @@ export function createRemoteRuntimePtyTransport( void transport.connect(lastConnectOptions) return true } + // Why: online/resume fires a parked retry; the button must not be weaker than an event (#12684). + if (!destroyed && !terminalEnded && recovery.currentPhase === 'disconnected') { + if (recovery.retryNow()) { + return true + } + } if ( destroyed || terminalEnded || diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index b546a930830..098e9448f9f 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -3251,7 +3251,7 @@ "TerminalRemoteRuntimeReconnectBanner": { "retryingTitle": "Reconnecting to remote runtime", "disconnectedTitle": "Remote runtime disconnected", - "retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.", + "retryingBody": "Orca is retrying automatically. This terminal will resume if the connection returns.", "disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.", "reconnectButton": "Reconnect" }, diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index f4a4fd931de..0ac5d624ede 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -2881,7 +2881,7 @@ "TerminalRemoteRuntimeReconnectBanner": { "retryingTitle": "Reconnecting to remote runtime", "disconnectedTitle": "Remote runtime disconnected", - "retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.", + "retryingBody": "Orca is retrying automatically. This terminal will resume if the connection returns.", "disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.", "reconnectButton": "Reconnect" } diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index ee6e4440f79..d3af25ad697 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -2881,7 +2881,7 @@ "TerminalRemoteRuntimeReconnectBanner": { "retryingTitle": "リモートランタイムに再接続中", "disconnectedTitle": "リモートランタイムが切断されました", - "retryingBody": "Orca は最大1分間再試行します。接続が復元されると、このターミナルが再開されます。", + "retryingBody": "Orca は自動的に再試行します。接続が復元されると、このターミナルが再開されます。", "disconnectedBody": "自動再試行が停止しました。このターミナルセッションを再開するには再接続してください。", "reconnectButton": "再接続" } diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 76ed151ca2c..47d9e48fd26 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -2886,7 +2886,7 @@ "TerminalRemoteRuntimeReconnectBanner": { "retryingTitle": "원격 런타임에 다시 연결 중", "disconnectedTitle": "원격 런타임 연결 끊김", - "retryingBody": "Orca는 최대 1분간 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.", + "retryingBody": "Orca가 자동으로 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.", "disconnectedBody": "자동 재시도가 중지되었습니다. 이 터미널 세션을 재개하려면 다시 연결하세요.", "reconnectButton": "다시 연결" } diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 5c1324afa12..b097b789571 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -2896,7 +2896,7 @@ "TerminalRemoteRuntimeReconnectBanner": { "retryingTitle": "正在重新连接到远程运行时", "disconnectedTitle": "远程运行时已断开连接", - "retryingBody": "Orca 将重试最多一分钟。如果连接恢复，此终端将恢复。", + "retryingBody": "Orca 正在自动重试。如果连接恢复，此终端将恢复。", "disconnectedBody": "自动重试已停止。重新连接以恢复此终端会话。", "reconnectButton": "重新连接" }