mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
fix(remote-runtime): make every advertised recovery attempt reachable, and stop two recovery latches (#17822)
* fix(remote-runtime): derive the recovery budget and stop faking a spent window #11305: RECOVERY_DELAYS_MS summed to 60,750ms against a hand-written REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS of 60,000ms, so the ladder's tail was unreachable. Derive the deadline from the schedule plus one RPC timeout per step so a half-open link can actually reach every backoff step, and pin the relation with a test that fails if the sum ever outgrows the budget. #12683: markDisconnected() is a UI latch, not proof the auto-recovery window ran out. Track deadline expiry on the recovery state and only let that license the same-handle reattach that bypasses require-replacement fencing. #12684: a recoverable connect() failure latched 'disconnected' with no armed retry, no parked retry and a Reconnect button that returned false. Schedule a bounded retry (which the deadline parks for online/resume) and let the button fire a parked retry. * fix(remote-runtime): stop a post-latch connect failure from re-arming the recovery window The last attempt's RPC budget expires at the same instant as the deadline, so a silently dropped link rejects after phase latched to 'disconnected'. begin() then started a fresh full-length window, so the budget never actually expired. Park the retry under the latched epoch instead, which keeps online/resume/Reconnect armed even when the deadline lands mid-attempt with nothing scheduled. Also fences the same-handle end-reuse window on its own 60s constant so the derived recovery budget no longer silently triples an unrelated stale-handle check. * fix(i18n): restore the activity-options key the rebase dropped * fix(i18n): union en.json with main so the rebase cannot drop keys
This commit is contained in:
+1
-1
@@ -17,7 +17,7 @@ describe('TerminalRemoteRuntimeReconnectBanner', () => {
|
||||
render(<TerminalRemoteRuntimeReconnectBanner phase="backoff" onReconnect={vi.fn()} />)
|
||||
|
||||
expect(screen.getByText('Reconnecting to remote runtime')).toBeInTheDocument()
|
||||
expect(screen.getByText(/retry for up to one minute/)).toBeInTheDocument()
|
||||
expect(screen.getByText(/retrying automatically/)).toBeInTheDocument()
|
||||
expect(screen.queryByRole('button')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@ export function TerminalRemoteRuntimeReconnectBanner({
|
||||
{retrying
|
||||
? translate(
|
||||
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.retryingBody',
|
||||
'Orca will retry for up to one minute. This terminal will resume if the connection returns.'
|
||||
'Orca is retrying automatically. This terminal will resume if the connection returns.'
|
||||
)
|
||||
: translate(
|
||||
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.disconnectedBody',
|
||||
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
createRemoteRuntimeTransportMocks,
|
||||
type MultiplexSubscriptionCallbacks
|
||||
} from './remote-runtime-pty-transport-test-harness'
|
||||
import {
|
||||
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
|
||||
REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
|
||||
} from './remote-runtime-pty-recovery-state'
|
||||
|
||||
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
|
||||
let resolvedPaneHandle = 'terminal-1'
|
||||
|
||||
const { runtimeCall, resetRemoteRuntimeTransport } = createRemoteRuntimeTransportMocks({
|
||||
getCallbacks: () => subscriptionCallbacks,
|
||||
setCallbacks: (callbacks) => {
|
||||
subscriptionCallbacks = callbacks
|
||||
},
|
||||
getResolvedPaneHandle: () => resolvedPaneHandle,
|
||||
setResolvedPaneHandle: (handle) => {
|
||||
resolvedPaneHandle = handle
|
||||
}
|
||||
})
|
||||
|
||||
// #12684: connect() classified these failures as recoverable and then latched 'disconnected' with
|
||||
// nothing armed — no backoff timer, no parked retry, and a Reconnect button that returned false.
|
||||
describe('recoverable connect failures on a remote runtime pane', () => {
|
||||
let resolvePaneCalls = 0
|
||||
|
||||
function installUnreachableRuntime(): void {
|
||||
resolvePaneCalls = 0
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method === 'terminal.resolvePane') {
|
||||
resolvePaneCalls += 1
|
||||
}
|
||||
throw Object.assign(new Error('Remote Orca runtime closed the connection.'), {
|
||||
code: 'remote_runtime_unavailable'
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Why: installUnreachableRuntime() rejects synchronously, so every failure lands during a backoff
|
||||
// wait. A silently dropped link instead burns the whole RPC budget, so the rejection arrives while
|
||||
// the attempt is still in flight — including after the auto-recovery deadline has already latched.
|
||||
function installSilentlyDroppedRuntime(): void {
|
||||
resolvePaneCalls = 0
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method === 'terminal.resolvePane') {
|
||||
resolvePaneCalls += 1
|
||||
}
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS)
|
||||
})
|
||||
throw Object.assign(new Error('Remote Orca runtime closed the connection.'), {
|
||||
code: 'remote_runtime_unavailable'
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetRemoteRuntimeTransport()
|
||||
})
|
||||
|
||||
it('keeps retrying a recoverable connect failure instead of latching immediately', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
installUnreachableRuntime()
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
|
||||
await transport.connect({
|
||||
url: '',
|
||||
sessionId: 'remote:env-1@@',
|
||||
callbacks: { onError }
|
||||
})
|
||||
|
||||
// Loss of contact is unverifiable, not a dead terminal: automatic recovery must still be running.
|
||||
expect(resolvePaneCalls).toBe(1)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('backoff')
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(resolvePaneCalls).toBeGreaterThan(1)
|
||||
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves both revival paths armed once the recovery window is spent', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
installUnreachableRuntime()
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
// Why dynamic: resetRemoteRuntimeTransport() re-registers the module graph, and the retry
|
||||
// registry only sees panes from the same instance the transport was loaded from.
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
|
||||
await transport.connect({ url: '', sessionId: 'remote:env-1@@', callbacks: {} })
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000)
|
||||
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const callsAtCutoff = resolvePaneCalls
|
||||
|
||||
// The cutoff stops self-initiated retries only; online/resume must still find a parked retry.
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(resolvePaneCalls).toBeGreaterThan(callsAtCutoff)
|
||||
|
||||
// ...and so must the Reconnect button, which returned false before #12684.
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
expect(transport.retryRecovery?.()).toBe(true)
|
||||
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
it('keeps the window bounded when a silent drop fails after the deadline latched', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
installSilentlyDroppedRuntime()
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
|
||||
void transport.connect({ url: '', sessionId: 'remote:env-1@@', callbacks: {} })
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS * 2)
|
||||
|
||||
// The in-flight rejection must not begin a new epoch; that re-arms a full-length window forever.
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const callsAtCutoff = resolvePaneCalls
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS * 2)
|
||||
expect(resolvePaneCalls).toBe(callsAtCutoff)
|
||||
|
||||
// A deadline that lands mid-attempt parks nothing, so the latch must still stay revivable.
|
||||
expect(transport.retryRecovery?.()).toBe(true)
|
||||
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
+3
-2
@@ -26,6 +26,7 @@ import {
|
||||
import { RuntimeRpcCallQueueOverloadError } from '../../../../shared/runtime-rpc-call-queue'
|
||||
import { withRemoteRuntimeTailscaleHint } from '../../../../shared/remote-runtime-tailscale-hint'
|
||||
import type { PtyTransportRecoveryState } from './pty-transport-types'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
const ELECTRON_IPC_PREFIX = "Error invoking remote method 'runtimeEnvironments:call': "
|
||||
|
||||
@@ -409,7 +410,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () =
|
||||
await vi.advanceTimersByTimeAsync(16_000)
|
||||
|
||||
// Auto-recovery deadline latches the pane 'disconnected'.
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
// Connectivity restored; 'online'/system-resume trigger fires.
|
||||
@@ -417,7 +418,7 @@ describe('remote runtime outage: toast flood and stuck reconnect (issue3)', () =
|
||||
await vi.advanceTimersByTimeAsync(16_000)
|
||||
|
||||
// Latch again, then the user clicks the Reconnect banner.
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
transport.retryRecovery?.()
|
||||
await vi.advanceTimersByTimeAsync(16_000)
|
||||
|
||||
|
||||
+62
-5
@@ -8,6 +8,7 @@ import {
|
||||
encodeTerminalStreamText
|
||||
} from '../../../../shared/terminal-stream-protocol'
|
||||
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
describe('remote runtime pty reattach after the bounded recovery window', () => {
|
||||
const runtimeCall = vi.fn()
|
||||
@@ -198,7 +199,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
|
||||
expect(transport.getRecoveryState?.().phase).not.toBe('disconnected')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(50_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
// The cutoff must not tear down the accepted-snapshot listener; it is the only path back.
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
|
||||
@@ -227,7 +228,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
|
||||
const { transport, onError } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const listCallsAtCutoff = hostListCalls
|
||||
|
||||
@@ -277,7 +278,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
|
||||
const { transport, onError } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
|
||||
const listCallsAtCutoff = hostListCalls
|
||||
@@ -310,7 +311,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const listCallsAtCutoff = hostListCalls
|
||||
|
||||
@@ -365,7 +366,7 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
|
||||
callbacks: { onError: vi.fn() }
|
||||
})
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
const callsBeforeRetry = runtimeCall.mock.calls.length
|
||||
@@ -378,4 +379,60 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
// #12683: a fatal resubscribe latches the banner via markDisconnected(), but that latch is not
|
||||
// evidence the auto-recovery window ran out, so it must not license reattaching a fenced handle.
|
||||
it('does not reattach a fenced same handle when only a UI latch closed the window', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'web-terminal-tab-1',
|
||||
leafId: 'pane:1',
|
||||
onPtyExit: vi.fn(),
|
||||
onPtyRebind: vi.fn()
|
||||
})
|
||||
transport.attach({
|
||||
existingPtyId: 'remote:env-1@@terminal-stale',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
callbacks: { onError: vi.fn() }
|
||||
})
|
||||
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'live before the drop')
|
||||
|
||||
// The host keeps publishing the same handle, so no replacement can ever arrive.
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method !== 'session.tabs.list') {
|
||||
return { ok: true, result: {} }
|
||||
}
|
||||
hostListCalls += 1
|
||||
return { ok: true, result: hostSnapshot('terminal-stale', hostListCalls + 1, 'epoch-1') }
|
||||
})
|
||||
// The stream drops and the resubscribe fails fatally with a stale handle: markDisconnected()
|
||||
// latches the banner, then stale routing fences the handle with require-replacement.
|
||||
// Only that one attempt fails, so a later reattach would succeed and be observable.
|
||||
runtimeSubscribe.mockImplementationOnce(async () => {
|
||||
throw new Error('terminal_handle_stale')
|
||||
})
|
||||
subscriptionCallbacks?.onClose?.()
|
||||
await vi.advanceTimersByTimeAsync(16_000)
|
||||
expect(transport.getRecoveryState?.().phase).not.toBe('idle')
|
||||
|
||||
const subscribesBeforeRepublish = subscribedTerminalHandles().length
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot('terminal-stale', 9, 'epoch-2'),
|
||||
'env-1'
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
// The recovery deadline never fired, so the fenced handle stays fenced.
|
||||
expect(subscribedTerminalHandles()).toHaveLength(subscribesBeforeRepublish)
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
+8
-7
@@ -5,6 +5,7 @@ import {
|
||||
decodeTerminalStreamJson
|
||||
} from '../../../../shared/terminal-stream-protocol'
|
||||
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
// Why: the recovery cutoff no longer tears down the retry registry entry or the accepted-snapshot
|
||||
// listener, so those two module-global collections are the only places a latched pane can accumulate.
|
||||
@@ -184,7 +185,7 @@ describe('remote runtime pty latched-pane retention', () => {
|
||||
|
||||
for (let cycle = 0; cycle < 20; cycle += 1) {
|
||||
const transport = await attachStalePane(cycle)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
latched.push(await registries())
|
||||
transport.destroy?.()
|
||||
@@ -208,7 +209,7 @@ describe('remote runtime pty latched-pane retention', () => {
|
||||
const settled: { subscribers: number; scheduled: number }[] = []
|
||||
for (let cycle = 0; cycle < 20; cycle += 1) {
|
||||
const transport = await attachStalePane(cycle)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
transport.detach?.()
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
@@ -227,7 +228,7 @@ describe('remote runtime pty latched-pane retention', () => {
|
||||
const transports: Awaited<ReturnType<typeof attachStalePane>>[] = []
|
||||
for (let pane = 0; pane < 8; pane += 1) {
|
||||
transports.push(await attachStalePane(pane))
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
}
|
||||
// Retention is per live pane, not per timeout: eight latched panes hold eight of each.
|
||||
expect(await registries()).toEqual({ subscribers: 8, scheduled: 8 })
|
||||
@@ -247,7 +248,7 @@ describe('remote runtime pty latched-pane retention', () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const transport = await attachStalePane(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
const baseline = await registries()
|
||||
@@ -277,7 +278,7 @@ describe('remote runtime pty latched-pane retention', () => {
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
const transport = await attachStalePane(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
const baseline = await registries()
|
||||
@@ -295,7 +296,7 @@ describe('remote runtime pty latched-pane retention', () => {
|
||||
// A second trigger in the same window must find nothing to advance, so an online/resume
|
||||
// storm cannot stack fresh recovery epochs on one pane.
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
observed.push({ ...(await registries()), timers: vi.getTimerCount(), revived })
|
||||
}
|
||||
@@ -325,7 +326,7 @@ describe('remote runtime pty latched-pane retention', () => {
|
||||
try {
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
const transport = await attachStalePane(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const baseline = await registries()
|
||||
|
||||
|
||||
+68
-2
@@ -1,6 +1,8 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
|
||||
REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS,
|
||||
REMOTE_RUNTIME_RECOVERY_DELAYS_MS,
|
||||
RemoteRuntimePtyRecoveryState,
|
||||
retryAllRemoteRuntimePtyRecoveriesNow
|
||||
} from './remote-runtime-pty-recovery-state'
|
||||
@@ -63,7 +65,7 @@ describe('RemoteRuntimePtyRecoveryState', () => {
|
||||
const epoch = state.begin()
|
||||
state.schedule(epoch, retry)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
expect(state.isActive).toBe(false)
|
||||
@@ -112,7 +114,7 @@ describe('RemoteRuntimePtyRecoveryState', () => {
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const firstEpoch = state.begin()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
const manualEpoch = state.begin()
|
||||
|
||||
expect(manualEpoch).toBe(firstEpoch + 1)
|
||||
@@ -227,4 +229,68 @@ describe('RemoteRuntimePtyRecoveryState', () => {
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
|
||||
state.dispose()
|
||||
})
|
||||
|
||||
// #11305: the schedule and the deadline lived as two independent literals and drifted apart.
|
||||
it('keeps the backoff schedule inside the auto-recovery budget it arms', () => {
|
||||
const scheduleSumMs = REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce(
|
||||
(total, delayMs) => total + delayMs,
|
||||
0
|
||||
)
|
||||
|
||||
expect(scheduleSumMs).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
// Every step also needs room for the attempt it leads into, or the tail is dead code
|
||||
// whenever a half-open link makes each attempt burn its full RPC timeout.
|
||||
expect(
|
||||
scheduleSumMs +
|
||||
REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
|
||||
).toBeLessThanOrEqual(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
})
|
||||
|
||||
it('reaches every backoff step when each attempt burns a full RPC timeout', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const attemptStartsMs: number[] = []
|
||||
const epoch = state.begin()
|
||||
const startedAt = Date.now()
|
||||
|
||||
const failSlowly = (currentEpoch: number): void => {
|
||||
attemptStartsMs.push(Date.now() - startedAt)
|
||||
// Silent-drop reconnects do not fail instantly; they time out.
|
||||
setTimeout(() => {
|
||||
if (state.isCurrent(currentEpoch)) {
|
||||
state.schedule(currentEpoch, failSlowly)
|
||||
}
|
||||
}, REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS)
|
||||
}
|
||||
state.schedule(epoch, failSlowly)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
|
||||
expect(attemptStartsMs.length).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length)
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
state.dispose()
|
||||
})
|
||||
|
||||
// #12683: markDisconnected() is a UI latch, not proof the window ran out.
|
||||
it('only reports the auto-recovery window spent when the deadline actually fired', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const epoch = state.begin()
|
||||
state.schedule(epoch, vi.fn())
|
||||
|
||||
state.markDisconnected()
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
expect(state.autoRecoveryDeadlineExpired).toBe(false)
|
||||
|
||||
const secondEpoch = state.begin()
|
||||
state.schedule(secondEpoch, vi.fn())
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
expect(state.autoRecoveryDeadlineExpired).toBe(true)
|
||||
|
||||
state.markHealthy()
|
||||
expect(state.autoRecoveryDeadlineExpired).toBe(false)
|
||||
state.dispose()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
const RECOVERY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
|
||||
export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS = 60_000
|
||||
export const REMOTE_RUNTIME_RECOVERY_DELAYS_MS = [
|
||||
250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000
|
||||
] as const
|
||||
|
||||
// Why: mirrors DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS in the main-process runtime router; a silently
|
||||
// dropped link burns the whole RPC timeout on the attempt each backoff step leads into.
|
||||
export const REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS = 15_000
|
||||
|
||||
// Why derived, not hand-tuned: a literal deadline drifted below the ladder it arms, making the last
|
||||
// backoff steps unreachable dead code (#11305). Loss of contact is never evidence of exit, so the
|
||||
// window must outlast the schedule it advertises rather than the schedule being trimmed to fit.
|
||||
export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS =
|
||||
REMOTE_RUNTIME_RECOVERY_DELAYS_MS.reduce((total, delayMs) => total + delayMs, 0) +
|
||||
REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length * REMOTE_RUNTIME_RECOVERY_ATTEMPT_BUDGET_MS
|
||||
|
||||
export type RemoteRuntimePtyRecoveryPhase =
|
||||
| 'idle'
|
||||
@@ -34,6 +46,9 @@ export class RemoteRuntimePtyRecoveryState {
|
||||
private deadlineTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private pendingRetry: ((epoch: number) => void) | null = null
|
||||
private pendingEpoch: number | null = null
|
||||
// Why: only the wall-clock deadline proves the auto-recovery window was actually spent; a UI latch
|
||||
// via markDisconnected() must not forge that evidence (#12683).
|
||||
private deadlineExpired = false
|
||||
|
||||
constructor(private readonly onChange?: () => void) {}
|
||||
|
||||
@@ -53,6 +68,10 @@ export class RemoteRuntimePtyRecoveryState {
|
||||
return this.attempt
|
||||
}
|
||||
|
||||
get autoRecoveryDeadlineExpired(): boolean {
|
||||
return this.deadlineExpired
|
||||
}
|
||||
|
||||
begin(): number {
|
||||
if (this.phase === 'disposed') {
|
||||
return this.epoch
|
||||
@@ -82,7 +101,10 @@ export class RemoteRuntimePtyRecoveryState {
|
||||
}
|
||||
this.clearRetryTimer()
|
||||
this.phase = 'backoff'
|
||||
const delayMs = RECOVERY_DELAYS_MS[Math.min(this.attempt, RECOVERY_DELAYS_MS.length - 1)]
|
||||
const delayMs =
|
||||
REMOTE_RUNTIME_RECOVERY_DELAYS_MS[
|
||||
Math.min(this.attempt, REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length - 1)
|
||||
]
|
||||
this.attempt += 1
|
||||
this.pendingRetry = retry
|
||||
this.pendingEpoch = epoch
|
||||
@@ -107,11 +129,22 @@ export class RemoteRuntimePtyRecoveryState {
|
||||
|
||||
// Why: a wait that ends with no liveness evidence arms no timer, so park a retry or online/resume/reconnect find nothing to revive.
|
||||
parkRetryForExternalTrigger(epoch: number, retry: (epoch: number) => void): boolean {
|
||||
if (!this.isCurrent(epoch) || this.pendingRetry !== null) {
|
||||
return this.isCurrent(epoch) && this.parkRetry(retry)
|
||||
}
|
||||
|
||||
// Why: the deadline can latch while an attempt is still in flight, before schedule() parked anything,
|
||||
// so the late failure has no live epoch to join and must not begin a new one — that would re-arm a
|
||||
// full-length window and the budget would never actually expire.
|
||||
parkRetryAfterDeadline(retry: (epoch: number) => void): boolean {
|
||||
return this.phase === 'disconnected' && this.parkRetry(retry)
|
||||
}
|
||||
|
||||
private parkRetry(retry: (epoch: number) => void): boolean {
|
||||
if (this.pendingRetry !== null) {
|
||||
return false
|
||||
}
|
||||
this.pendingRetry = retry
|
||||
this.pendingEpoch = epoch
|
||||
this.pendingEpoch = this.epoch
|
||||
scheduledRecoveries.add(this)
|
||||
return true
|
||||
}
|
||||
@@ -152,6 +185,7 @@ export class RemoteRuntimePtyRecoveryState {
|
||||
if (this.phase === 'disposed') {
|
||||
return
|
||||
}
|
||||
this.deadlineExpired = false
|
||||
this.clearTimers()
|
||||
this.phase = 'idle'
|
||||
this.attempt = 0
|
||||
@@ -175,6 +209,7 @@ export class RemoteRuntimePtyRecoveryState {
|
||||
if (this.phase === 'disposed') {
|
||||
return
|
||||
}
|
||||
this.deadlineExpired = false
|
||||
this.epoch += 1
|
||||
this.clearTimers()
|
||||
this.phase = 'idle'
|
||||
@@ -191,11 +226,13 @@ export class RemoteRuntimePtyRecoveryState {
|
||||
|
||||
private armDeadline(epoch: number): void {
|
||||
this.clearDeadlineTimer()
|
||||
this.deadlineExpired = false
|
||||
const timer = setTimeout(() => {
|
||||
if (this.deadlineTimer !== timer || !this.isCurrent(epoch)) {
|
||||
return
|
||||
}
|
||||
this.deadlineTimer = null
|
||||
this.deadlineExpired = true
|
||||
// Why: the cutoff stops self-initiated retries but must keep the pane revivable by online/resume/reconnect.
|
||||
this.stopRetryTimer()
|
||||
this.phase = 'disconnected'
|
||||
|
||||
+6
-5
@@ -4,6 +4,7 @@ import {
|
||||
createRemoteRuntimeTransportMocks,
|
||||
type MultiplexSubscriptionCallbacks
|
||||
} from './remote-runtime-pty-transport-test-harness'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
|
||||
let resolvedPaneHandle = 'terminal-1'
|
||||
@@ -81,7 +82,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
let createCalls = 0
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method === 'status.get') {
|
||||
vi.setSystemTime(startedAt + 59_000)
|
||||
vi.setSystemTime(startedAt + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS - 1_000)
|
||||
return {
|
||||
ok: true,
|
||||
result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] }
|
||||
@@ -163,7 +164,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('stops unknown terminal-create recovery after one minute and remains manually retryable', async () => {
|
||||
it('stops unknown terminal-create recovery at the cutoff and remains manually retryable', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let reachable = false
|
||||
@@ -205,7 +206,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
onRecoveryStateChange: (state) => recoveryStates.push(state.phase)
|
||||
}
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
await connect
|
||||
const callsAtCutoff = runtimeCall.mock.calls.length
|
||||
|
||||
@@ -218,7 +219,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
|
||||
statusTimesOut = true
|
||||
expect(transport.retryRecovery?.()).toBe(true)
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
const callsAtManualCutoff = runtimeCall.mock.calls.length
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000)
|
||||
@@ -278,7 +279,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
})
|
||||
|
||||
const connect = transport.connect({ url: '', callbacks: {} })
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
await connect
|
||||
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
+3
-2
@@ -4,6 +4,7 @@ import {
|
||||
readyHostSessionInventoryResponse,
|
||||
type MultiplexSubscriptionCallbacks
|
||||
} from './remote-runtime-pty-transport-test-harness'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
|
||||
let resolvedPaneHandle = 'terminal-1'
|
||||
@@ -72,7 +73,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
expect(hostListCalls).toBe(callsAfterTwoWindows)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('recovering')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(9_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
expect(subscribedTerminalHandles()).toEqual(['terminal-stable'])
|
||||
transport.destroy?.()
|
||||
@@ -180,7 +181,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
'terminal-flapping'
|
||||
])
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
await vi.advanceTimersByTimeAsync(45_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
|
||||
+2
-1
@@ -9,6 +9,7 @@ import {
|
||||
readyHostSessionInventoryResponse,
|
||||
type MultiplexSubscriptionCallbacks
|
||||
} from './remote-runtime-pty-transport-test-harness'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
|
||||
let resolvedPaneHandle = 'terminal-1'
|
||||
@@ -160,7 +161,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
expect(onPtyExit).not.toHaveBeenCalled()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(44_001)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
expect(subscribedTerminalHandles()).toHaveLength(3)
|
||||
} finally {
|
||||
|
||||
+2
-1
@@ -9,6 +9,7 @@ import {
|
||||
createRemoteRuntimeTransportMocks,
|
||||
type MultiplexSubscriptionCallbacks
|
||||
} from './remote-runtime-pty-transport-test-harness'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
|
||||
let resolvedPaneHandle = 'terminal-1'
|
||||
@@ -534,7 +535,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
|
||||
partitioned = true
|
||||
callbacksByConnection[0].onClose?.()
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
|
||||
const disconnectedState = transport.getRecoveryState?.()
|
||||
const callsAtCutoff = runtimeSubscribe.mock.calls.length
|
||||
|
||||
+10
-5
@@ -3,6 +3,10 @@ import {
|
||||
createRemoteRuntimeTransportMocks,
|
||||
type MultiplexSubscriptionCallbacks
|
||||
} from './remote-runtime-pty-transport-test-harness'
|
||||
import {
|
||||
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
|
||||
REMOTE_RUNTIME_RECOVERY_DELAYS_MS
|
||||
} from './remote-runtime-pty-recovery-state'
|
||||
|
||||
let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null
|
||||
let resolvedPaneHandle = 'terminal-1'
|
||||
@@ -140,10 +144,11 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
existingPtyId: 'remote:env-1@@stale-client-handle',
|
||||
callbacks: {}
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
|
||||
const attemptsAtCutoff = runtimeSubscribe.mock.calls.length
|
||||
expect(attemptsAtCutoff).toBe(8)
|
||||
// Why: every backoff step must be reachable inside the window it arms (#11305).
|
||||
expect(attemptsAtCutoff).toBeGreaterThanOrEqual(REMOTE_RUNTIME_RECOVERY_DELAYS_MS.length)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000)
|
||||
@@ -235,7 +240,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
expect(activateAttempts).toBe(2)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
rejectInFlight(
|
||||
@@ -292,7 +297,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
expect(runtimeSubscribe).toHaveBeenCalledTimes(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
rejectSubscription(
|
||||
@@ -349,7 +354,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
expect.objectContaining({ method: 'terminal.resolvePane' })
|
||||
)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
resolveMetadata({
|
||||
|
||||
@@ -90,6 +90,10 @@ const HOST_SESSION_POLL_MAX_MS = 1_000
|
||||
const HOST_SESSION_ATTACH_TIMEOUT_MS = 15_000
|
||||
const HOST_SESSION_INVENTORY_MAX_WINDOWS_PER_RECOVERY = 2
|
||||
const HOST_SESSION_SAME_HANDLE_END_REUSE_LIMIT = 2
|
||||
// Why its own constant: this fences how long an end-then-reattach on the same handle still counts as
|
||||
// one recovery, which is unrelated to how long auto-recovery keeps retrying. It read the recovery
|
||||
// budget before that budget became a derived value, and must not drift with it.
|
||||
const HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS = 60_000
|
||||
const MAX_SURFACED_TERMINAL_ERRORS = 8
|
||||
const TERMINAL_CREATE_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
|
||||
|
||||
@@ -247,7 +251,9 @@ export function createRemoteRuntimePtyTransport(
|
||||
clearPublishedHandleWait()
|
||||
}
|
||||
if (recovery.currentPhase === 'disconnected') {
|
||||
autoRecoveryWindowSpent = true
|
||||
// Why: only the wall-clock deadline is evidence the window was spent; a UI latch from a fatal
|
||||
// resubscribe must not license reattaching a fenced same handle (#12683).
|
||||
autoRecoveryWindowSpent ||= recovery.autoRecoveryDeadlineExpired
|
||||
// Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic.
|
||||
subscriptionGeneration += 1
|
||||
closeMultiplexedStream()
|
||||
@@ -326,7 +332,7 @@ export function createRemoteRuntimePtyTransport(
|
||||
if (
|
||||
sameHandleEndReuseHandle !== targetHandle ||
|
||||
sameHandleEndReuseAttachedAt === null ||
|
||||
Date.now() - sameHandleEndReuseAttachedAt >= REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
|
||||
Date.now() - sameHandleEndReuseAttachedAt >= HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS
|
||||
) {
|
||||
resetSameHandleEndReuse()
|
||||
return 'prefer-replacement'
|
||||
@@ -870,6 +876,43 @@ export function createRemoteRuntimePtyTransport(
|
||||
return false
|
||||
}
|
||||
|
||||
// Why: a recoverable connect failure is unverifiable contact loss, not a dead terminal, so retry
|
||||
// whichever path can still reach the pane instead of latching with nothing armed (#12684).
|
||||
function retryAfterRecoverableConnectFailure(nextEpoch: number): void {
|
||||
if (destroyed || terminalEnded) {
|
||||
return
|
||||
}
|
||||
if (connected && handle) {
|
||||
scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(handle), nextEpoch)
|
||||
return
|
||||
}
|
||||
replayLastTransportEntryPoint()
|
||||
}
|
||||
|
||||
// Why: schedule() both auto-retries inside the window and leaves the retry parked when the deadline
|
||||
// latches, so online/resume and the Reconnect button always find something to fire.
|
||||
function scheduleConnectRetryAfterRecoverableFailure(): void {
|
||||
if (destroyed) {
|
||||
return
|
||||
}
|
||||
// Why: an ambiguous create already owns a reconciliation-gated retry that only Reconnect may
|
||||
// re-enter; auto-replaying here would just re-probe a runtime that cannot reconcile.
|
||||
if (terminalCreateNeedsReconciliation || agentSessionRequiresHostAuthorityReplay) {
|
||||
recovery.markDisconnected()
|
||||
return
|
||||
}
|
||||
// Why: the last attempt's RPC budget expires at the same instant as the deadline, so a silent drop
|
||||
// rejects after the latch. Beginning a new epoch there re-arms the whole window, so park instead.
|
||||
if (recovery.currentPhase === 'disconnected') {
|
||||
recovery.parkRetryAfterDeadline(retryAfterRecoverableConnectFailure)
|
||||
return
|
||||
}
|
||||
const recoveryEpoch = recovery.isActive ? recovery.currentEpoch : recovery.begin()
|
||||
if (!recovery.schedule(recoveryEpoch, retryAfterRecoverableConnectFailure)) {
|
||||
recovery.markDisconnected()
|
||||
}
|
||||
}
|
||||
|
||||
async function attachHostSessionMirror(
|
||||
options: { cols?: number; rows?: number },
|
||||
notifySpawn = true,
|
||||
@@ -1033,6 +1076,8 @@ export function createRemoteRuntimePtyTransport(
|
||||
kind === 'agent-session'
|
||||
? agentSessionRequiresHostAuthorityReplay
|
||||
: terminalCreateNeedsReconciliation
|
||||
// Why the same budget: this loop calls recovery.begin(), so a shorter local deadline would abandon
|
||||
// the create while the recovery state still reports 'recovering' with nothing in flight.
|
||||
let recoveryDeadlineAt: number | null = recovery.isActive
|
||||
? Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
|
||||
: null
|
||||
@@ -2314,7 +2359,7 @@ export function createRemoteRuntimePtyTransport(
|
||||
} else if (
|
||||
isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))
|
||||
) {
|
||||
recovery.markDisconnected()
|
||||
scheduleConnectRetryAfterRecoverableFailure()
|
||||
} else {
|
||||
recovery.cancel()
|
||||
emitRecoveryState()
|
||||
@@ -2616,6 +2661,12 @@ export function createRemoteRuntimePtyTransport(
|
||||
void transport.connect(lastConnectOptions)
|
||||
return true
|
||||
}
|
||||
// Why: online/resume fires a parked retry; the button must not be weaker than an event (#12684).
|
||||
if (!destroyed && !terminalEnded && recovery.currentPhase === 'disconnected') {
|
||||
if (recovery.retryNow()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if (
|
||||
destroyed ||
|
||||
terminalEnded ||
|
||||
|
||||
@@ -2881,7 +2881,7 @@
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "Reconnecting to remote runtime",
|
||||
"disconnectedTitle": "Remote runtime disconnected",
|
||||
"retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
|
||||
"retryingBody": "Orca is retrying automatically. This terminal will resume if the connection returns.",
|
||||
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
|
||||
"reconnectButton": "Reconnect"
|
||||
}
|
||||
|
||||
@@ -2881,7 +2881,7 @@
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "リモートランタイムに再接続中",
|
||||
"disconnectedTitle": "リモートランタイムが切断されました",
|
||||
"retryingBody": "Orca は最大1分間再試行します。接続が復元されると、このターミナルが再開されます。",
|
||||
"retryingBody": "Orca は自動的に再試行します。接続が復元されると、このターミナルが再開されます。",
|
||||
"disconnectedBody": "自動再試行が停止しました。このターミナルセッションを再開するには再接続してください。",
|
||||
"reconnectButton": "再接続"
|
||||
}
|
||||
|
||||
@@ -2886,7 +2886,7 @@
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "원격 런타임에 다시 연결 중",
|
||||
"disconnectedTitle": "원격 런타임 연결 끊김",
|
||||
"retryingBody": "Orca는 최대 1분간 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.",
|
||||
"retryingBody": "Orca가 자동으로 재시도합니다. 연결이 복원되면 이 터미널이 재개됩니다.",
|
||||
"disconnectedBody": "자동 재시도가 중지되었습니다. 이 터미널 세션을 재개하려면 다시 연결하세요.",
|
||||
"reconnectButton": "다시 연결"
|
||||
}
|
||||
|
||||
@@ -2896,7 +2896,7 @@
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "正在重新连接到远程运行时",
|
||||
"disconnectedTitle": "远程运行时已断开连接",
|
||||
"retryingBody": "Orca 将重试最多一分钟。如果连接恢复,此终端将恢复。",
|
||||
"retryingBody": "Orca 正在自动重试。如果连接恢复,此终端将恢复。",
|
||||
"disconnectedBody": "自动重试已停止。重新连接以恢复此终端会话。",
|
||||
"reconnectButton": "重新连接"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user