Files
orca/src/shared/remote-runtime-shared-control-state.ts
T
Jinwoo HongandJinwoo-H 211b2d1a35 fix(runtime): require consecutive missed probes before reaping a paired socket (#12790)
The paired-runtime WS heartbeat terminated a client after a single unanswered
15s ping. One missed pong is UNKNOWN, not proof the peer is gone: a cellular or
Tailscale blackhole, or a stalled TCP retransmit, routinely swallows one pong
from a peer that is still there. Users on flaky paths saw constant drops, each
costing a full redial plus E2EE re-handshake and subscription replay.

Reap now needs MISSED_PROBE_LIMIT (3) consecutive unanswered probes, counted per
socket rather than timed. Any proof of life -- pong or any inbound frame -- clears
the count, as does a resume from a server-loop pause, since a gap the client was
never given a chance to answer must not top up its budget. Missed sweeps still
re-probe, so a recovered path proves itself on the next tick.

Three matches the liveness budgets already in the product: the web client gives
45s (25s idle + 20s probe grace) and the relay control gives 75s. The paired
transport's single miss was the outlier.

Also gives the web client's redial the one-sided jitter the shared-control path
already had, so a fleet dropped by one shared blip does not re-dial in lockstep;
the helper is extracted to src/shared/reconnect-jitter.ts and shared by both.

STA-3320, #12327

Co-authored-by: Jinwoo-H <Jinwoo-H@users.noreply.github.com>
2026-08-05 18:59:20 -07:00

212 lines
6.8 KiB
TypeScript

import type { RemoteRuntimeClientError } from './remote-runtime-client-error'
import { releaseRemoteRuntimePreparedRequest } from './remote-runtime-prepared-request-admission'
import { remoteRuntimeUnavailableError } from './remote-runtime-request-frames'
import type { RuntimeRpcResponse } from './runtime-rpc-envelope'
import type {
RemoteRuntimeSharedConnectionDiagnostics,
SharedControlConnectionState,
SharedControlLogicalSubscription,
SharedControlPendingRequest,
SharedControlReadyWaiter
} from './remote-runtime-shared-control-types'
import { getSubscriptionId, isEndResult } from './remote-runtime-shared-control-protocol'
import { withReconnectJitter } from './reconnect-jitter'
import { tagRuntimeSubscriptionReplayResponse } from './runtime-subscription-replay'
export function buildSharedControlDiagnostics(args: {
state: SharedControlConnectionState
reconnecting: boolean
pendingRequestCount: number
subscriptionCount: number
reconnectAttempt: number
lastConnectedAt: number | null
lastClose: { code: number; reason: string } | null
lastError: string | null
}): RemoteRuntimeSharedConnectionDiagnostics {
return {
state: args.reconnecting ? 'reconnecting' : args.state,
pendingRequestCount: args.pendingRequestCount,
subscriptionCount: args.subscriptionCount,
reconnectAttempt: args.reconnectAttempt,
lastConnectedAt: args.lastConnectedAt,
lastClose: args.lastClose,
lastError: args.lastError
}
}
export function rejectSharedControlPendingRequest(
pendingRequests: Map<string, SharedControlPendingRequest<unknown>>,
requestId: string,
error: Error
): void {
const pending = pendingRequests.get(requestId)
if (!pending) {
return
}
pendingRequests.delete(requestId)
clearTimeout(pending.timeout)
releaseRemoteRuntimePreparedRequest(pending)
pending.reject(error)
}
export function resolveSharedControlPendingResponse(
pendingRequests: Map<string, SharedControlPendingRequest<unknown>>,
requestId: string,
response: RuntimeRpcResponse<unknown>
): void {
const pending = pendingRequests.get(requestId)
if (!pending) {
return
}
pendingRequests.delete(requestId)
clearTimeout(pending.timeout)
releaseRemoteRuntimePreparedRequest(pending)
pending.resolve(response)
}
export function refreshSharedControlPendingRequestTimeouts(
pendingRequests: Map<string, SharedControlPendingRequest<unknown>>
): void {
for (const pending of pendingRequests.values()) {
// Why: only long-poll requests opted into keepalive refresh; refreshing an
// ordinary short RPC would keep a genuinely-stuck server call alive forever.
if (!pending.refreshTimeoutOnKeepalive) {
continue
}
const timeout = pending.timeout as ReturnType<typeof setTimeout> & { refresh?: () => void }
timeout.refresh?.()
}
}
export function rejectAllSharedControlPendingRequests(
pendingRequests: Map<string, SharedControlPendingRequest<unknown>>,
error?: Error
): void {
const closeError = error ?? remoteRuntimeUnavailableError()
for (const [requestId, pending] of pendingRequests) {
clearTimeout(pending.timeout)
pendingRequests.delete(requestId)
releaseRemoteRuntimePreparedRequest(pending)
pending.reject(closeError)
}
}
export function markSharedControlSubscriptionsUnsent(
subscriptions: Map<string, SharedControlLogicalSubscription<unknown>>
): void {
for (const subscription of subscriptions.values()) {
subscription.sent = false
}
}
export function finishSharedControlSubscription(
subscriptions: Map<string, SharedControlLogicalSubscription<unknown>>,
subscription: SharedControlLogicalSubscription<unknown>,
notifyClose: boolean,
error?: RemoteRuntimeClientError
): void {
if (subscription.closed) {
return
}
subscription.closed = true
subscriptions.delete(subscription.requestId)
if (error) {
subscription.callbacks.onError(error)
}
if (notifyClose) {
subscription.callbacks.onClose?.()
}
}
export function resolveSharedControlReadyWaiters(waiters: SharedControlReadyWaiter[]): void {
for (const waiter of waiters.splice(0)) {
waiter.resolve()
}
}
export function rejectSharedControlReadyWaiters(
waiters: SharedControlReadyWaiter[],
error: Error
): void {
for (const waiter of waiters.splice(0)) {
waiter.reject(error)
}
}
export function handleSharedControlSubscriptionResponse(
subscriptions: Map<string, SharedControlLogicalSubscription<unknown>>,
subscription: SharedControlLogicalSubscription<unknown>,
response: RuntimeRpcResponse<unknown>
): void {
// The replay window ends on either success or error. An error created no
// remote subscription, so a later close must finish locally instead of
// waiting forever for an id that will never arrive.
subscription.awaitingResubscribe = false
if (!response.ok) {
subscription.sent = false
}
if (response.ok) {
const subscriptionId = getSubscriptionId(response.result)
if (subscriptionId) {
subscription.remoteSubscriptionId = subscriptionId
}
}
let delivered = response
if (subscription.pendingReplayTag) {
subscription.pendingReplayTag = false
if (response.ok) {
delivered = tagRuntimeSubscriptionReplayResponse(response)
}
}
subscription.callbacks.onResponse(delivered)
if (response.ok && isEndResult(response.result)) {
finishSharedControlSubscription(subscriptions, subscription, false)
}
}
export function closeSharedControlSocketState(args: {
readyWaiters: SharedControlReadyWaiter[]
pendingRequests: Map<string, SharedControlPendingRequest<unknown>>
subscriptions: Map<string, SharedControlLogicalSubscription<unknown>>
socketCleanup: (() => void) | null
ws: { close: () => void } | null
error?: Error
preserveReadyWaitersAndPendingRequests?: boolean
}): void {
if (!args.preserveReadyWaitersAndPendingRequests) {
rejectSharedControlReadyWaiters(
args.readyWaiters,
args.error ?? remoteRuntimeUnavailableError()
)
rejectAllSharedControlPendingRequests(args.pendingRequests, args.error)
}
markSharedControlSubscriptionsUnsent(args.subscriptions)
try {
args.socketCleanup?.()
args.ws?.close()
} catch {
// Best-effort cleanup of remote runtime control socket.
}
}
export function scheduleSharedControlReconnect(args: {
current: ReturnType<typeof setTimeout> | null
intentionallyClosed: boolean
reconnectAttempt: number
delaysMs: readonly number[]
open: () => void
}): { timer: ReturnType<typeof setTimeout> | null; reconnectAttempt: number } {
if (args.current || args.intentionallyClosed) {
return { timer: args.current, reconnectAttempt: args.reconnectAttempt }
}
const delay = withReconnectJitter(
args.delaysMs[Math.min(args.reconnectAttempt, args.delaysMs.length - 1)]
)
const timer = setTimeout(args.open, delay)
if (typeof timer.unref === 'function') {
timer.unref()
}
return { timer, reconnectAttempt: args.reconnectAttempt + 1 }
}