Files
orca/src/shared/remote-runtime-shared-control-open.ts
T
NeilandMeapri adce59a584 fix(remote-terminal): a remote pane comes back on its own once its host answers again (#29444)
* fix(remote-runtime): probe sockets and reset backoff on wake, and keep late resubscribes revivable

Rebuilt from #23359 on current main. On OS resume or network return every
remote-runtime socket pings once and is replaced if it stays silent for 8 s,
the shared-control retry ladder restarts from its short delays, and a
resubscribe that fails or opens after the pane's recovery window latched is
parked so resume/online can revive it.

Co-authored-by: Meapri <104259736+Meapri@users.noreply.github.com>

* fix(remote-terminal): revive a latched remote pane when its host answers again

A pane whose ~3 min auto-recovery window ran out only came back on an OS
online/resume event or a Reconnect click. A Tailscale or VPN route that
returns fires neither, so the pane stayed dead over a working link even
though the host shell was alive. The environment's shared control socket
reaching ready again is now the revive signal: the pane advances its backoff
or replays its attach, exactly as Reconnect does.

Adds e2e repros for paired runtimes and managed SSH hosts: 30 s silent and
reset outages, and a silent outage longer than the recovery window.

* fix(remote-terminal): hold the control-ready reviver only while a pane is recovering

* fix(remote-runtime): re-probe a wake check whose deadline fired late, and trim the SSH outage spec

---------

Co-authored-by: Meapri <104259736+Meapri@users.noreply.github.com>
2026-10-15 23:23:54 -07:00

115 lines
3.4 KiB
TypeScript

import type WebSocket from 'ws'
import type { PairedRuntimeConnection } from './paired-runtime-connection'
import type { RemoteRuntimeClientError } from './remote-runtime-client-error'
import { remoteRuntimeUnavailableError } from './remote-runtime-request-frames'
import {
openRemoteRuntimeWebSocket,
type RemoteRuntimeWebSocket
} from './remote-runtime-request-websocket'
import { formatSharedControlCloseMessage } from './remote-runtime-shared-control-protocol'
import {
startRemoteRuntimeSocketLiveness,
type RemoteRuntimeSocketLivenessOptions
} from './remote-runtime-socket-liveness'
export function openSharedControlSocket(
connection: PairedRuntimeConnection,
callbacks: {
getCurrentSocket: () => WebSocket | null
onClose: (close: { code: number; reason: string }, error: RemoteRuntimeClientError) => void
onError: (error: RemoteRuntimeClientError) => void
onTextFrame: (frame: string) => void
/** Any inbound frame, ping or pong on the current socket. */
onActivity?: () => void
// Why: reconnect is edge-triggered on `close`, but a half-open tunnel
// never delivers one. Liveness pings and declares the socket dead when
// the server goes silent, so the close/reconnect path can run (#7718).
liveness?: {
options?: RemoteRuntimeSocketLivenessOptions
onDead: (error: RemoteRuntimeClientError) => void
}
}
): { ok: true; socket: RemoteRuntimeWebSocket } | { ok: false; error: RemoteRuntimeClientError } {
let noteLiveness: () => void = () => {}
const noteActivity = (): void => {
noteLiveness()
callbacks.onActivity?.()
}
const opened = openRemoteRuntimeWebSocket(connection, {
onClose: (ws, code, reason) => {
if (callbacks.getCurrentSocket() === ws) {
callbacks.onClose(
{ code, reason: reason.toString() },
remoteRuntimeUnavailableError(formatSharedControlCloseMessage(code, reason))
)
}
},
onError: (ws, error) => {
if (callbacks.getCurrentSocket() === ws) {
callbacks.onError(error)
}
},
onTextFrame: (ws, frame) => {
if (callbacks.getCurrentSocket() === ws) {
noteActivity()
callbacks.onTextFrame(frame)
}
},
onPong: (ws) => {
if (callbacks.getCurrentSocket() === ws) {
noteActivity()
}
},
onPing: (ws) => {
if (callbacks.getCurrentSocket() === ws) {
noteActivity()
}
}
})
if (!opened.ok || !callbacks.liveness) {
return opened
}
const { ws, handshake, cleanup } = opened.socket
const liveness = callbacks.liveness
const monitor = startRemoteRuntimeSocketLiveness({
ping: () => {
if (ws.readyState !== 1) {
return false
}
ws.ping()
return true
},
onDead: () => {
if (callbacks.getCurrentSocket() !== ws) {
return
}
try {
// Why: close() on a half-open socket can hang for the OS TCP timeout.
ws.terminate()
} catch {
// Best-effort terminate; the dead callback resets connection state.
}
liveness.onDead(
remoteRuntimeUnavailableError(
'Remote Orca runtime stopped responding; resetting the control connection.'
)
)
},
options: liveness.options
})
noteLiveness = monitor.noteActivity
return {
ok: true,
socket: {
ws,
handshake,
cleanup: () => {
monitor.stop()
cleanup()
}
}
}
}