mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
fix(remote-terminal): a remote pane comes back on its own once its host answers again (#29444)
* fix(remote-runtime): probe sockets and reset backoff on wake, and keep late resubscribes revivable Rebuilt from #23359 on current main. On OS resume or network return every remote-runtime socket pings once and is replaced if it stays silent for 8 s, the shared-control retry ladder restarts from its short delays, and a resubscribe that fails or opens after the pane's recovery window latched is parked so resume/online can revive it. Co-authored-by: Meapri <104259736+Meapri@users.noreply.github.com> * fix(remote-terminal): revive a latched remote pane when its host answers again A pane whose ~3 min auto-recovery window ran out only came back on an OS online/resume event or a Reconnect click. A Tailscale or VPN route that returns fires neither, so the pane stayed dead over a working link even though the host shell was alive. The environment's shared control socket reaching ready again is now the revive signal: the pane advances its backoff or replays its attach, exactly as Reconnect does. Adds e2e repros for paired runtimes and managed SSH hosts: 30 s silent and reset outages, and a silent outage longer than the recovery window. * fix(remote-terminal): hold the control-ready reviver only while a pane is recovering * fix(remote-runtime): re-probe a wake check whose deadline fired late, and trim the SSH outage spec --------- Co-authored-by: Meapri <104259736+Meapri@users.noreply.github.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import type {
|
||||
} from '../../shared/runtime-rpc-envelope'
|
||||
import { RemoteRuntimeRequestConnection } from '../../shared/remote-runtime-request-connection'
|
||||
import { RemoteRuntimeSharedControlConnection } from '../../shared/remote-runtime-shared-control-connection'
|
||||
import { probeAllRemoteRuntimeSocketsNow } from '../../shared/remote-runtime-socket-liveness'
|
||||
import type {
|
||||
RemoteRuntimeSharedConnectionDiagnostics,
|
||||
RemoteRuntimeSharedSubscription
|
||||
@@ -204,6 +205,9 @@ export function retryRemoteRuntimeSharedControlConnectionsNow(): void {
|
||||
for (const { connection } of sharedControlConnections.values()) {
|
||||
connection.retryNow()
|
||||
}
|
||||
// Why: a socket that still reads OPEN after wake arms no reconnect timer, so retryNow finds
|
||||
// nothing; probing every socket (shared control and per-stream) settles a dead path quickly (#9092).
|
||||
probeAllRemoteRuntimeSocketsNow()
|
||||
}
|
||||
|
||||
function getSharedControlConnection(
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type * as SocketLiveness from '../../shared/remote-runtime-socket-liveness'
|
||||
|
||||
const probeAllRemoteRuntimeSocketsNow = vi.hoisted(() => vi.fn(() => 0))
|
||||
|
||||
vi.mock('../../shared/remote-runtime-socket-liveness', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof SocketLiveness>()),
|
||||
probeAllRemoteRuntimeSocketsNow
|
||||
}))
|
||||
|
||||
import { retryRemoteRuntimeSharedControlConnectionsNow } from './runtime-environment-request-connections'
|
||||
|
||||
// Regression for #9092: the resume/online IPC must probe sockets that still read OPEN after wake.
|
||||
describe('retryRemoteRuntimeSharedControlConnectionsNow', () => {
|
||||
it('probes every remote runtime socket even when no reconnect is pending', () => {
|
||||
retryRemoteRuntimeSharedControlConnectionsNow()
|
||||
|
||||
expect(probeAllRemoteRuntimeSocketsNow).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
+39
@@ -434,4 +434,43 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
// #9092: a rebound subscribe still opening its stream when a second deadline latches must stay
|
||||
// revivable by system resume / network online instead of needing a manual Reconnect.
|
||||
it('parks a rebound subscribe whose stream open fails after its deadline', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { transport } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
// The stream open for the rebound handle hangs through the whole new recovery window.
|
||||
const streamOpen: { reject: ((error: Error) => void) | null } = { reject: null }
|
||||
runtimeSubscribe.mockImplementation(
|
||||
() =>
|
||||
new Promise((_resolve, reject) => {
|
||||
streamOpen.reject = reject
|
||||
})
|
||||
)
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot('terminal-rotated', 3, 'epoch-2'),
|
||||
'env-1'
|
||||
)
|
||||
await vi.waitFor(() => expect(streamOpen.reject).not.toBeNull())
|
||||
expect(transport.getRecoveryState?.().phase).toBe('recovering')
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
streamOpen.reject?.(new Error('Could not connect to the remote Orca runtime.'))
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -38,6 +38,44 @@ export function retryAllRemoteRuntimePtyRecoveriesNow(): number {
|
||||
return advanced
|
||||
}
|
||||
|
||||
// Why: a route that returns without an OS `online` event (Tailscale, VPN, a re-keyed link) fires no
|
||||
// trigger, so a pane latched past its window stayed dead over a working link. The environment's
|
||||
// control socket answering again is positive reachability evidence, as good as a Reconnect click.
|
||||
const controlReadyRevivers = new Map<string, Set<() => boolean>>()
|
||||
|
||||
export function registerRemoteRuntimePtyControlReadyReviver(
|
||||
environmentId: string,
|
||||
revive: () => boolean
|
||||
): () => void {
|
||||
let revivers = controlReadyRevivers.get(environmentId)
|
||||
if (!revivers) {
|
||||
revivers = new Set()
|
||||
controlReadyRevivers.set(environmentId, revivers)
|
||||
}
|
||||
revivers.add(revive)
|
||||
return () => {
|
||||
const current = controlReadyRevivers.get(environmentId)
|
||||
current?.delete(revive)
|
||||
if (current?.size === 0) {
|
||||
controlReadyRevivers.delete(environmentId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function getControlReadyReviverCountForTests(environmentId: string): number {
|
||||
return controlReadyRevivers.get(environmentId)?.size ?? 0
|
||||
}
|
||||
|
||||
export function reviveRemoteRuntimePtyRecoveriesForEnvironment(environmentId: string): number {
|
||||
let revived = 0
|
||||
for (const revive of Array.from(controlReadyRevivers.get(environmentId) ?? [])) {
|
||||
if (revive()) {
|
||||
revived += 1
|
||||
}
|
||||
}
|
||||
return revived
|
||||
}
|
||||
|
||||
export class RemoteRuntimePtyRecoveryState {
|
||||
private phase: RemoteRuntimePtyRecoveryPhase = 'idle'
|
||||
private epoch = 0
|
||||
|
||||
+62
@@ -572,6 +572,68 @@ describe('createRemoteRuntimePtyTransport', () => {
|
||||
}
|
||||
})
|
||||
|
||||
// #25783/#9092: a route returning without `online` (Tailscale, VPN) left a latched pane dead.
|
||||
it('revives a pane latched past its window when its environment answers again', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let partitioned = false
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
if (partitioned) {
|
||||
throw Object.assign(new Error('Could not connect to the remote Orca runtime.'), {
|
||||
code: 'remote_runtime_unavailable'
|
||||
})
|
||||
}
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } }))
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const {
|
||||
getControlReadyReviverCountForTests,
|
||||
reviveRemoteRuntimePtyRecoveriesForEnvironment
|
||||
} = await import('./remote-runtime-pty-recovery-state')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
transport.attach({ existingPtyId: 'remote:env-1@@terminal-1', callbacks: {} })
|
||||
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'before partition')
|
||||
expect(getControlReadyReviverCountForTests('env-1')).toBe(0)
|
||||
|
||||
partitioned = true
|
||||
subscriptionCallbacks?.onClose?.()
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 60_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const callsAtCutoff = runtimeSubscribe.mock.calls.length
|
||||
|
||||
partitioned = false
|
||||
expect(reviveRemoteRuntimePtyRecoveriesForEnvironment('env-2')).toBe(0)
|
||||
expect(reviveRemoteRuntimePtyRecoveriesForEnvironment('env-1')).toBe(1)
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(callsAtCutoff + 1))
|
||||
await vi.waitFor(() => expect(subscribeFrameCount()).toBe(2))
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'after the route returned')
|
||||
await vi.advanceTimersByTimeAsync(50)
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
|
||||
// Why: a healthy pane holds no registration, so a detached-and-discarded transport is never retained.
|
||||
expect(getControlReadyReviverCountForTests('env-1')).toBe(0)
|
||||
|
||||
partitioned = true
|
||||
subscriptionCallbacks?.onClose?.()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(getControlReadyReviverCountForTests('env-1')).toBe(1)
|
||||
transport.detach?.()
|
||||
expect(getControlReadyReviverCountForTests('env-1')).toBe(0)
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('releases pending claimed input when reconnect subscription fails', async () => {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
|
||||
@@ -72,6 +72,7 @@ import {
|
||||
import {
|
||||
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
|
||||
RemoteRuntimePtyRecoveryState,
|
||||
registerRemoteRuntimePtyControlReadyReviver,
|
||||
type RemoteRuntimePtyRecoveryPhase
|
||||
} from './remote-runtime-pty-recovery-state'
|
||||
import { createBrowserUuid } from '@/lib/browser-uuid'
|
||||
@@ -324,7 +325,23 @@ export function createRemoteRuntimePtyTransport(
|
||||
})
|
||||
const disconnectedInputGrace = createRemoteRuntimeDisconnectedInputGrace()
|
||||
let previousRecoveryPhase: RemoteRuntimePtyRecoveryPhase = 'idle'
|
||||
let unregisterControlReadyReviver: (() => void) | null = null
|
||||
// Why registered only while recovering: detach and destroy go idle, so a discarded transport is never retained.
|
||||
function syncControlReadyReviver(): void {
|
||||
const wantsRevival = recovery.isActive || recovery.currentPhase === 'disconnected'
|
||||
if (wantsRevival && !unregisterControlReadyReviver) {
|
||||
// Why: a backoff advances and a latched pane replays its attach, exactly as online/resume/Reconnect would.
|
||||
unregisterControlReadyReviver = registerRemoteRuntimePtyControlReadyReviver(
|
||||
runtimeEnvironmentId,
|
||||
() => !destroyed && (recovery.retryNow() || (transport.retryRecovery?.() ?? false))
|
||||
)
|
||||
} else if (!wantsRevival && unregisterControlReadyReviver) {
|
||||
unregisterControlReadyReviver()
|
||||
unregisterControlReadyReviver = null
|
||||
}
|
||||
}
|
||||
const recovery = new RemoteRuntimePtyRecoveryState(() => {
|
||||
syncControlReadyReviver()
|
||||
// Why: any recovery transition after the ack (a stale send, a close, the deadline) re-holds typing.
|
||||
inputReadySubscription = null
|
||||
const leftLatchForRetry = previousRecoveryPhase === 'disconnected' && recovery.isActive
|
||||
@@ -2137,12 +2154,22 @@ export function createRemoteRuntimePtyTransport(
|
||||
return false
|
||||
}
|
||||
if (recovery.currentPhase === 'disconnected') {
|
||||
parkResubscribeAfterDeadline(targetHandle)
|
||||
return true
|
||||
}
|
||||
scheduleResubscribeAfterTransportClose()
|
||||
return true
|
||||
}
|
||||
|
||||
// Why: a recoverable failure that lands after the auto-recovery deadline latched has no live epoch
|
||||
// to join; without a parked retry, resume/online have nothing to fire and the pane stays
|
||||
// disconnected until the user clicks Reconnect (#9092). Mirrors the connect path's park.
|
||||
function parkResubscribeAfterDeadline(targetHandle: string): void {
|
||||
recovery.parkRetryAfterDeadline((nextEpoch) =>
|
||||
scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(targetHandle), nextEpoch)
|
||||
)
|
||||
}
|
||||
|
||||
// Why: after a transport drop the host may have re-minted this handle; re-derive from the snapshot so we don't mirror/type into whatever PTY now sits behind the stale one (#7718).
|
||||
async function resubscribeAfterTransportClose(
|
||||
previousHandle: string,
|
||||
@@ -2300,6 +2327,17 @@ export function createRemoteRuntimePtyTransport(
|
||||
let retryScheduled = false
|
||||
void resubscribeAfterTransportClose(resubscribeHandle, replacementPolicy, recoveryEpoch)
|
||||
.catch((error) => {
|
||||
if (
|
||||
!destroyed &&
|
||||
connected &&
|
||||
handle &&
|
||||
recovery.currentPhase === 'disconnected' &&
|
||||
recovery.ownsEpoch(recoveryEpoch) &&
|
||||
isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))
|
||||
) {
|
||||
parkResubscribeAfterDeadline(handle)
|
||||
return
|
||||
}
|
||||
if (!destroyed && connected && handle && recovery.isCurrent(recoveryEpoch)) {
|
||||
clearPendingViewportClaim()
|
||||
if (isRecoverablePaneBindingError(error)) {
|
||||
@@ -2605,6 +2643,17 @@ export function createRemoteRuntimePtyTransport(
|
||||
remotePtyId !== subscribedPtyId
|
||||
) {
|
||||
nextStream.close()
|
||||
// Why: a stream that opened after the deadline latched is discarded like its late failure would be,
|
||||
// so park it too or resume/online find nothing to revive (#9092).
|
||||
if (
|
||||
!destroyed &&
|
||||
connected &&
|
||||
handle === subscribedHandle &&
|
||||
recovery.currentPhase === 'disconnected' &&
|
||||
(expectedRecoveryEpoch === undefined || recovery.ownsEpoch(expectedRecoveryEpoch))
|
||||
) {
|
||||
parkResubscribeAfterDeadline(subscribedHandle)
|
||||
}
|
||||
return
|
||||
}
|
||||
closeMultiplexedStream()
|
||||
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* Regression for #9092: a remote pane whose resubscribe attempt was still in flight when the
|
||||
* auto-recovery deadline latched dropped the late (recoverable) failure, so nothing was parked
|
||||
* and system resume / network online had nothing to fire. The pane stayed disconnected until
|
||||
* the user clicked Reconnect, even after the host came back.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
TerminalStreamOpcode,
|
||||
decodeTerminalStreamFrame,
|
||||
decodeTerminalStreamJson,
|
||||
encodeTerminalStreamFrame,
|
||||
encodeTerminalStreamJson,
|
||||
encodeTerminalStreamText
|
||||
} from '../../../../shared/terminal-stream-protocol'
|
||||
import { REMOTE_RUNTIME_CONNECT_FAILURE_PHRASE } from '../../../../shared/remote-runtime-connect-bound'
|
||||
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
const FIRST_HANDLE = 'terminal-1'
|
||||
const FIRST_PTY_ID = 'remote:env-1@@terminal-1'
|
||||
|
||||
describe('remote runtime resubscribe failing after the recovery deadline', () => {
|
||||
const subscriptionSendBinary = vi.fn()
|
||||
let subscriptionCallbacks: {
|
||||
onResponse: (response: unknown) => void
|
||||
onBinary?: (bytes: Uint8Array<ArrayBufferLike>) => void
|
||||
onClose?: () => void
|
||||
} | null = null
|
||||
let resolvePaneCalls = 0
|
||||
let holdResolvePane: { reject: (error: Error) => void } | null = null
|
||||
let hangLateResolvePane = true
|
||||
let streamOpens = 0
|
||||
const lateStreamOpen: { hang: boolean; release: (() => void) | null } = {
|
||||
hang: false,
|
||||
release: null
|
||||
}
|
||||
|
||||
function latestStreamId(): number {
|
||||
const frame = subscriptionSendBinary.mock.calls
|
||||
.map((call) => decodeTerminalStreamFrame(call[0]))
|
||||
.findLast((candidate) => candidate?.opcode === TerminalStreamOpcode.Subscribe)
|
||||
const payload = frame ? decodeTerminalStreamJson<{ streamId: number }>(frame.payload) : null
|
||||
if (!payload) {
|
||||
throw new Error('missing terminal subscribe frame')
|
||||
}
|
||||
return payload.streamId
|
||||
}
|
||||
|
||||
function emitSnapshot(streamId: number): void {
|
||||
const frames = [
|
||||
{ opcode: TerminalStreamOpcode.SnapshotStart, payload: encodeTerminalStreamJson({}) },
|
||||
{ opcode: TerminalStreamOpcode.SnapshotChunk, payload: encodeTerminalStreamText('live') },
|
||||
{ opcode: TerminalStreamOpcode.SnapshotEnd, payload: new Uint8Array() }
|
||||
]
|
||||
frames.forEach((frame, index) =>
|
||||
subscriptionCallbacks?.onBinary?.(
|
||||
encodeTerminalStreamFrame({ ...frame, streamId, seq: index + 1 })
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules()
|
||||
vi.doMock('@/runtime/web-runtime-session', () => ({
|
||||
refreshWebRuntimeSessionTabsSnapshot: vi.fn(async () => {})
|
||||
}))
|
||||
subscriptionCallbacks = null
|
||||
subscriptionSendBinary.mockReset()
|
||||
resolvePaneCalls = 0
|
||||
holdResolvePane = null
|
||||
hangLateResolvePane = true
|
||||
streamOpens = 0
|
||||
lateStreamOpen.hang = false
|
||||
lateStreamOpen.release = null
|
||||
const runtimeCall = vi.fn(async (request: { method: string; params?: unknown }) => {
|
||||
if (request.method === 'terminal.resolvePane') {
|
||||
resolvePaneCalls += 1
|
||||
const params = request.params as { paneKey: string; worktreeId: string }
|
||||
if (hangLateResolvePane && holdResolvePane === null && resolvePaneCalls > 1) {
|
||||
// Why: the post-deadline attempt hangs so the test observes that it was issued.
|
||||
return new Promise<never>((_resolve, reject) => {
|
||||
holdResolvePane = { reject }
|
||||
})
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
terminal: {
|
||||
handle: FIRST_HANDLE,
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1',
|
||||
worktreeId: params.worktreeId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ok: true, result: { terminal: { handle: FIRST_HANDLE } } }
|
||||
})
|
||||
const runtimeSubscribe = vi.fn(
|
||||
async (_args: unknown, callbacks: typeof subscriptionCallbacks) => {
|
||||
streamOpens += 1
|
||||
if (lateStreamOpen.hang && streamOpens > 1) {
|
||||
await new Promise<void>((resolve) => {
|
||||
lateStreamOpen.release = resolve
|
||||
})
|
||||
}
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(() =>
|
||||
subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'ready' } })
|
||||
)
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
vi.stubGlobal('window', {
|
||||
api: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } }
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('parks the late failure so a resume trigger can revive the pane', async () => {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
transport.attach({ existingPtyId: FIRST_PTY_ID, cols: 80, rows: 24, callbacks: {} })
|
||||
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
|
||||
emitSnapshot(latestStreamId())
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
|
||||
vi.useFakeTimers()
|
||||
subscriptionCallbacks?.onClose?.()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
await vi.waitFor(() => expect(holdResolvePane).not.toBeNull())
|
||||
|
||||
// The host stays unreachable past the whole auto-recovery window.
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
holdResolvePane?.reject(new Error(`${REMOTE_RUNTIME_CONNECT_FAILURE_PHRASE}.`))
|
||||
holdResolvePane = null
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
const callsBeforeResume = resolvePaneCalls
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(resolvePaneCalls).toBeGreaterThan(callsBeforeResume)
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('parks a late success whose stream opened after the deadline', async () => {
|
||||
hangLateResolvePane = false
|
||||
lateStreamOpen.hang = true
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
transport.attach({ existingPtyId: FIRST_PTY_ID, cols: 80, rows: 24, callbacks: {} })
|
||||
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
|
||||
emitSnapshot(latestStreamId())
|
||||
|
||||
vi.useFakeTimers()
|
||||
subscriptionCallbacks?.onClose?.()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
await vi.waitFor(() => expect(lateStreamOpen.release).not.toBeNull())
|
||||
|
||||
// The stream open stays in flight past the whole window, then succeeds.
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
lateStreamOpen.release?.()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
|
||||
transport.destroy?.()
|
||||
})
|
||||
})
|
||||
@@ -2,14 +2,18 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { renderHook } from '@testing-library/react'
|
||||
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNowMock } = vi.hoisted(() => ({
|
||||
retryAllRemoteRuntimePtyRecoveriesNowMock: vi.fn()
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNowMock, reviveForEnvironmentMock } = vi.hoisted(() => ({
|
||||
retryAllRemoteRuntimePtyRecoveriesNowMock: vi.fn(),
|
||||
reviveForEnvironmentMock: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('@/components/terminal-pane/remote-runtime-pty-recovery-state', () => ({
|
||||
retryAllRemoteRuntimePtyRecoveriesNow: retryAllRemoteRuntimePtyRecoveriesNowMock
|
||||
retryAllRemoteRuntimePtyRecoveriesNow: retryAllRemoteRuntimePtyRecoveriesNowMock,
|
||||
reviveRemoteRuntimePtyRecoveriesForEnvironment: reviveForEnvironmentMock
|
||||
}))
|
||||
|
||||
type DiagnosticsEvent = { environmentId: string; diagnostics: { state: string } }
|
||||
|
||||
import { useRemoteRuntimeRecoveryTriggers } from './use-remote-runtime-recovery-triggers'
|
||||
|
||||
describe('useRemoteRuntimeRecoveryTriggers', () => {
|
||||
@@ -20,6 +24,14 @@ describe('useRemoteRuntimeRecoveryTriggers', () => {
|
||||
return unsubscribeSystemResumed
|
||||
})
|
||||
const retryConnectionsNow = vi.fn(() => Promise.resolve())
|
||||
let diagnosticsCallback: ((event: DiagnosticsEvent) => void) | null = null
|
||||
const unsubscribeDiagnostics = vi.fn()
|
||||
const onSharedControlDiagnostics = vi.fn((callback: (event: DiagnosticsEvent) => void) => {
|
||||
diagnosticsCallback = callback
|
||||
return unsubscribeDiagnostics
|
||||
})
|
||||
const emitState = (environmentId: string, state: string): void =>
|
||||
diagnosticsCallback?.({ environmentId, diagnostics: { state } })
|
||||
|
||||
beforeEach(() => {
|
||||
systemResumedCallback = null
|
||||
@@ -27,9 +39,12 @@ describe('useRemoteRuntimeRecoveryTriggers', () => {
|
||||
onSystemResumed.mockClear()
|
||||
retryConnectionsNow.mockClear()
|
||||
retryAllRemoteRuntimePtyRecoveriesNowMock.mockClear()
|
||||
reviveForEnvironmentMock.mockClear()
|
||||
diagnosticsCallback = null
|
||||
unsubscribeDiagnostics.mockClear()
|
||||
;(window as unknown as { api: unknown }).api = {
|
||||
ui: { onSystemResumed },
|
||||
runtimeEnvironments: { retryConnectionsNow }
|
||||
runtimeEnvironments: { retryConnectionsNow, onSharedControlDiagnostics }
|
||||
}
|
||||
})
|
||||
|
||||
@@ -60,4 +75,22 @@ describe('useRemoteRuntimeRecoveryTriggers', () => {
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNowMock).not.toHaveBeenCalled()
|
||||
expect(unsubscribeSystemResumed).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
// #9092/#25783: a returning Tailscale or VPN route fires no `online`, so the control socket is the signal.
|
||||
it("revives an environment's panes when its control connection becomes ready again", () => {
|
||||
const { unmount } = renderHook(() => useRemoteRuntimeRecoveryTriggers())
|
||||
|
||||
emitState('env-1', 'ready')
|
||||
emitState('env-1', 'ready')
|
||||
expect(reviveForEnvironmentMock.mock.calls).toEqual([['env-1']])
|
||||
|
||||
emitState('env-1', 'reconnecting')
|
||||
emitState('env-2', 'connecting')
|
||||
expect(reviveForEnvironmentMock).toHaveBeenCalledTimes(1)
|
||||
|
||||
emitState('env-1', 'ready')
|
||||
expect(reviveForEnvironmentMock.mock.calls).toEqual([['env-1'], ['env-1']])
|
||||
unmount()
|
||||
expect(unsubscribeDiagnostics).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { useEffect } from 'react'
|
||||
import { retryAllRemoteRuntimePtyRecoveriesNow } from '@/components/terminal-pane/remote-runtime-pty-recovery-state'
|
||||
import {
|
||||
retryAllRemoteRuntimePtyRecoveriesNow,
|
||||
reviveRemoteRuntimePtyRecoveriesForEnvironment
|
||||
} from '@/components/terminal-pane/remote-runtime-pty-recovery-state'
|
||||
|
||||
export function useRemoteRuntimeRecoveryTriggers(): void {
|
||||
useEffect(() => {
|
||||
@@ -13,9 +16,24 @@ export function useRemoteRuntimeRecoveryTriggers(): void {
|
||||
typeof window.api?.ui?.onSystemResumed === 'function'
|
||||
? window.api.ui.onSystemResumed(advanceRemoteRuntimeRecoveryBackoffs)
|
||||
: null
|
||||
const readyByEnvironment = new Map<string, boolean>()
|
||||
const unsubscribeControlDiagnostics =
|
||||
typeof window.api?.runtimeEnvironments?.onSharedControlDiagnostics === 'function'
|
||||
? window.api.runtimeEnvironments.onSharedControlDiagnostics(
|
||||
({ environmentId, diagnostics }) => {
|
||||
const ready = diagnostics.state === 'ready'
|
||||
const wasReady = readyByEnvironment.get(environmentId) ?? false
|
||||
readyByEnvironment.set(environmentId, ready)
|
||||
if (ready && !wasReady) {
|
||||
reviveRemoteRuntimePtyRecoveriesForEnvironment(environmentId)
|
||||
}
|
||||
}
|
||||
)
|
||||
: null
|
||||
return () => {
|
||||
window.removeEventListener('online', advanceRemoteRuntimeRecoveryBackoffs)
|
||||
unsubscribeSystemResumed?.()
|
||||
unsubscribeControlDiagnostics?.()
|
||||
}
|
||||
}, [])
|
||||
}
|
||||
|
||||
@@ -80,9 +80,11 @@ export function openRemoteRuntimePassthroughSocket(
|
||||
)
|
||||
const liveness = startRemoteRuntimeSocketLiveness({
|
||||
ping: () => {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.ping()
|
||||
if (ws.readyState !== WebSocket.OPEN) {
|
||||
return false
|
||||
}
|
||||
ws.ping()
|
||||
return true
|
||||
},
|
||||
onDead: () =>
|
||||
end(
|
||||
|
||||
@@ -74,9 +74,11 @@ export function openSharedControlSocket(
|
||||
const liveness = callbacks.liveness
|
||||
const monitor = startRemoteRuntimeSocketLiveness({
|
||||
ping: () => {
|
||||
if (ws.readyState === 1) {
|
||||
ws.ping()
|
||||
if (ws.readyState !== 1) {
|
||||
return false
|
||||
}
|
||||
ws.ping()
|
||||
return true
|
||||
},
|
||||
onDead: () => {
|
||||
if (callbacks.getCurrentSocket() !== ws) {
|
||||
|
||||
@@ -75,6 +75,10 @@ export class SharedControlReconnectScheduler {
|
||||
|
||||
// Why: OS resume / browser online should advance an already-scheduled reconnect, not start a new one.
|
||||
retryNow(): boolean {
|
||||
// Why: a wake or network change is fresh evidence, so the next failure starts again from the short
|
||||
// delays instead of the 5-minute idle cap earned during the outage — also when an open is already
|
||||
// in flight and no timer is pending (#9092).
|
||||
this.attempt = 0
|
||||
if (!this.timer || !this.pendingOpen) {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -18,13 +18,33 @@ export type RemoteRuntimeSocketLivenessOptions = {
|
||||
livenessTimeoutMs?: number
|
||||
}
|
||||
|
||||
// Why: after sleep or a network change a socket can read OPEN while its path is gone; the normal
|
||||
// cadence needs ~30-40 s to prove that. A resume probe settles it in one short, explicit window.
|
||||
export const REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS = 8_000
|
||||
|
||||
export type RemoteRuntimeSocketLivenessMonitor = {
|
||||
noteActivity: () => void
|
||||
/** Ping now; declare the socket dead unless anything arrives within `deadlineMs`. */
|
||||
probeNow: (deadlineMs?: number) => void
|
||||
stop: () => void
|
||||
}
|
||||
|
||||
const activeMonitors = new Set<RemoteRuntimeSocketLivenessMonitor>()
|
||||
|
||||
/** Probe every live remote-runtime socket in this process, e.g. on OS resume or network change. */
|
||||
export function probeAllRemoteRuntimeSocketsNow(
|
||||
deadlineMs = REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS
|
||||
): number {
|
||||
const monitors = Array.from(activeMonitors)
|
||||
for (const monitor of monitors) {
|
||||
monitor.probeNow(deadlineMs)
|
||||
}
|
||||
return monitors.length
|
||||
}
|
||||
|
||||
export function startRemoteRuntimeSocketLiveness(args: {
|
||||
ping: () => void
|
||||
/** Returns false when nothing was sent, e.g. the socket is still connecting. */
|
||||
ping: () => boolean
|
||||
onDead: () => void
|
||||
options?: RemoteRuntimeSocketLivenessOptions
|
||||
now?: () => number
|
||||
@@ -35,6 +55,8 @@ export function startRemoteRuntimeSocketLiveness(args: {
|
||||
args.options?.livenessTimeoutMs ?? REMOTE_RUNTIME_SOCKET_LIVENESS_TIMEOUT_MS
|
||||
let lastTickAt = now()
|
||||
let probeSentAt: number | null = null
|
||||
let activitySinceResumeProbe = false
|
||||
let resumeProbeTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let stopped = false
|
||||
|
||||
const timer = setInterval(() => {
|
||||
@@ -72,20 +94,64 @@ export function startRemoteRuntimeSocketLiveness(args: {
|
||||
}
|
||||
stopped = true
|
||||
clearInterval(timer)
|
||||
if (resumeProbeTimer !== null) {
|
||||
clearTimeout(resumeProbeTimer)
|
||||
resumeProbeTimer = null
|
||||
}
|
||||
activeMonitors.delete(monitor)
|
||||
}
|
||||
|
||||
function tryPing(): void {
|
||||
try {
|
||||
args.ping()
|
||||
} catch {
|
||||
// Why: ping() can throw while a socket is mid-teardown; the probe deadline still settles it.
|
||||
function probeNow(deadlineMs = REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS): void {
|
||||
if (stopped || resumeProbeTimer !== null) {
|
||||
return
|
||||
}
|
||||
activitySinceResumeProbe = false
|
||||
// Why: a socket still connecting has no path to probe yet; a deadline with no ping could kill it mid-handshake.
|
||||
if (!tryPing()) {
|
||||
return
|
||||
}
|
||||
const probedAt = now()
|
||||
resumeProbeTimer = setTimeout(() => {
|
||||
resumeProbeTimer = null
|
||||
if (stopped || activitySinceResumeProbe) {
|
||||
return
|
||||
}
|
||||
// Why: a late timer means this process slept or stalled, so replies may still be queued; probe afresh.
|
||||
const elapsedMs = now() - probedAt
|
||||
if (elapsedMs < 0 || elapsedMs > deadlineMs * 1.5) {
|
||||
probeNow(deadlineMs)
|
||||
return
|
||||
}
|
||||
stop()
|
||||
args.onDead()
|
||||
}, deadlineMs)
|
||||
// Why: mobile typechecks shared code with DOM timer types where unref is absent.
|
||||
if (
|
||||
typeof resumeProbeTimer === 'object' &&
|
||||
'unref' in resumeProbeTimer &&
|
||||
typeof resumeProbeTimer.unref === 'function'
|
||||
) {
|
||||
resumeProbeTimer.unref()
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
function tryPing(): boolean {
|
||||
try {
|
||||
return args.ping()
|
||||
} catch {
|
||||
// Why: ping() can throw while a socket is mid-teardown; the probe deadline still settles it.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
const monitor: RemoteRuntimeSocketLivenessMonitor = {
|
||||
noteActivity: () => {
|
||||
probeSentAt = null
|
||||
activitySinceResumeProbe = true
|
||||
},
|
||||
probeNow,
|
||||
stop
|
||||
}
|
||||
activeMonitors.add(monitor)
|
||||
return monitor
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
probeAllRemoteRuntimeSocketsNow,
|
||||
startRemoteRuntimeSocketLiveness
|
||||
} from './remote-runtime-socket-liveness'
|
||||
import { SharedControlReconnectScheduler } from './remote-runtime-shared-control-reconnect'
|
||||
|
||||
// Regression for #9092: after sleep a socket can still read OPEN while its path is gone, and the
|
||||
// normal cadence took ~30-40 s to notice; resume now probes every socket with a short deadline.
|
||||
describe('remote runtime resume probe', () => {
|
||||
const monitors: { stop: () => void }[] = []
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
afterEach(() => {
|
||||
for (const monitor of monitors.splice(0)) {
|
||||
monitor.stop()
|
||||
}
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
function start(onDead = vi.fn(), ping = vi.fn(() => true)) {
|
||||
const monitor = startRemoteRuntimeSocketLiveness({
|
||||
ping,
|
||||
onDead,
|
||||
options: { pingIntervalMs: 10_000, livenessTimeoutMs: 25_000 }
|
||||
})
|
||||
monitors.push(monitor)
|
||||
return { monitor, onDead, ping }
|
||||
}
|
||||
|
||||
it('declares a silent socket dead within the resume deadline', async () => {
|
||||
const { onDead, ping } = start()
|
||||
|
||||
expect(probeAllRemoteRuntimeSocketsNow(2_000)).toBe(1)
|
||||
expect(ping).toHaveBeenCalledTimes(1)
|
||||
await vi.advanceTimersByTimeAsync(1_999)
|
||||
expect(onDead).not.toHaveBeenCalled()
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(onDead).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('probes afresh instead of failing a socket when the process stalled through the deadline', async () => {
|
||||
let clock = 0
|
||||
const onDead = vi.fn()
|
||||
const ping = vi.fn(() => true)
|
||||
monitors.push(
|
||||
startRemoteRuntimeSocketLiveness({
|
||||
ping,
|
||||
onDead,
|
||||
now: () => clock,
|
||||
options: { pingIntervalMs: 60_000, livenessTimeoutMs: 120_000 }
|
||||
})
|
||||
)
|
||||
|
||||
probeAllRemoteRuntimeSocketsNow(2_000)
|
||||
clock += 20_000
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
expect(onDead).not.toHaveBeenCalled()
|
||||
expect(ping).toHaveBeenCalledTimes(2)
|
||||
|
||||
clock += 2_000
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
expect(onDead).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('keeps a socket that answers the probe', async () => {
|
||||
const { monitor, onDead } = start()
|
||||
|
||||
probeAllRemoteRuntimeSocketsNow(2_000)
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
monitor.noteActivity()
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
expect(onDead).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('skips stopped monitors and coalesces a repeated resume into one probe', async () => {
|
||||
const stopped = start()
|
||||
stopped.monitor.stop()
|
||||
const live = start()
|
||||
|
||||
expect(probeAllRemoteRuntimeSocketsNow(2_000)).toBe(1)
|
||||
probeAllRemoteRuntimeSocketsNow(2_000)
|
||||
expect(stopped.ping).not.toHaveBeenCalled()
|
||||
expect(live.ping).toHaveBeenCalledTimes(1)
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
expect(live.onDead).toHaveBeenCalledTimes(1)
|
||||
expect(stopped.onDead).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not arm a deadline for a socket that is still connecting', async () => {
|
||||
let open = false
|
||||
const ping = vi.fn(() => open)
|
||||
const { onDead } = start(vi.fn(), ping)
|
||||
|
||||
probeAllRemoteRuntimeSocketsNow(2_000)
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
expect(onDead).not.toHaveBeenCalled()
|
||||
|
||||
// Once open, the next resume probes it normally.
|
||||
open = true
|
||||
probeAllRemoteRuntimeSocketsNow(2_000)
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
expect(onDead).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('SharedControlReconnectScheduler.retryNow after an outage', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('restarts the backoff ladder so a failure right after wake retries quickly', () => {
|
||||
const scheduler = new SharedControlReconnectScheduler()
|
||||
const open = vi.fn()
|
||||
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||
scheduler.scheduleWithIdleBackoff(false, open)
|
||||
vi.runOnlyPendingTimers()
|
||||
}
|
||||
expect(scheduler.attemptCount).toBeGreaterThanOrEqual(6)
|
||||
|
||||
scheduler.scheduleWithIdleBackoff(false, open)
|
||||
expect(scheduler.retryNow()).toBe(true)
|
||||
expect(scheduler.attemptCount).toBe(0)
|
||||
})
|
||||
|
||||
it('restarts the ladder even when the wake lands while a reconnect is already opening', () => {
|
||||
const scheduler = new SharedControlReconnectScheduler()
|
||||
const open = vi.fn()
|
||||
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||
scheduler.scheduleWithIdleBackoff(false, open)
|
||||
vi.runOnlyPendingTimers()
|
||||
}
|
||||
// The last open() is in flight, so no timer is pending for retryNow to advance.
|
||||
expect(scheduler.isScheduled).toBe(false)
|
||||
expect(scheduler.retryNow()).toBe(false)
|
||||
expect(scheduler.attemptCount).toBe(0)
|
||||
})
|
||||
|
||||
it('does nothing when no reconnect is pending', () => {
|
||||
const scheduler = new SharedControlReconnectScheduler()
|
||||
expect(scheduler.retryNow()).toBe(false)
|
||||
expect(scheduler.attemptCount).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -282,9 +282,11 @@ export async function subscribeRemoteRuntimeTransport<TResult>(
|
||||
const monitoredWs = ws
|
||||
liveness = startRemoteRuntimeSocketLiveness({
|
||||
ping: () => {
|
||||
if (monitoredWs.readyState === WebSocket.OPEN) {
|
||||
monitoredWs.ping()
|
||||
if (monitoredWs.readyState !== WebSocket.OPEN) {
|
||||
return false
|
||||
}
|
||||
monitoredWs.ping()
|
||||
return true
|
||||
},
|
||||
onDead: () => {
|
||||
fail(
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* A paired client's open remote terminal must answer typing again on its own after the network to
|
||||
* its host goes away for 30 s (a lid close, a Tailscale re-key, a subway tunnel) and comes back.
|
||||
* Neither process restarts and nobody clicks Reconnect.
|
||||
*
|
||||
* Run:
|
||||
* ORCA_BACKGROUND_LAUNCH=1 pnpm exec playwright test \
|
||||
* tests/e2e/paired-remote-terminal-network-outage.spec.ts \
|
||||
* --config tests/playwright.config.ts --project electron-headless --workers=1
|
||||
*/
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id'
|
||||
import { decodePairingOffer, encodePairingOffer } from '../../src/shared/pairing'
|
||||
import { quotePosixShell } from '../../src/shared/wsl-login-shell-command'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host'
|
||||
import { cleanupE2EDaemons } from './helpers/electron-process-shutdown'
|
||||
import {
|
||||
launchPairedElectronClient,
|
||||
type PairedElectronClient
|
||||
} from './helpers/paired-electron-client'
|
||||
import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy'
|
||||
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-terminal-outage-'))
|
||||
const fixturePath = path.join(scratch, 'outage-terminal.mjs')
|
||||
|
||||
writeFileSync(
|
||||
fixturePath,
|
||||
[
|
||||
"import { appendFileSync } from 'node:fs'",
|
||||
'const sink = process.argv[2]',
|
||||
"process.stdout.write('READY\\r\\n')",
|
||||
"process.stdin.setEncoding('utf8')",
|
||||
"let pending = ''",
|
||||
"process.stdin.on('data', (data) => {",
|
||||
' pending += data',
|
||||
' const lines = pending.split(/\\r\\n|\\r|\\n/)',
|
||||
" pending = lines.pop() ?? ''",
|
||||
' for (const line of lines) {',
|
||||
' appendFileSync(sink, `${line}\\n`)',
|
||||
' process.stdout.write(`LIVE:${line}\\r\\n`)',
|
||||
' }',
|
||||
'})',
|
||||
'process.stdin.resume()'
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
test.afterAll(() => {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function fixtureCommand(sinkPath: string): string {
|
||||
const command = [process.execPath, fixturePath, sinkPath]
|
||||
return process.platform === 'win32'
|
||||
? `& ${command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')}`
|
||||
: command.map(quotePosixShell).join(' ')
|
||||
}
|
||||
|
||||
function readText(filePath: string): string {
|
||||
try {
|
||||
return readFileSync(filePath, 'utf8')
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(
|
||||
page: Page,
|
||||
environmentId: string,
|
||||
method: string,
|
||||
params: unknown
|
||||
): Promise<TResult> {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: test-only RPC result shape.
|
||||
return page.evaluate(
|
||||
async ({ environmentId, method, params }) => {
|
||||
const response = await window.api.runtimeEnvironments.call({
|
||||
selector: environmentId,
|
||||
method,
|
||||
params
|
||||
})
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ environmentId, method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
type PaneProbe = { content: string; ptyId: string | null; recovery: string | null }
|
||||
|
||||
async function readPane(page: Page, webTabId: string): Promise<PaneProbe> {
|
||||
return page.evaluate((id) => {
|
||||
const manager = window.__paneManagers?.get(id)
|
||||
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
|
||||
return {
|
||||
content: pane?.serializeAddon?.serialize?.() ?? '',
|
||||
ptyId: pane?.container.dataset.ptyId ?? null,
|
||||
recovery: pane?.container.dataset.ptyRecoveryState ?? null
|
||||
}
|
||||
}, webTabId)
|
||||
}
|
||||
|
||||
async function typeIntoPane(page: Page, webTabId: string, text: string): Promise<void> {
|
||||
await page.evaluate((id) => {
|
||||
const manager = window.__paneManagers?.get(id)
|
||||
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
|
||||
if (!pane) {
|
||||
throw new Error(`No pane mounted for ${id}`)
|
||||
}
|
||||
pane.terminal.focus()
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: xterm always renders its helper textarea.
|
||||
const textarea = pane.container.querySelector('.xterm-helper-textarea') as HTMLTextAreaElement
|
||||
textarea.focus()
|
||||
}, webTabId)
|
||||
await page.keyboard.type(text)
|
||||
await page.keyboard.press('Enter')
|
||||
}
|
||||
|
||||
const OUTAGES = [
|
||||
{ mode: 'silent', ms: 30_000, label: '30 s' },
|
||||
{ mode: 'reset', ms: 30_000, label: '30 s' },
|
||||
// Why: outlasts the pane's ~3 min auto-recovery window, with no `online` event when it returns.
|
||||
{ mode: 'silent', ms: 200_000, label: 'long' }
|
||||
] as const
|
||||
|
||||
for (const { mode, ms, label } of OUTAGES) {
|
||||
test(`an open remote terminal answers typing again after a ${label} ${mode} network outage`, async ({
|
||||
testRepoPath
|
||||
}, testInfo) => {
|
||||
test.setTimeout(ms + 330_000)
|
||||
const host = await launchHeadlessPairedRuntimeHost()
|
||||
let client: PairedElectronClient | null = null
|
||||
let proxy: FreezableTcpProxy | null = null
|
||||
try {
|
||||
await host.client.call('repo.add', { path: testRepoPath, kind: 'git' })
|
||||
const decoded = decodePairingOffer(host.offer.pairingUrl)
|
||||
if (!('endpoint' in decoded)) {
|
||||
throw new Error('expected a direct pairing offer')
|
||||
}
|
||||
const endpoint = new URL(decoded.endpoint)
|
||||
proxy = await startFreezableTcpProxy(endpoint.hostname, Number(endpoint.port))
|
||||
endpoint.port = String(proxy.port)
|
||||
client = await launchPairedElectronClient(
|
||||
{ pairingUrl: encodePairingOffer({ ...decoded, endpoint: endpoint.toString() }) },
|
||||
testInfo,
|
||||
`terminal ${mode} outage`
|
||||
)
|
||||
const page = client.page
|
||||
const environmentId = client.environmentId
|
||||
await expect
|
||||
.poll(() => page.evaluate(() => window.__store?.getState().allWorktrees()[0]?.id ?? null), {
|
||||
timeout: 60_000
|
||||
})
|
||||
.not.toBeNull()
|
||||
const worktreeId = await page.evaluate(() => window.__store!.getState().allWorktrees()[0]!.id)
|
||||
const sinkPath = path.join(scratch, `${mode}.log`)
|
||||
const created = await callRuntime<{ tab: { parentTabId: string; terminal: string | null } }>(
|
||||
page,
|
||||
environmentId,
|
||||
'session.tabs.createTerminal',
|
||||
{
|
||||
worktree: `id:${worktreeId}`,
|
||||
command: fixtureCommand(sinkPath),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
}
|
||||
)
|
||||
expect(created.tab.terminal).toBeTruthy()
|
||||
const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
page.evaluate(
|
||||
({ worktreeId, webTabId }) =>
|
||||
(window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some(
|
||||
(tab) => tab.id === webTabId
|
||||
),
|
||||
{ worktreeId, webTabId }
|
||||
),
|
||||
{ timeout: 60_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
await page.evaluate(
|
||||
({ worktreeId, webTabId }) => {
|
||||
const state = window.__store!.getState()
|
||||
state.setActiveView('terminal')
|
||||
state.setActiveWorktree(worktreeId)
|
||||
state.setActiveTab(webTabId)
|
||||
state.setActiveTabType('terminal', worktreeId)
|
||||
},
|
||||
{ worktreeId, webTabId }
|
||||
)
|
||||
await expect
|
||||
.poll(async () => (await readPane(page, webTabId)).content, { timeout: 60_000 })
|
||||
.toContain('READY')
|
||||
await typeIntoPane(page, webTabId, 'before')
|
||||
await expect
|
||||
.poll(async () => (await readPane(page, webTabId)).content, { timeout: 15_000 })
|
||||
.toContain('LIVE:before')
|
||||
const ptyBefore = (await readPane(page, webTabId)).ptyId
|
||||
|
||||
proxy.cut(mode)
|
||||
await page.waitForTimeout(ms)
|
||||
proxy.restore()
|
||||
|
||||
const marker = `after_${mode}_${Date.now()}`
|
||||
// Why retyped: typing during recovery may be dropped by design; the pane must answer once back.
|
||||
try {
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
if (!readText(sinkPath).includes(marker)) {
|
||||
await typeIntoPane(page, webTabId, marker)
|
||||
}
|
||||
return (await readPane(page, webTabId)).content
|
||||
},
|
||||
{ timeout: 120_000, intervals: [5_000] }
|
||||
)
|
||||
.toContain(`LIVE:${marker}`)
|
||||
} catch (error) {
|
||||
const probe = await readPane(page, webTabId)
|
||||
const evidence = {
|
||||
...probe,
|
||||
content: probe.content.slice(-600),
|
||||
ptyBefore,
|
||||
sink: readText(sinkPath),
|
||||
sharedControl: await page.evaluate(
|
||||
async (id) =>
|
||||
(await window.api.runtimeEnvironments.list()).find((entry) => entry.id === id) ??
|
||||
null,
|
||||
environmentId
|
||||
),
|
||||
proxyEvents: proxy.events
|
||||
}
|
||||
console.log('[outage] evidence', JSON.stringify(evidence))
|
||||
await page.screenshot({ path: testInfo.outputPath('after-outage.png') })
|
||||
throw error
|
||||
}
|
||||
expect((await readPane(page, webTabId)).ptyId).toBe(ptyBefore)
|
||||
} finally {
|
||||
if (client) {
|
||||
await cleanupE2EDaemons(client.userDataDir).catch(() => undefined)
|
||||
await client.dispose()
|
||||
}
|
||||
await proxy?.close()
|
||||
await host.dispose()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* #25783 on the managed-server path: an open pane on an orcad-managed SSH host answers typing again
|
||||
* on its own after the network to the host goes away for 30 s and comes back. Nobody clicks
|
||||
* Reconnect, and the pane keeps its original shell.
|
||||
*/
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import { readPairedPaneContent } from './helpers/paired-host-terminal'
|
||||
import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy'
|
||||
import {
|
||||
echoMarker,
|
||||
expectPaneText,
|
||||
expectTerminalAnswers,
|
||||
launchManagedWorkspace,
|
||||
openTerminalTab,
|
||||
skipUnlessManagedDockerHost,
|
||||
typeInTerminal,
|
||||
withManagedHost
|
||||
} from './helpers/orcad-managed-workspace'
|
||||
|
||||
skipUnlessManagedDockerHost()
|
||||
test.skip(process.platform === 'win32', 'The Docker SSH host uses POSIX tooling')
|
||||
|
||||
const OUTAGES = [
|
||||
{ mode: 'silent', ms: 30_000, label: '30 s' },
|
||||
{ mode: 'reset', ms: 30_000, label: '30 s' }
|
||||
] as const
|
||||
|
||||
function sshStatus(page: Page, targetId: string): Promise<string | null> {
|
||||
return page.evaluate(
|
||||
(id) => window.__store?.getState().sshConnectionStates.get(id)?.status ?? null,
|
||||
targetId
|
||||
)
|
||||
}
|
||||
|
||||
for (const { mode, ms, label } of OUTAGES) {
|
||||
test(`a managed pane answers typing again after a ${label} ${mode} network outage`, async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
|
||||
{}, testInfo) => {
|
||||
test.setTimeout(ms + 10 * 60_000)
|
||||
let proxy: FreezableTcpProxy | null = null
|
||||
try {
|
||||
await withManagedHost(testInfo, async (fixture) => {
|
||||
const { host } = fixture
|
||||
proxy = await startFreezableTcpProxy(host.input.host, host.input.port ?? 22)
|
||||
// Why the proxy: the profile reaches sshd only through it, so cutting it is the network.
|
||||
host.input = { ...host.input, port: proxy.port }
|
||||
const { page, seeded } = await launchManagedWorkspace(fixture)
|
||||
const tabId = await openTerminalTab(page, seeded.worktreeId)
|
||||
// Quoted apart so the typed command line itself never matches.
|
||||
await typeInTerminal(page, tabId, "echo SHELL''_PID=$$")
|
||||
await expect
|
||||
.poll(async () => /SHELL_PID=(\d+)/.exec(await readPairedPaneContent(page, tabId))?.[1])
|
||||
.toBeTruthy()
|
||||
const shellPid = /SHELL_PID=(\d+)/.exec(await readPairedPaneContent(page, tabId))![1]
|
||||
const marker = await echoMarker(page, tabId, 'BEFORE_OUTAGE')
|
||||
|
||||
proxy.cut(mode)
|
||||
await page.waitForTimeout(ms)
|
||||
proxy.restore()
|
||||
|
||||
try {
|
||||
await expectTerminalAnswers(page, tabId, 180_000)
|
||||
} catch (error) {
|
||||
console.log(
|
||||
'[outage] evidence',
|
||||
JSON.stringify({
|
||||
sshStatus: await sshStatus(page, seeded.targetId),
|
||||
pane: (await readPairedPaneContent(page, tabId)).slice(-600),
|
||||
hostShellAlive: host.exec!(`kill -0 ${shellPid} && echo alive || echo gone`),
|
||||
proxyEvents: proxy.events
|
||||
})
|
||||
)
|
||||
await page.screenshot({ path: testInfo.outputPath('after-outage.png') })
|
||||
throw error
|
||||
}
|
||||
await expectPaneText(page, tabId, marker)
|
||||
await typeInTerminal(page, tabId, "echo SHELL''_PID=$$")
|
||||
await expect
|
||||
.poll(
|
||||
async () =>
|
||||
(await readPairedPaneContent(page, tabId)).split(`SHELL_PID=${shellPid}`).length
|
||||
)
|
||||
.toBeGreaterThan(2)
|
||||
})
|
||||
} finally {
|
||||
await proxy?.close()
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user