fix(remote-terminal): a remote pane comes back on its own once its host answers again (#29444)

* fix(remote-runtime): probe sockets and reset backoff on wake, and keep late resubscribes revivable

Rebuilt from #23359 on current main. On OS resume or network return every
remote-runtime socket pings once and is replaced if it stays silent for 8 s,
the shared-control retry ladder restarts from its short delays, and a
resubscribe that fails or opens after the pane's recovery window latched is
parked so resume/online can revive it.

Co-authored-by: Meapri <104259736+Meapri@users.noreply.github.com>

* fix(remote-terminal): revive a latched remote pane when its host answers again

A pane whose ~3 min auto-recovery window ran out only came back on an OS
online/resume event or a Reconnect click. A Tailscale or VPN route that
returns fires neither, so the pane stayed dead over a working link even
though the host shell was alive. The environment's shared control socket
reaching ready again is now the revive signal: the pane advances its backoff
or replays its attach, exactly as Reconnect does.

Adds e2e repros for paired runtimes and managed SSH hosts: 30 s silent and
reset outages, and a silent outage longer than the recovery window.

* fix(remote-terminal): hold the control-ready reviver only while a pane is recovering

* fix(remote-runtime): re-probe a wake check whose deadline fired late, and trim the SSH outage spec

---------

Co-authored-by: Meapri <104259736+Meapri@users.noreply.github.com>
This commit is contained in:
Neil
2026-10-15 23:23:54 -07:00
committed by GitHub
co-authored by Meapri
parent 6ae1386f98
commit adce59a584
17 changed files with 1035 additions and 18 deletions
@@ -11,6 +11,7 @@ import type {
} from '../../shared/runtime-rpc-envelope'
import { RemoteRuntimeRequestConnection } from '../../shared/remote-runtime-request-connection'
import { RemoteRuntimeSharedControlConnection } from '../../shared/remote-runtime-shared-control-connection'
import { probeAllRemoteRuntimeSocketsNow } from '../../shared/remote-runtime-socket-liveness'
import type {
RemoteRuntimeSharedConnectionDiagnostics,
RemoteRuntimeSharedSubscription
@@ -204,6 +205,9 @@ export function retryRemoteRuntimeSharedControlConnectionsNow(): void {
for (const { connection } of sharedControlConnections.values()) {
connection.retryNow()
}
// Why: a socket that still reads OPEN after wake arms no reconnect timer, so retryNow finds
// nothing; probing every socket (shared control and per-stream) settles a dead path quickly (#9092).
probeAllRemoteRuntimeSocketsNow()
}
function getSharedControlConnection(
@@ -0,0 +1,20 @@
import { describe, expect, it, vi } from 'vitest'
import type * as SocketLiveness from '../../shared/remote-runtime-socket-liveness'
const probeAllRemoteRuntimeSocketsNow = vi.hoisted(() => vi.fn(() => 0))
vi.mock('../../shared/remote-runtime-socket-liveness', async (importOriginal) => ({
...(await importOriginal<typeof SocketLiveness>()),
probeAllRemoteRuntimeSocketsNow
}))
import { retryRemoteRuntimeSharedControlConnectionsNow } from './runtime-environment-request-connections'
// Regression for #9092: the resume/online IPC must probe sockets that still read OPEN after wake.
describe('retryRemoteRuntimeSharedControlConnectionsNow', () => {
it('probes every remote runtime socket even when no reconnect is pending', () => {
retryRemoteRuntimeSharedControlConnectionsNow()
expect(probeAllRemoteRuntimeSocketsNow).toHaveBeenCalledTimes(1)
})
})
@@ -434,4 +434,43 @@ describe('remote runtime pty reattach after the bounded recovery window', () =>
vi.useRealTimers()
}
})
// #9092: a rebound subscribe still opening its stream when a second deadline latches must stay
// revivable by system resume / network online instead of needing a manual Reconnect.
it('parks a rebound subscribe whose stream open fails after its deadline', async () => {
vi.useFakeTimers()
try {
const { transport } = await attachStalePane()
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
// The stream open for the rebound handle hangs through the whole new recovery window.
const streamOpen: { reject: ((error: Error) => void) | null } = { reject: null }
runtimeSubscribe.mockImplementation(
() =>
new Promise((_resolve, reject) => {
streamOpen.reject = reject
})
)
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
hostSnapshot('terminal-rotated', 3, 'epoch-2'),
'env-1'
)
await vi.waitFor(() => expect(streamOpen.reject).not.toBeNull())
expect(transport.getRecoveryState?.().phase).toBe('recovering')
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
streamOpen.reject?.(new Error('Could not connect to the remote Orca runtime.'))
await vi.advanceTimersByTimeAsync(0)
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
transport.destroy?.()
} finally {
vi.useRealTimers()
}
})
})
@@ -38,6 +38,44 @@ export function retryAllRemoteRuntimePtyRecoveriesNow(): number {
return advanced
}
// Why: a route that returns without an OS `online` event (Tailscale, VPN, a re-keyed link) fires no
// trigger, so a pane latched past its window stayed dead over a working link. The environment's
// control socket answering again is positive reachability evidence, as good as a Reconnect click.
const controlReadyRevivers = new Map<string, Set<() => boolean>>()
export function registerRemoteRuntimePtyControlReadyReviver(
environmentId: string,
revive: () => boolean
): () => void {
let revivers = controlReadyRevivers.get(environmentId)
if (!revivers) {
revivers = new Set()
controlReadyRevivers.set(environmentId, revivers)
}
revivers.add(revive)
return () => {
const current = controlReadyRevivers.get(environmentId)
current?.delete(revive)
if (current?.size === 0) {
controlReadyRevivers.delete(environmentId)
}
}
}
export function getControlReadyReviverCountForTests(environmentId: string): number {
return controlReadyRevivers.get(environmentId)?.size ?? 0
}
export function reviveRemoteRuntimePtyRecoveriesForEnvironment(environmentId: string): number {
let revived = 0
for (const revive of Array.from(controlReadyRevivers.get(environmentId) ?? [])) {
if (revive()) {
revived += 1
}
}
return revived
}
export class RemoteRuntimePtyRecoveryState {
private phase: RemoteRuntimePtyRecoveryPhase = 'idle'
private epoch = 0
@@ -572,6 +572,68 @@ describe('createRemoteRuntimePtyTransport', () => {
}
})
// #25783/#9092: a route returning without `online` (Tailscale, VPN) left a latched pane dead.
it('revives a pane latched past its window when its environment answers again', async () => {
vi.useFakeTimers()
try {
let partitioned = false
runtimeSubscribe.mockImplementation(
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
if (partitioned) {
throw Object.assign(new Error('Could not connect to the remote Orca runtime.'), {
code: 'remote_runtime_unavailable'
})
}
subscriptionCallbacks = callbacks
queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } }))
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const {
getControlReadyReviverCountForTests,
reviveRemoteRuntimePtyRecoveriesForEnvironment
} = await import('./remote-runtime-pty-recovery-state')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
transport.attach({ existingPtyId: 'remote:env-1@@terminal-1', callbacks: {} })
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
emitSnapshot(latestSubscribePayload().streamId, 'before partition')
expect(getControlReadyReviverCountForTests('env-1')).toBe(0)
partitioned = true
subscriptionCallbacks?.onClose?.()
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 60_000)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
const callsAtCutoff = runtimeSubscribe.mock.calls.length
partitioned = false
expect(reviveRemoteRuntimePtyRecoveriesForEnvironment('env-2')).toBe(0)
expect(reviveRemoteRuntimePtyRecoveriesForEnvironment('env-1')).toBe(1)
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(callsAtCutoff + 1))
await vi.waitFor(() => expect(subscribeFrameCount()).toBe(2))
emitSnapshot(latestSubscribePayload().streamId, 'after the route returned')
await vi.advanceTimersByTimeAsync(50)
expect(transport.isConnected()).toBe(true)
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
// Why: a healthy pane holds no registration, so a detached-and-discarded transport is never retained.
expect(getControlReadyReviverCountForTests('env-1')).toBe(0)
partitioned = true
subscriptionCallbacks?.onClose?.()
await vi.advanceTimersByTimeAsync(0)
expect(getControlReadyReviverCountForTests('env-1')).toBe(1)
transport.detach?.()
expect(getControlReadyReviverCountForTests('env-1')).toBe(0)
transport.destroy?.()
} finally {
vi.useRealTimers()
}
})
it('releases pending claimed input when reconnect subscription fails', async () => {
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
@@ -72,6 +72,7 @@ import {
import {
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
RemoteRuntimePtyRecoveryState,
registerRemoteRuntimePtyControlReadyReviver,
type RemoteRuntimePtyRecoveryPhase
} from './remote-runtime-pty-recovery-state'
import { createBrowserUuid } from '@/lib/browser-uuid'
@@ -324,7 +325,23 @@ export function createRemoteRuntimePtyTransport(
})
const disconnectedInputGrace = createRemoteRuntimeDisconnectedInputGrace()
let previousRecoveryPhase: RemoteRuntimePtyRecoveryPhase = 'idle'
let unregisterControlReadyReviver: (() => void) | null = null
// Why registered only while recovering: detach and destroy go idle, so a discarded transport is never retained.
function syncControlReadyReviver(): void {
const wantsRevival = recovery.isActive || recovery.currentPhase === 'disconnected'
if (wantsRevival && !unregisterControlReadyReviver) {
// Why: a backoff advances and a latched pane replays its attach, exactly as online/resume/Reconnect would.
unregisterControlReadyReviver = registerRemoteRuntimePtyControlReadyReviver(
runtimeEnvironmentId,
() => !destroyed && (recovery.retryNow() || (transport.retryRecovery?.() ?? false))
)
} else if (!wantsRevival && unregisterControlReadyReviver) {
unregisterControlReadyReviver()
unregisterControlReadyReviver = null
}
}
const recovery = new RemoteRuntimePtyRecoveryState(() => {
syncControlReadyReviver()
// Why: any recovery transition after the ack (a stale send, a close, the deadline) re-holds typing.
inputReadySubscription = null
const leftLatchForRetry = previousRecoveryPhase === 'disconnected' && recovery.isActive
@@ -2137,12 +2154,22 @@ export function createRemoteRuntimePtyTransport(
return false
}
if (recovery.currentPhase === 'disconnected') {
parkResubscribeAfterDeadline(targetHandle)
return true
}
scheduleResubscribeAfterTransportClose()
return true
}
// Why: a recoverable failure that lands after the auto-recovery deadline latched has no live epoch
// to join; without a parked retry, resume/online have nothing to fire and the pane stays
// disconnected until the user clicks Reconnect (#9092). Mirrors the connect path's park.
function parkResubscribeAfterDeadline(targetHandle: string): void {
recovery.parkRetryAfterDeadline((nextEpoch) =>
scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(targetHandle), nextEpoch)
)
}
// Why: after a transport drop the host may have re-minted this handle; re-derive from the snapshot so we don't mirror/type into whatever PTY now sits behind the stale one (#7718).
async function resubscribeAfterTransportClose(
previousHandle: string,
@@ -2300,6 +2327,17 @@ export function createRemoteRuntimePtyTransport(
let retryScheduled = false
void resubscribeAfterTransportClose(resubscribeHandle, replacementPolicy, recoveryEpoch)
.catch((error) => {
if (
!destroyed &&
connected &&
handle &&
recovery.currentPhase === 'disconnected' &&
recovery.ownsEpoch(recoveryEpoch) &&
isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))
) {
parkResubscribeAfterDeadline(handle)
return
}
if (!destroyed && connected && handle && recovery.isCurrent(recoveryEpoch)) {
clearPendingViewportClaim()
if (isRecoverablePaneBindingError(error)) {
@@ -2605,6 +2643,17 @@ export function createRemoteRuntimePtyTransport(
remotePtyId !== subscribedPtyId
) {
nextStream.close()
// Why: a stream that opened after the deadline latched is discarded like its late failure would be,
// so park it too or resume/online find nothing to revive (#9092).
if (
!destroyed &&
connected &&
handle === subscribedHandle &&
recovery.currentPhase === 'disconnected' &&
(expectedRecoveryEpoch === undefined || recovery.ownsEpoch(expectedRecoveryEpoch))
) {
parkResubscribeAfterDeadline(subscribedHandle)
}
return
}
closeMultiplexedStream()
@@ -0,0 +1,186 @@
/**
* Regression for #9092: a remote pane whose resubscribe attempt was still in flight when the
* auto-recovery deadline latched dropped the late (recoverable) failure, so nothing was parked
* and system resume / network online had nothing to fire. The pane stayed disconnected until
* the user clicked Reconnect, even after the host came back.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
TerminalStreamOpcode,
decodeTerminalStreamFrame,
decodeTerminalStreamJson,
encodeTerminalStreamFrame,
encodeTerminalStreamJson,
encodeTerminalStreamText
} from '../../../../shared/terminal-stream-protocol'
import { REMOTE_RUNTIME_CONNECT_FAILURE_PHRASE } from '../../../../shared/remote-runtime-connect-bound'
import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state'
const FIRST_HANDLE = 'terminal-1'
const FIRST_PTY_ID = 'remote:env-1@@terminal-1'
describe('remote runtime resubscribe failing after the recovery deadline', () => {
const subscriptionSendBinary = vi.fn()
let subscriptionCallbacks: {
onResponse: (response: unknown) => void
onBinary?: (bytes: Uint8Array<ArrayBufferLike>) => void
onClose?: () => void
} | null = null
let resolvePaneCalls = 0
let holdResolvePane: { reject: (error: Error) => void } | null = null
let hangLateResolvePane = true
let streamOpens = 0
const lateStreamOpen: { hang: boolean; release: (() => void) | null } = {
hang: false,
release: null
}
function latestStreamId(): number {
const frame = subscriptionSendBinary.mock.calls
.map((call) => decodeTerminalStreamFrame(call[0]))
.findLast((candidate) => candidate?.opcode === TerminalStreamOpcode.Subscribe)
const payload = frame ? decodeTerminalStreamJson<{ streamId: number }>(frame.payload) : null
if (!payload) {
throw new Error('missing terminal subscribe frame')
}
return payload.streamId
}
function emitSnapshot(streamId: number): void {
const frames = [
{ opcode: TerminalStreamOpcode.SnapshotStart, payload: encodeTerminalStreamJson({}) },
{ opcode: TerminalStreamOpcode.SnapshotChunk, payload: encodeTerminalStreamText('live') },
{ opcode: TerminalStreamOpcode.SnapshotEnd, payload: new Uint8Array() }
]
frames.forEach((frame, index) =>
subscriptionCallbacks?.onBinary?.(
encodeTerminalStreamFrame({ ...frame, streamId, seq: index + 1 })
)
)
}
beforeEach(() => {
vi.resetModules()
vi.doMock('@/runtime/web-runtime-session', () => ({
refreshWebRuntimeSessionTabsSnapshot: vi.fn(async () => {})
}))
subscriptionCallbacks = null
subscriptionSendBinary.mockReset()
resolvePaneCalls = 0
holdResolvePane = null
hangLateResolvePane = true
streamOpens = 0
lateStreamOpen.hang = false
lateStreamOpen.release = null
const runtimeCall = vi.fn(async (request: { method: string; params?: unknown }) => {
if (request.method === 'terminal.resolvePane') {
resolvePaneCalls += 1
const params = request.params as { paneKey: string; worktreeId: string }
if (hangLateResolvePane && holdResolvePane === null && resolvePaneCalls > 1) {
// Why: the post-deadline attempt hangs so the test observes that it was issued.
return new Promise<never>((_resolve, reject) => {
holdResolvePane = { reject }
})
}
return {
ok: true,
result: {
terminal: {
handle: FIRST_HANDLE,
tabId: 'tab-1',
leafId: 'pane:1',
worktreeId: params.worktreeId
}
}
}
}
return { ok: true, result: { terminal: { handle: FIRST_HANDLE } } }
})
const runtimeSubscribe = vi.fn(
async (_args: unknown, callbacks: typeof subscriptionCallbacks) => {
streamOpens += 1
if (lateStreamOpen.hang && streamOpens > 1) {
await new Promise<void>((resolve) => {
lateStreamOpen.release = resolve
})
}
subscriptionCallbacks = callbacks
queueMicrotask(() =>
subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'ready' } })
)
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
}
)
vi.stubGlobal('window', {
api: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } }
})
})
afterEach(() => {
vi.useRealTimers()
})
it('parks the late failure so a resume trigger can revive the pane', async () => {
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
transport.attach({ existingPtyId: FIRST_PTY_ID, cols: 80, rows: 24, callbacks: {} })
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
emitSnapshot(latestStreamId())
expect(transport.isConnected()).toBe(true)
vi.useFakeTimers()
subscriptionCallbacks?.onClose?.()
await vi.advanceTimersByTimeAsync(0)
await vi.waitFor(() => expect(holdResolvePane).not.toBeNull())
// The host stays unreachable past the whole auto-recovery window.
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
holdResolvePane?.reject(new Error(`${REMOTE_RUNTIME_CONNECT_FAILURE_PHRASE}.`))
holdResolvePane = null
await vi.advanceTimersByTimeAsync(0)
const callsBeforeResume = resolvePaneCalls
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
await vi.advanceTimersByTimeAsync(0)
expect(resolvePaneCalls).toBeGreaterThan(callsBeforeResume)
transport.destroy?.()
})
it('parks a late success whose stream opened after the deadline', async () => {
hangLateResolvePane = false
lateStreamOpen.hang = true
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const { retryAllRemoteRuntimePtyRecoveriesNow } =
await import('./remote-runtime-pty-recovery-state')
const transport = createRemoteRuntimePtyTransport('env-1', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
transport.attach({ existingPtyId: FIRST_PTY_ID, cols: 80, rows: 24, callbacks: {} })
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
emitSnapshot(latestStreamId())
vi.useFakeTimers()
subscriptionCallbacks?.onClose?.()
await vi.advanceTimersByTimeAsync(0)
await vi.waitFor(() => expect(lateStreamOpen.release).not.toBeNull())
// The stream open stays in flight past the whole window, then succeeds.
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
lateStreamOpen.release?.()
await vi.advanceTimersByTimeAsync(0)
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
transport.destroy?.()
})
})
@@ -2,14 +2,18 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { renderHook } from '@testing-library/react'
const { retryAllRemoteRuntimePtyRecoveriesNowMock } = vi.hoisted(() => ({
retryAllRemoteRuntimePtyRecoveriesNowMock: vi.fn()
const { retryAllRemoteRuntimePtyRecoveriesNowMock, reviveForEnvironmentMock } = vi.hoisted(() => ({
retryAllRemoteRuntimePtyRecoveriesNowMock: vi.fn(),
reviveForEnvironmentMock: vi.fn()
}))
vi.mock('@/components/terminal-pane/remote-runtime-pty-recovery-state', () => ({
retryAllRemoteRuntimePtyRecoveriesNow: retryAllRemoteRuntimePtyRecoveriesNowMock
retryAllRemoteRuntimePtyRecoveriesNow: retryAllRemoteRuntimePtyRecoveriesNowMock,
reviveRemoteRuntimePtyRecoveriesForEnvironment: reviveForEnvironmentMock
}))
type DiagnosticsEvent = { environmentId: string; diagnostics: { state: string } }
import { useRemoteRuntimeRecoveryTriggers } from './use-remote-runtime-recovery-triggers'
describe('useRemoteRuntimeRecoveryTriggers', () => {
@@ -20,6 +24,14 @@ describe('useRemoteRuntimeRecoveryTriggers', () => {
return unsubscribeSystemResumed
})
const retryConnectionsNow = vi.fn(() => Promise.resolve())
let diagnosticsCallback: ((event: DiagnosticsEvent) => void) | null = null
const unsubscribeDiagnostics = vi.fn()
const onSharedControlDiagnostics = vi.fn((callback: (event: DiagnosticsEvent) => void) => {
diagnosticsCallback = callback
return unsubscribeDiagnostics
})
const emitState = (environmentId: string, state: string): void =>
diagnosticsCallback?.({ environmentId, diagnostics: { state } })
beforeEach(() => {
systemResumedCallback = null
@@ -27,9 +39,12 @@ describe('useRemoteRuntimeRecoveryTriggers', () => {
onSystemResumed.mockClear()
retryConnectionsNow.mockClear()
retryAllRemoteRuntimePtyRecoveriesNowMock.mockClear()
reviveForEnvironmentMock.mockClear()
diagnosticsCallback = null
unsubscribeDiagnostics.mockClear()
;(window as unknown as { api: unknown }).api = {
ui: { onSystemResumed },
runtimeEnvironments: { retryConnectionsNow }
runtimeEnvironments: { retryConnectionsNow, onSharedControlDiagnostics }
}
})
@@ -60,4 +75,22 @@ describe('useRemoteRuntimeRecoveryTriggers', () => {
expect(retryAllRemoteRuntimePtyRecoveriesNowMock).not.toHaveBeenCalled()
expect(unsubscribeSystemResumed).toHaveBeenCalledTimes(1)
})
// #9092/#25783: a returning Tailscale or VPN route fires no `online`, so the control socket is the signal.
it("revives an environment's panes when its control connection becomes ready again", () => {
const { unmount } = renderHook(() => useRemoteRuntimeRecoveryTriggers())
emitState('env-1', 'ready')
emitState('env-1', 'ready')
expect(reviveForEnvironmentMock.mock.calls).toEqual([['env-1']])
emitState('env-1', 'reconnecting')
emitState('env-2', 'connecting')
expect(reviveForEnvironmentMock).toHaveBeenCalledTimes(1)
emitState('env-1', 'ready')
expect(reviveForEnvironmentMock.mock.calls).toEqual([['env-1'], ['env-1']])
unmount()
expect(unsubscribeDiagnostics).toHaveBeenCalledTimes(1)
})
})
@@ -1,5 +1,8 @@
import { useEffect } from 'react'
import { retryAllRemoteRuntimePtyRecoveriesNow } from '@/components/terminal-pane/remote-runtime-pty-recovery-state'
import {
retryAllRemoteRuntimePtyRecoveriesNow,
reviveRemoteRuntimePtyRecoveriesForEnvironment
} from '@/components/terminal-pane/remote-runtime-pty-recovery-state'
export function useRemoteRuntimeRecoveryTriggers(): void {
useEffect(() => {
@@ -13,9 +16,24 @@ export function useRemoteRuntimeRecoveryTriggers(): void {
typeof window.api?.ui?.onSystemResumed === 'function'
? window.api.ui.onSystemResumed(advanceRemoteRuntimeRecoveryBackoffs)
: null
const readyByEnvironment = new Map<string, boolean>()
const unsubscribeControlDiagnostics =
typeof window.api?.runtimeEnvironments?.onSharedControlDiagnostics === 'function'
? window.api.runtimeEnvironments.onSharedControlDiagnostics(
({ environmentId, diagnostics }) => {
const ready = diagnostics.state === 'ready'
const wasReady = readyByEnvironment.get(environmentId) ?? false
readyByEnvironment.set(environmentId, ready)
if (ready && !wasReady) {
reviveRemoteRuntimePtyRecoveriesForEnvironment(environmentId)
}
}
)
: null
return () => {
window.removeEventListener('online', advanceRemoteRuntimeRecoveryBackoffs)
unsubscribeSystemResumed?.()
unsubscribeControlDiagnostics?.()
}
}, [])
}
@@ -80,9 +80,11 @@ export function openRemoteRuntimePassthroughSocket(
)
const liveness = startRemoteRuntimeSocketLiveness({
ping: () => {
if (ws.readyState === WebSocket.OPEN) {
ws.ping()
if (ws.readyState !== WebSocket.OPEN) {
return false
}
ws.ping()
return true
},
onDead: () =>
end(
@@ -74,9 +74,11 @@ export function openSharedControlSocket(
const liveness = callbacks.liveness
const monitor = startRemoteRuntimeSocketLiveness({
ping: () => {
if (ws.readyState === 1) {
ws.ping()
if (ws.readyState !== 1) {
return false
}
ws.ping()
return true
},
onDead: () => {
if (callbacks.getCurrentSocket() !== ws) {
@@ -75,6 +75,10 @@ export class SharedControlReconnectScheduler {
// Why: OS resume / browser online should advance an already-scheduled reconnect, not start a new one.
retryNow(): boolean {
// Why: a wake or network change is fresh evidence, so the next failure starts again from the short
// delays instead of the 5-minute idle cap earned during the outage — also when an open is already
// in flight and no timer is pending (#9092).
this.attempt = 0
if (!this.timer || !this.pendingOpen) {
return false
}
+73 -7
View File
@@ -18,13 +18,33 @@ export type RemoteRuntimeSocketLivenessOptions = {
livenessTimeoutMs?: number
}
// Why: after sleep or a network change a socket can read OPEN while its path is gone; the normal
// cadence needs ~30-40 s to prove that. A resume probe settles it in one short, explicit window.
export const REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS = 8_000
export type RemoteRuntimeSocketLivenessMonitor = {
noteActivity: () => void
/** Ping now; declare the socket dead unless anything arrives within `deadlineMs`. */
probeNow: (deadlineMs?: number) => void
stop: () => void
}
const activeMonitors = new Set<RemoteRuntimeSocketLivenessMonitor>()
/** Probe every live remote-runtime socket in this process, e.g. on OS resume or network change. */
export function probeAllRemoteRuntimeSocketsNow(
deadlineMs = REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS
): number {
const monitors = Array.from(activeMonitors)
for (const monitor of monitors) {
monitor.probeNow(deadlineMs)
}
return monitors.length
}
export function startRemoteRuntimeSocketLiveness(args: {
ping: () => void
/** Returns false when nothing was sent, e.g. the socket is still connecting. */
ping: () => boolean
onDead: () => void
options?: RemoteRuntimeSocketLivenessOptions
now?: () => number
@@ -35,6 +55,8 @@ export function startRemoteRuntimeSocketLiveness(args: {
args.options?.livenessTimeoutMs ?? REMOTE_RUNTIME_SOCKET_LIVENESS_TIMEOUT_MS
let lastTickAt = now()
let probeSentAt: number | null = null
let activitySinceResumeProbe = false
let resumeProbeTimer: ReturnType<typeof setTimeout> | null = null
let stopped = false
const timer = setInterval(() => {
@@ -72,20 +94,64 @@ export function startRemoteRuntimeSocketLiveness(args: {
}
stopped = true
clearInterval(timer)
if (resumeProbeTimer !== null) {
clearTimeout(resumeProbeTimer)
resumeProbeTimer = null
}
activeMonitors.delete(monitor)
}
function tryPing(): void {
try {
args.ping()
} catch {
// Why: ping() can throw while a socket is mid-teardown; the probe deadline still settles it.
function probeNow(deadlineMs = REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS): void {
if (stopped || resumeProbeTimer !== null) {
return
}
activitySinceResumeProbe = false
// Why: a socket still connecting has no path to probe yet; a deadline with no ping could kill it mid-handshake.
if (!tryPing()) {
return
}
const probedAt = now()
resumeProbeTimer = setTimeout(() => {
resumeProbeTimer = null
if (stopped || activitySinceResumeProbe) {
return
}
// Why: a late timer means this process slept or stalled, so replies may still be queued; probe afresh.
const elapsedMs = now() - probedAt
if (elapsedMs < 0 || elapsedMs > deadlineMs * 1.5) {
probeNow(deadlineMs)
return
}
stop()
args.onDead()
}, deadlineMs)
// Why: mobile typechecks shared code with DOM timer types where unref is absent.
if (
typeof resumeProbeTimer === 'object' &&
'unref' in resumeProbeTimer &&
typeof resumeProbeTimer.unref === 'function'
) {
resumeProbeTimer.unref()
}
}
return {
function tryPing(): boolean {
try {
return args.ping()
} catch {
// Why: ping() can throw while a socket is mid-teardown; the probe deadline still settles it.
return true
}
}
const monitor: RemoteRuntimeSocketLivenessMonitor = {
noteActivity: () => {
probeSentAt = null
activitySinceResumeProbe = true
},
probeNow,
stop
}
activeMonitors.add(monitor)
return monitor
}
@@ -0,0 +1,149 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
probeAllRemoteRuntimeSocketsNow,
startRemoteRuntimeSocketLiveness
} from './remote-runtime-socket-liveness'
import { SharedControlReconnectScheduler } from './remote-runtime-shared-control-reconnect'
// Regression for #9092: after sleep a socket can still read OPEN while its path is gone, and the
// normal cadence took ~30-40 s to notice; resume now probes every socket with a short deadline.
describe('remote runtime resume probe', () => {
const monitors: { stop: () => void }[] = []
beforeEach(() => {
vi.useFakeTimers()
})
afterEach(() => {
for (const monitor of monitors.splice(0)) {
monitor.stop()
}
vi.useRealTimers()
})
function start(onDead = vi.fn(), ping = vi.fn(() => true)) {
const monitor = startRemoteRuntimeSocketLiveness({
ping,
onDead,
options: { pingIntervalMs: 10_000, livenessTimeoutMs: 25_000 }
})
monitors.push(monitor)
return { monitor, onDead, ping }
}
it('declares a silent socket dead within the resume deadline', async () => {
const { onDead, ping } = start()
expect(probeAllRemoteRuntimeSocketsNow(2_000)).toBe(1)
expect(ping).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(1_999)
expect(onDead).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1)
expect(onDead).toHaveBeenCalledTimes(1)
})
it('probes afresh instead of failing a socket when the process stalled through the deadline', async () => {
let clock = 0
const onDead = vi.fn()
const ping = vi.fn(() => true)
monitors.push(
startRemoteRuntimeSocketLiveness({
ping,
onDead,
now: () => clock,
options: { pingIntervalMs: 60_000, livenessTimeoutMs: 120_000 }
})
)
probeAllRemoteRuntimeSocketsNow(2_000)
clock += 20_000
await vi.advanceTimersByTimeAsync(2_000)
expect(onDead).not.toHaveBeenCalled()
expect(ping).toHaveBeenCalledTimes(2)
clock += 2_000
await vi.advanceTimersByTimeAsync(2_000)
expect(onDead).toHaveBeenCalledTimes(1)
})
it('keeps a socket that answers the probe', async () => {
const { monitor, onDead } = start()
probeAllRemoteRuntimeSocketsNow(2_000)
await vi.advanceTimersByTimeAsync(500)
monitor.noteActivity()
await vi.advanceTimersByTimeAsync(5_000)
expect(onDead).not.toHaveBeenCalled()
})
it('skips stopped monitors and coalesces a repeated resume into one probe', async () => {
const stopped = start()
stopped.monitor.stop()
const live = start()
expect(probeAllRemoteRuntimeSocketsNow(2_000)).toBe(1)
probeAllRemoteRuntimeSocketsNow(2_000)
expect(stopped.ping).not.toHaveBeenCalled()
expect(live.ping).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(2_000)
expect(live.onDead).toHaveBeenCalledTimes(1)
expect(stopped.onDead).not.toHaveBeenCalled()
})
it('does not arm a deadline for a socket that is still connecting', async () => {
let open = false
const ping = vi.fn(() => open)
const { onDead } = start(vi.fn(), ping)
probeAllRemoteRuntimeSocketsNow(2_000)
await vi.advanceTimersByTimeAsync(2_000)
expect(onDead).not.toHaveBeenCalled()
// Once open, the next resume probes it normally.
open = true
probeAllRemoteRuntimeSocketsNow(2_000)
await vi.advanceTimersByTimeAsync(2_000)
expect(onDead).toHaveBeenCalledTimes(1)
})
})
describe('SharedControlReconnectScheduler.retryNow after an outage', () => {
beforeEach(() => {
vi.useFakeTimers()
})
afterEach(() => {
vi.useRealTimers()
})
it('restarts the backoff ladder so a failure right after wake retries quickly', () => {
const scheduler = new SharedControlReconnectScheduler()
const open = vi.fn()
for (let attempt = 0; attempt < 6; attempt += 1) {
scheduler.scheduleWithIdleBackoff(false, open)
vi.runOnlyPendingTimers()
}
expect(scheduler.attemptCount).toBeGreaterThanOrEqual(6)
scheduler.scheduleWithIdleBackoff(false, open)
expect(scheduler.retryNow()).toBe(true)
expect(scheduler.attemptCount).toBe(0)
})
it('restarts the ladder even when the wake lands while a reconnect is already opening', () => {
const scheduler = new SharedControlReconnectScheduler()
const open = vi.fn()
for (let attempt = 0; attempt < 6; attempt += 1) {
scheduler.scheduleWithIdleBackoff(false, open)
vi.runOnlyPendingTimers()
}
// The last open() is in flight, so no timer is pending for retryNow to advance.
expect(scheduler.isScheduled).toBe(false)
expect(scheduler.retryNow()).toBe(false)
expect(scheduler.attemptCount).toBe(0)
})
it('does nothing when no reconnect is pending', () => {
const scheduler = new SharedControlReconnectScheduler()
expect(scheduler.retryNow()).toBe(false)
expect(scheduler.attemptCount).toBe(0)
})
})
@@ -282,9 +282,11 @@ export async function subscribeRemoteRuntimeTransport<TResult>(
const monitoredWs = ws
liveness = startRemoteRuntimeSocketLiveness({
ping: () => {
if (monitoredWs.readyState === WebSocket.OPEN) {
monitoredWs.ping()
if (monitoredWs.readyState !== WebSocket.OPEN) {
return false
}
monitoredWs.ping()
return true
},
onDead: () => {
fail(
@@ -0,0 +1,254 @@
/**
* A paired client's open remote terminal must answer typing again on its own after the network to
* its host goes away for 30 s (a lid close, a Tailscale re-key, a subway tunnel) and comes back.
* Neither process restarts and nobody clicks Reconnect.
*
* Run:
* ORCA_BACKGROUND_LAUNCH=1 pnpm exec playwright test \
* tests/e2e/paired-remote-terminal-network-outage.spec.ts \
* --config tests/playwright.config.ts --project electron-headless --workers=1
*/
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import type { Page } from '@stablyai/playwright-test'
import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id'
import { decodePairingOffer, encodePairingOffer } from '../../src/shared/pairing'
import { quotePosixShell } from '../../src/shared/wsl-login-shell-command'
import { expect, test } from './helpers/orca-app'
import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host'
import { cleanupE2EDaemons } from './helpers/electron-process-shutdown'
import {
launchPairedElectronClient,
type PairedElectronClient
} from './helpers/paired-electron-client'
import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy'
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-terminal-outage-'))
const fixturePath = path.join(scratch, 'outage-terminal.mjs')
writeFileSync(
fixturePath,
[
"import { appendFileSync } from 'node:fs'",
'const sink = process.argv[2]',
"process.stdout.write('READY\\r\\n')",
"process.stdin.setEncoding('utf8')",
"let pending = ''",
"process.stdin.on('data', (data) => {",
' pending += data',
' const lines = pending.split(/\\r\\n|\\r|\\n/)',
" pending = lines.pop() ?? ''",
' for (const line of lines) {',
' appendFileSync(sink, `${line}\\n`)',
' process.stdout.write(`LIVE:${line}\\r\\n`)',
' }',
'})',
'process.stdin.resume()'
].join('\n')
)
test.afterAll(() => {
rmSync(scratch, { recursive: true, force: true })
})
function fixtureCommand(sinkPath: string): string {
const command = [process.execPath, fixturePath, sinkPath]
return process.platform === 'win32'
? `& ${command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')}`
: command.map(quotePosixShell).join(' ')
}
function readText(filePath: string): string {
try {
return readFileSync(filePath, 'utf8')
} catch {
return ''
}
}
async function callRuntime<TResult>(
page: Page,
environmentId: string,
method: string,
params: unknown
): Promise<TResult> {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: test-only RPC result shape.
return page.evaluate(
async ({ environmentId, method, params }) => {
const response = await window.api.runtimeEnvironments.call({
selector: environmentId,
method,
params
})
if (!response.ok) {
throw new Error(`${response.error.code}: ${response.error.message}`)
}
return response.result
},
{ environmentId, method, params }
) as Promise<TResult>
}
type PaneProbe = { content: string; ptyId: string | null; recovery: string | null }
async function readPane(page: Page, webTabId: string): Promise<PaneProbe> {
return page.evaluate((id) => {
const manager = window.__paneManagers?.get(id)
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
return {
content: pane?.serializeAddon?.serialize?.() ?? '',
ptyId: pane?.container.dataset.ptyId ?? null,
recovery: pane?.container.dataset.ptyRecoveryState ?? null
}
}, webTabId)
}
async function typeIntoPane(page: Page, webTabId: string, text: string): Promise<void> {
await page.evaluate((id) => {
const manager = window.__paneManagers?.get(id)
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
if (!pane) {
throw new Error(`No pane mounted for ${id}`)
}
pane.terminal.focus()
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: xterm always renders its helper textarea.
const textarea = pane.container.querySelector('.xterm-helper-textarea') as HTMLTextAreaElement
textarea.focus()
}, webTabId)
await page.keyboard.type(text)
await page.keyboard.press('Enter')
}
const OUTAGES = [
{ mode: 'silent', ms: 30_000, label: '30 s' },
{ mode: 'reset', ms: 30_000, label: '30 s' },
// Why: outlasts the pane's ~3 min auto-recovery window, with no `online` event when it returns.
{ mode: 'silent', ms: 200_000, label: 'long' }
] as const
for (const { mode, ms, label } of OUTAGES) {
test(`an open remote terminal answers typing again after a ${label} ${mode} network outage`, async ({
testRepoPath
}, testInfo) => {
test.setTimeout(ms + 330_000)
const host = await launchHeadlessPairedRuntimeHost()
let client: PairedElectronClient | null = null
let proxy: FreezableTcpProxy | null = null
try {
await host.client.call('repo.add', { path: testRepoPath, kind: 'git' })
const decoded = decodePairingOffer(host.offer.pairingUrl)
if (!('endpoint' in decoded)) {
throw new Error('expected a direct pairing offer')
}
const endpoint = new URL(decoded.endpoint)
proxy = await startFreezableTcpProxy(endpoint.hostname, Number(endpoint.port))
endpoint.port = String(proxy.port)
client = await launchPairedElectronClient(
{ pairingUrl: encodePairingOffer({ ...decoded, endpoint: endpoint.toString() }) },
testInfo,
`terminal ${mode} outage`
)
const page = client.page
const environmentId = client.environmentId
await expect
.poll(() => page.evaluate(() => window.__store?.getState().allWorktrees()[0]?.id ?? null), {
timeout: 60_000
})
.not.toBeNull()
const worktreeId = await page.evaluate(() => window.__store!.getState().allWorktrees()[0]!.id)
const sinkPath = path.join(scratch, `${mode}.log`)
const created = await callRuntime<{ tab: { parentTabId: string; terminal: string | null } }>(
page,
environmentId,
'session.tabs.createTerminal',
{
worktree: `id:${worktreeId}`,
command: fixtureCommand(sinkPath),
activate: false,
select: false,
navigation: 'caller'
}
)
expect(created.tab.terminal).toBeTruthy()
const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId)
await expect
.poll(
() =>
page.evaluate(
({ worktreeId, webTabId }) =>
(window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some(
(tab) => tab.id === webTabId
),
{ worktreeId, webTabId }
),
{ timeout: 60_000 }
)
.toBe(true)
await page.evaluate(
({ worktreeId, webTabId }) => {
const state = window.__store!.getState()
state.setActiveView('terminal')
state.setActiveWorktree(worktreeId)
state.setActiveTab(webTabId)
state.setActiveTabType('terminal', worktreeId)
},
{ worktreeId, webTabId }
)
await expect
.poll(async () => (await readPane(page, webTabId)).content, { timeout: 60_000 })
.toContain('READY')
await typeIntoPane(page, webTabId, 'before')
await expect
.poll(async () => (await readPane(page, webTabId)).content, { timeout: 15_000 })
.toContain('LIVE:before')
const ptyBefore = (await readPane(page, webTabId)).ptyId
proxy.cut(mode)
await page.waitForTimeout(ms)
proxy.restore()
const marker = `after_${mode}_${Date.now()}`
// Why retyped: typing during recovery may be dropped by design; the pane must answer once back.
try {
await expect
.poll(
async () => {
if (!readText(sinkPath).includes(marker)) {
await typeIntoPane(page, webTabId, marker)
}
return (await readPane(page, webTabId)).content
},
{ timeout: 120_000, intervals: [5_000] }
)
.toContain(`LIVE:${marker}`)
} catch (error) {
const probe = await readPane(page, webTabId)
const evidence = {
...probe,
content: probe.content.slice(-600),
ptyBefore,
sink: readText(sinkPath),
sharedControl: await page.evaluate(
async (id) =>
(await window.api.runtimeEnvironments.list()).find((entry) => entry.id === id) ??
null,
environmentId
),
proxyEvents: proxy.events
}
console.log('[outage] evidence', JSON.stringify(evidence))
await page.screenshot({ path: testInfo.outputPath('after-outage.png') })
throw error
}
expect((await readPane(page, webTabId)).ptyId).toBe(ptyBefore)
} finally {
if (client) {
await cleanupE2EDaemons(client.userDataDir).catch(() => undefined)
await client.dispose()
}
await proxy?.close()
await host.dispose()
}
})
}
@@ -0,0 +1,89 @@
/**
* #25783 on the managed-server path: an open pane on an orcad-managed SSH host answers typing again
* on its own after the network to the host goes away for 30 s and comes back. Nobody clicks
* Reconnect, and the pane keeps its original shell.
*/
import type { Page } from '@stablyai/playwright-test'
import { expect, test } from './helpers/orca-app'
import { readPairedPaneContent } from './helpers/paired-host-terminal'
import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy'
import {
echoMarker,
expectPaneText,
expectTerminalAnswers,
launchManagedWorkspace,
openTerminalTab,
skipUnlessManagedDockerHost,
typeInTerminal,
withManagedHost
} from './helpers/orcad-managed-workspace'
skipUnlessManagedDockerHost()
test.skip(process.platform === 'win32', 'The Docker SSH host uses POSIX tooling')
const OUTAGES = [
{ mode: 'silent', ms: 30_000, label: '30 s' },
{ mode: 'reset', ms: 30_000, label: '30 s' }
] as const
function sshStatus(page: Page, targetId: string): Promise<string | null> {
return page.evaluate(
(id) => window.__store?.getState().sshConnectionStates.get(id)?.status ?? null,
targetId
)
}
for (const { mode, ms, label } of OUTAGES) {
test(`a managed pane answers typing again after a ${label} ${mode} network outage`, async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session.
{}, testInfo) => {
test.setTimeout(ms + 10 * 60_000)
let proxy: FreezableTcpProxy | null = null
try {
await withManagedHost(testInfo, async (fixture) => {
const { host } = fixture
proxy = await startFreezableTcpProxy(host.input.host, host.input.port ?? 22)
// Why the proxy: the profile reaches sshd only through it, so cutting it is the network.
host.input = { ...host.input, port: proxy.port }
const { page, seeded } = await launchManagedWorkspace(fixture)
const tabId = await openTerminalTab(page, seeded.worktreeId)
// Quoted apart so the typed command line itself never matches.
await typeInTerminal(page, tabId, "echo SHELL''_PID=$$")
await expect
.poll(async () => /SHELL_PID=(\d+)/.exec(await readPairedPaneContent(page, tabId))?.[1])
.toBeTruthy()
const shellPid = /SHELL_PID=(\d+)/.exec(await readPairedPaneContent(page, tabId))![1]
const marker = await echoMarker(page, tabId, 'BEFORE_OUTAGE')
proxy.cut(mode)
await page.waitForTimeout(ms)
proxy.restore()
try {
await expectTerminalAnswers(page, tabId, 180_000)
} catch (error) {
console.log(
'[outage] evidence',
JSON.stringify({
sshStatus: await sshStatus(page, seeded.targetId),
pane: (await readPairedPaneContent(page, tabId)).slice(-600),
hostShellAlive: host.exec!(`kill -0 ${shellPid} && echo alive || echo gone`),
proxyEvents: proxy.events
})
)
await page.screenshot({ path: testInfo.outputPath('after-outage.png') })
throw error
}
await expectPaneText(page, tabId, marker)
await typeInTerminal(page, tabId, "echo SHELL''_PID=$$")
await expect
.poll(
async () =>
(await readPairedPaneContent(page, tabId)).split(`SHELL_PID=${shellPid}`).length
)
.toBeGreaterThan(2)
})
} finally {
await proxy?.close()
}
})
}