diff --git a/src/main/ipc/runtime-environment-request-connections.ts b/src/main/ipc/runtime-environment-request-connections.ts index c2fdfbe2e13..1ccb3b7df76 100644 --- a/src/main/ipc/runtime-environment-request-connections.ts +++ b/src/main/ipc/runtime-environment-request-connections.ts @@ -11,6 +11,7 @@ import type { } from '../../shared/runtime-rpc-envelope' import { RemoteRuntimeRequestConnection } from '../../shared/remote-runtime-request-connection' import { RemoteRuntimeSharedControlConnection } from '../../shared/remote-runtime-shared-control-connection' +import { probeAllRemoteRuntimeSocketsNow } from '../../shared/remote-runtime-socket-liveness' import type { RemoteRuntimeSharedConnectionDiagnostics, RemoteRuntimeSharedSubscription @@ -204,6 +205,9 @@ export function retryRemoteRuntimeSharedControlConnectionsNow(): void { for (const { connection } of sharedControlConnections.values()) { connection.retryNow() } + // Why: a socket that still reads OPEN after wake arms no reconnect timer, so retryNow finds + // nothing; probing every socket (shared control and per-stream) settles a dead path quickly (#9092). + probeAllRemoteRuntimeSocketsNow() } function getSharedControlConnection( diff --git a/src/main/ipc/runtime-environment-retry-connections-probe.test.ts b/src/main/ipc/runtime-environment-retry-connections-probe.test.ts new file mode 100644 index 00000000000..39789556ddf --- /dev/null +++ b/src/main/ipc/runtime-environment-retry-connections-probe.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it, vi } from 'vitest' +import type * as SocketLiveness from '../../shared/remote-runtime-socket-liveness' + +const probeAllRemoteRuntimeSocketsNow = vi.hoisted(() => vi.fn(() => 0)) + +vi.mock('../../shared/remote-runtime-socket-liveness', async (importOriginal) => ({ + ...(await importOriginal()), + probeAllRemoteRuntimeSocketsNow +})) + +import { retryRemoteRuntimeSharedControlConnectionsNow } from './runtime-environment-request-connections' + +// Regression for #9092: the resume/online IPC must probe sockets that still read OPEN after wake. +describe('retryRemoteRuntimeSharedControlConnectionsNow', () => { + it('probes every remote runtime socket even when no reconnect is pending', () => { + retryRemoteRuntimeSharedControlConnectionsNow() + + expect(probeAllRemoteRuntimeSocketsNow).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts index 32987b47702..ba29a4eb787 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts @@ -434,4 +434,43 @@ describe('remote runtime pty reattach after the bounded recovery window', () => vi.useRealTimers() } }) + + // #9092: a rebound subscribe still opening its stream when a second deadline latches must stay + // revivable by system resume / network online instead of needing a manual Reconnect. + it('parks a rebound subscribe whose stream open fails after its deadline', async () => { + vi.useFakeTimers() + try { + const { transport } = await attachStalePane() + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + const { retryAllRemoteRuntimePtyRecoveriesNow } = + await import('./remote-runtime-pty-recovery-state') + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 6_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + + // The stream open for the rebound handle hangs through the whole new recovery window. + const streamOpen: { reject: ((error: Error) => void) | null } = { reject: null } + runtimeSubscribe.mockImplementation( + () => + new Promise((_resolve, reject) => { + streamOpen.reject = reject + }) + ) + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot('terminal-rotated', 3, 'epoch-2'), + 'env-1' + ) + await vi.waitFor(() => expect(streamOpen.reject).not.toBeNull()) + expect(transport.getRecoveryState?.().phase).toBe('recovering') + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + + streamOpen.reject?.(new Error('Could not connect to the remote Orca runtime.')) + await vi.advanceTimersByTimeAsync(0) + + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1) + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) }) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts index d226ca109dd..bd40493e3b7 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts @@ -38,6 +38,44 @@ export function retryAllRemoteRuntimePtyRecoveriesNow(): number { return advanced } +// Why: a route that returns without an OS `online` event (Tailscale, VPN, a re-keyed link) fires no +// trigger, so a pane latched past its window stayed dead over a working link. The environment's +// control socket answering again is positive reachability evidence, as good as a Reconnect click. +const controlReadyRevivers = new Map boolean>>() + +export function registerRemoteRuntimePtyControlReadyReviver( + environmentId: string, + revive: () => boolean +): () => void { + let revivers = controlReadyRevivers.get(environmentId) + if (!revivers) { + revivers = new Set() + controlReadyRevivers.set(environmentId, revivers) + } + revivers.add(revive) + return () => { + const current = controlReadyRevivers.get(environmentId) + current?.delete(revive) + if (current?.size === 0) { + controlReadyRevivers.delete(environmentId) + } + } +} + +export function getControlReadyReviverCountForTests(environmentId: string): number { + return controlReadyRevivers.get(environmentId)?.size ?? 0 +} + +export function reviveRemoteRuntimePtyRecoveriesForEnvironment(environmentId: string): number { + let revived = 0 + for (const revive of Array.from(controlReadyRevivers.get(environmentId) ?? [])) { + if (revive()) { + revived += 1 + } + } + return revived +} + export class RemoteRuntimePtyRecoveryState { private phase: RemoteRuntimePtyRecoveryPhase = 'idle' private epoch = 0 diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts index 2e57fc57930..7ab12c647ec 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts @@ -572,6 +572,68 @@ describe('createRemoteRuntimePtyTransport', () => { } }) + // #25783/#9092: a route returning without `online` (Tailscale, VPN) left a latched pane dead. + it('revives a pane latched past its window when its environment answers again', async () => { + vi.useFakeTimers() + try { + let partitioned = false + runtimeSubscribe.mockImplementation( + async (_args: unknown, callbacks: NonNullable) => { + if (partitioned) { + throw Object.assign(new Error('Could not connect to the remote Orca runtime.'), { + code: 'remote_runtime_unavailable' + }) + } + subscriptionCallbacks = callbacks + queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } })) + return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary } + } + ) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const { + getControlReadyReviverCountForTests, + reviveRemoteRuntimePtyRecoveriesForEnvironment + } = await import('./remote-runtime-pty-recovery-state') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + transport.attach({ existingPtyId: 'remote:env-1@@terminal-1', callbacks: {} }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + emitSnapshot(latestSubscribePayload().streamId, 'before partition') + expect(getControlReadyReviverCountForTests('env-1')).toBe(0) + + partitioned = true + subscriptionCallbacks?.onClose?.() + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 60_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + const callsAtCutoff = runtimeSubscribe.mock.calls.length + + partitioned = false + expect(reviveRemoteRuntimePtyRecoveriesForEnvironment('env-2')).toBe(0) + expect(reviveRemoteRuntimePtyRecoveriesForEnvironment('env-1')).toBe(1) + await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(callsAtCutoff + 1)) + await vi.waitFor(() => expect(subscribeFrameCount()).toBe(2)) + emitSnapshot(latestSubscribePayload().streamId, 'after the route returned') + await vi.advanceTimersByTimeAsync(50) + expect(transport.isConnected()).toBe(true) + expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1') + // Why: a healthy pane holds no registration, so a detached-and-discarded transport is never retained. + expect(getControlReadyReviverCountForTests('env-1')).toBe(0) + + partitioned = true + subscriptionCallbacks?.onClose?.() + await vi.advanceTimersByTimeAsync(0) + expect(getControlReadyReviverCountForTests('env-1')).toBe(1) + transport.detach?.() + expect(getControlReadyReviverCountForTests('env-1')).toBe(0) + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + it('releases pending claimed input when reconnect subscription fails', async () => { const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') const onError = vi.fn() diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index c78232b3331..2e7da5b0073 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -72,6 +72,7 @@ import { import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS, RemoteRuntimePtyRecoveryState, + registerRemoteRuntimePtyControlReadyReviver, type RemoteRuntimePtyRecoveryPhase } from './remote-runtime-pty-recovery-state' import { createBrowserUuid } from '@/lib/browser-uuid' @@ -324,7 +325,23 @@ export function createRemoteRuntimePtyTransport( }) const disconnectedInputGrace = createRemoteRuntimeDisconnectedInputGrace() let previousRecoveryPhase: RemoteRuntimePtyRecoveryPhase = 'idle' + let unregisterControlReadyReviver: (() => void) | null = null + // Why registered only while recovering: detach and destroy go idle, so a discarded transport is never retained. + function syncControlReadyReviver(): void { + const wantsRevival = recovery.isActive || recovery.currentPhase === 'disconnected' + if (wantsRevival && !unregisterControlReadyReviver) { + // Why: a backoff advances and a latched pane replays its attach, exactly as online/resume/Reconnect would. + unregisterControlReadyReviver = registerRemoteRuntimePtyControlReadyReviver( + runtimeEnvironmentId, + () => !destroyed && (recovery.retryNow() || (transport.retryRecovery?.() ?? false)) + ) + } else if (!wantsRevival && unregisterControlReadyReviver) { + unregisterControlReadyReviver() + unregisterControlReadyReviver = null + } + } const recovery = new RemoteRuntimePtyRecoveryState(() => { + syncControlReadyReviver() // Why: any recovery transition after the ack (a stale send, a close, the deadline) re-holds typing. inputReadySubscription = null const leftLatchForRetry = previousRecoveryPhase === 'disconnected' && recovery.isActive @@ -2137,12 +2154,22 @@ export function createRemoteRuntimePtyTransport( return false } if (recovery.currentPhase === 'disconnected') { + parkResubscribeAfterDeadline(targetHandle) return true } scheduleResubscribeAfterTransportClose() return true } + // Why: a recoverable failure that lands after the auto-recovery deadline latched has no live epoch + // to join; without a parked retry, resume/online have nothing to fire and the pane stays + // disconnected until the user clicks Reconnect (#9092). Mirrors the connect path's park. + function parkResubscribeAfterDeadline(targetHandle: string): void { + recovery.parkRetryAfterDeadline((nextEpoch) => + scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(targetHandle), nextEpoch) + ) + } + // Why: after a transport drop the host may have re-minted this handle; re-derive from the snapshot so we don't mirror/type into whatever PTY now sits behind the stale one (#7718). async function resubscribeAfterTransportClose( previousHandle: string, @@ -2300,6 +2327,17 @@ export function createRemoteRuntimePtyTransport( let retryScheduled = false void resubscribeAfterTransportClose(resubscribeHandle, replacementPolicy, recoveryEpoch) .catch((error) => { + if ( + !destroyed && + connected && + handle && + recovery.currentPhase === 'disconnected' && + recovery.ownsEpoch(recoveryEpoch) && + isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error)) + ) { + parkResubscribeAfterDeadline(handle) + return + } if (!destroyed && connected && handle && recovery.isCurrent(recoveryEpoch)) { clearPendingViewportClaim() if (isRecoverablePaneBindingError(error)) { @@ -2605,6 +2643,17 @@ export function createRemoteRuntimePtyTransport( remotePtyId !== subscribedPtyId ) { nextStream.close() + // Why: a stream that opened after the deadline latched is discarded like its late failure would be, + // so park it too or resume/online find nothing to revive (#9092). + if ( + !destroyed && + connected && + handle === subscribedHandle && + recovery.currentPhase === 'disconnected' && + (expectedRecoveryEpoch === undefined || recovery.ownsEpoch(expectedRecoveryEpoch)) + ) { + parkResubscribeAfterDeadline(subscribedHandle) + } return } closeMultiplexedStream() diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-resubscribe-after-deadline.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-resubscribe-after-deadline.test.ts new file mode 100644 index 00000000000..a83c29cd826 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-runtime-resubscribe-after-deadline.test.ts @@ -0,0 +1,186 @@ +/** + * Regression for #9092: a remote pane whose resubscribe attempt was still in flight when the + * auto-recovery deadline latched dropped the late (recoverable) failure, so nothing was parked + * and system resume / network online had nothing to fire. The pane stayed disconnected until + * the user clicked Reconnect, even after the host came back. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + encodeTerminalStreamFrame, + encodeTerminalStreamJson, + encodeTerminalStreamText +} from '../../../../shared/terminal-stream-protocol' +import { REMOTE_RUNTIME_CONNECT_FAILURE_PHRASE } from '../../../../shared/remote-runtime-connect-bound' +import { REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS } from './remote-runtime-pty-recovery-state' + +const FIRST_HANDLE = 'terminal-1' +const FIRST_PTY_ID = 'remote:env-1@@terminal-1' + +describe('remote runtime resubscribe failing after the recovery deadline', () => { + const subscriptionSendBinary = vi.fn() + let subscriptionCallbacks: { + onResponse: (response: unknown) => void + onBinary?: (bytes: Uint8Array) => void + onClose?: () => void + } | null = null + let resolvePaneCalls = 0 + let holdResolvePane: { reject: (error: Error) => void } | null = null + let hangLateResolvePane = true + let streamOpens = 0 + const lateStreamOpen: { hang: boolean; release: (() => void) | null } = { + hang: false, + release: null + } + + function latestStreamId(): number { + const frame = subscriptionSendBinary.mock.calls + .map((call) => decodeTerminalStreamFrame(call[0])) + .findLast((candidate) => candidate?.opcode === TerminalStreamOpcode.Subscribe) + const payload = frame ? decodeTerminalStreamJson<{ streamId: number }>(frame.payload) : null + if (!payload) { + throw new Error('missing terminal subscribe frame') + } + return payload.streamId + } + + function emitSnapshot(streamId: number): void { + const frames = [ + { opcode: TerminalStreamOpcode.SnapshotStart, payload: encodeTerminalStreamJson({}) }, + { opcode: TerminalStreamOpcode.SnapshotChunk, payload: encodeTerminalStreamText('live') }, + { opcode: TerminalStreamOpcode.SnapshotEnd, payload: new Uint8Array() } + ] + frames.forEach((frame, index) => + subscriptionCallbacks?.onBinary?.( + encodeTerminalStreamFrame({ ...frame, streamId, seq: index + 1 }) + ) + ) + } + + beforeEach(() => { + vi.resetModules() + vi.doMock('@/runtime/web-runtime-session', () => ({ + refreshWebRuntimeSessionTabsSnapshot: vi.fn(async () => {}) + })) + subscriptionCallbacks = null + subscriptionSendBinary.mockReset() + resolvePaneCalls = 0 + holdResolvePane = null + hangLateResolvePane = true + streamOpens = 0 + lateStreamOpen.hang = false + lateStreamOpen.release = null + const runtimeCall = vi.fn(async (request: { method: string; params?: unknown }) => { + if (request.method === 'terminal.resolvePane') { + resolvePaneCalls += 1 + const params = request.params as { paneKey: string; worktreeId: string } + if (hangLateResolvePane && holdResolvePane === null && resolvePaneCalls > 1) { + // Why: the post-deadline attempt hangs so the test observes that it was issued. + return new Promise((_resolve, reject) => { + holdResolvePane = { reject } + }) + } + return { + ok: true, + result: { + terminal: { + handle: FIRST_HANDLE, + tabId: 'tab-1', + leafId: 'pane:1', + worktreeId: params.worktreeId + } + } + } + } + return { ok: true, result: { terminal: { handle: FIRST_HANDLE } } } + }) + const runtimeSubscribe = vi.fn( + async (_args: unknown, callbacks: typeof subscriptionCallbacks) => { + streamOpens += 1 + if (lateStreamOpen.hang && streamOpens > 1) { + await new Promise((resolve) => { + lateStreamOpen.release = resolve + }) + } + subscriptionCallbacks = callbacks + queueMicrotask(() => + subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'ready' } }) + ) + return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary } + } + ) + vi.stubGlobal('window', { + api: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } } + }) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('parks the late failure so a resume trigger can revive the pane', async () => { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const { retryAllRemoteRuntimePtyRecoveriesNow } = + await import('./remote-runtime-pty-recovery-state') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + transport.attach({ existingPtyId: FIRST_PTY_ID, cols: 80, rows: 24, callbacks: {} }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + emitSnapshot(latestStreamId()) + expect(transport.isConnected()).toBe(true) + + vi.useFakeTimers() + subscriptionCallbacks?.onClose?.() + await vi.advanceTimersByTimeAsync(0) + await vi.waitFor(() => expect(holdResolvePane).not.toBeNull()) + + // The host stays unreachable past the whole auto-recovery window. + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + holdResolvePane?.reject(new Error(`${REMOTE_RUNTIME_CONNECT_FAILURE_PHRASE}.`)) + holdResolvePane = null + await vi.advanceTimersByTimeAsync(0) + + const callsBeforeResume = resolvePaneCalls + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1) + await vi.advanceTimersByTimeAsync(0) + expect(resolvePaneCalls).toBeGreaterThan(callsBeforeResume) + transport.destroy?.() + }) + + it('parks a late success whose stream opened after the deadline', async () => { + hangLateResolvePane = false + lateStreamOpen.hang = true + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const { retryAllRemoteRuntimePtyRecoveriesNow } = + await import('./remote-runtime-pty-recovery-state') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + transport.attach({ existingPtyId: FIRST_PTY_ID, cols: 80, rows: 24, callbacks: {} }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + emitSnapshot(latestStreamId()) + + vi.useFakeTimers() + subscriptionCallbacks?.onClose?.() + await vi.advanceTimersByTimeAsync(0) + await vi.waitFor(() => expect(lateStreamOpen.release).not.toBeNull()) + + // The stream open stays in flight past the whole window, then succeeds. + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 1) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + lateStreamOpen.release?.() + await vi.advanceTimersByTimeAsync(0) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1) + transport.destroy?.() + }) +}) diff --git a/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.test.ts b/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.test.ts index 15956a10665..91f2365211d 100644 --- a/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.test.ts +++ b/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.test.ts @@ -2,14 +2,18 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { renderHook } from '@testing-library/react' -const { retryAllRemoteRuntimePtyRecoveriesNowMock } = vi.hoisted(() => ({ - retryAllRemoteRuntimePtyRecoveriesNowMock: vi.fn() +const { retryAllRemoteRuntimePtyRecoveriesNowMock, reviveForEnvironmentMock } = vi.hoisted(() => ({ + retryAllRemoteRuntimePtyRecoveriesNowMock: vi.fn(), + reviveForEnvironmentMock: vi.fn() })) vi.mock('@/components/terminal-pane/remote-runtime-pty-recovery-state', () => ({ - retryAllRemoteRuntimePtyRecoveriesNow: retryAllRemoteRuntimePtyRecoveriesNowMock + retryAllRemoteRuntimePtyRecoveriesNow: retryAllRemoteRuntimePtyRecoveriesNowMock, + reviveRemoteRuntimePtyRecoveriesForEnvironment: reviveForEnvironmentMock })) +type DiagnosticsEvent = { environmentId: string; diagnostics: { state: string } } + import { useRemoteRuntimeRecoveryTriggers } from './use-remote-runtime-recovery-triggers' describe('useRemoteRuntimeRecoveryTriggers', () => { @@ -20,6 +24,14 @@ describe('useRemoteRuntimeRecoveryTriggers', () => { return unsubscribeSystemResumed }) const retryConnectionsNow = vi.fn(() => Promise.resolve()) + let diagnosticsCallback: ((event: DiagnosticsEvent) => void) | null = null + const unsubscribeDiagnostics = vi.fn() + const onSharedControlDiagnostics = vi.fn((callback: (event: DiagnosticsEvent) => void) => { + diagnosticsCallback = callback + return unsubscribeDiagnostics + }) + const emitState = (environmentId: string, state: string): void => + diagnosticsCallback?.({ environmentId, diagnostics: { state } }) beforeEach(() => { systemResumedCallback = null @@ -27,9 +39,12 @@ describe('useRemoteRuntimeRecoveryTriggers', () => { onSystemResumed.mockClear() retryConnectionsNow.mockClear() retryAllRemoteRuntimePtyRecoveriesNowMock.mockClear() + reviveForEnvironmentMock.mockClear() + diagnosticsCallback = null + unsubscribeDiagnostics.mockClear() ;(window as unknown as { api: unknown }).api = { ui: { onSystemResumed }, - runtimeEnvironments: { retryConnectionsNow } + runtimeEnvironments: { retryConnectionsNow, onSharedControlDiagnostics } } }) @@ -60,4 +75,22 @@ describe('useRemoteRuntimeRecoveryTriggers', () => { expect(retryAllRemoteRuntimePtyRecoveriesNowMock).not.toHaveBeenCalled() expect(unsubscribeSystemResumed).toHaveBeenCalledTimes(1) }) + + // #9092/#25783: a returning Tailscale or VPN route fires no `online`, so the control socket is the signal. + it("revives an environment's panes when its control connection becomes ready again", () => { + const { unmount } = renderHook(() => useRemoteRuntimeRecoveryTriggers()) + + emitState('env-1', 'ready') + emitState('env-1', 'ready') + expect(reviveForEnvironmentMock.mock.calls).toEqual([['env-1']]) + + emitState('env-1', 'reconnecting') + emitState('env-2', 'connecting') + expect(reviveForEnvironmentMock).toHaveBeenCalledTimes(1) + + emitState('env-1', 'ready') + expect(reviveForEnvironmentMock.mock.calls).toEqual([['env-1'], ['env-1']]) + unmount() + expect(unsubscribeDiagnostics).toHaveBeenCalledTimes(1) + }) }) diff --git a/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.ts b/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.ts index 0ecd493b2ae..32c04d9de95 100644 --- a/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.ts +++ b/src/renderer/src/runtime/use-remote-runtime-recovery-triggers.ts @@ -1,5 +1,8 @@ import { useEffect } from 'react' -import { retryAllRemoteRuntimePtyRecoveriesNow } from '@/components/terminal-pane/remote-runtime-pty-recovery-state' +import { + retryAllRemoteRuntimePtyRecoveriesNow, + reviveRemoteRuntimePtyRecoveriesForEnvironment +} from '@/components/terminal-pane/remote-runtime-pty-recovery-state' export function useRemoteRuntimeRecoveryTriggers(): void { useEffect(() => { @@ -13,9 +16,24 @@ export function useRemoteRuntimeRecoveryTriggers(): void { typeof window.api?.ui?.onSystemResumed === 'function' ? window.api.ui.onSystemResumed(advanceRemoteRuntimeRecoveryBackoffs) : null + const readyByEnvironment = new Map() + const unsubscribeControlDiagnostics = + typeof window.api?.runtimeEnvironments?.onSharedControlDiagnostics === 'function' + ? window.api.runtimeEnvironments.onSharedControlDiagnostics( + ({ environmentId, diagnostics }) => { + const ready = diagnostics.state === 'ready' + const wasReady = readyByEnvironment.get(environmentId) ?? false + readyByEnvironment.set(environmentId, ready) + if (ready && !wasReady) { + reviveRemoteRuntimePtyRecoveriesForEnvironment(environmentId) + } + } + ) + : null return () => { window.removeEventListener('online', advanceRemoteRuntimeRecoveryBackoffs) unsubscribeSystemResumed?.() + unsubscribeControlDiagnostics?.() } }, []) } diff --git a/src/shared/remote-runtime-passthrough-socket.ts b/src/shared/remote-runtime-passthrough-socket.ts index 569b31dafe7..16c37f201e5 100644 --- a/src/shared/remote-runtime-passthrough-socket.ts +++ b/src/shared/remote-runtime-passthrough-socket.ts @@ -80,9 +80,11 @@ export function openRemoteRuntimePassthroughSocket( ) const liveness = startRemoteRuntimeSocketLiveness({ ping: () => { - if (ws.readyState === WebSocket.OPEN) { - ws.ping() + if (ws.readyState !== WebSocket.OPEN) { + return false } + ws.ping() + return true }, onDead: () => end( diff --git a/src/shared/remote-runtime-shared-control-open.ts b/src/shared/remote-runtime-shared-control-open.ts index 56961236a2b..db6d0f174cb 100644 --- a/src/shared/remote-runtime-shared-control-open.ts +++ b/src/shared/remote-runtime-shared-control-open.ts @@ -74,9 +74,11 @@ export function openSharedControlSocket( const liveness = callbacks.liveness const monitor = startRemoteRuntimeSocketLiveness({ ping: () => { - if (ws.readyState === 1) { - ws.ping() + if (ws.readyState !== 1) { + return false } + ws.ping() + return true }, onDead: () => { if (callbacks.getCurrentSocket() !== ws) { diff --git a/src/shared/remote-runtime-shared-control-reconnect.ts b/src/shared/remote-runtime-shared-control-reconnect.ts index 6704db6cdcd..fd27efb6316 100644 --- a/src/shared/remote-runtime-shared-control-reconnect.ts +++ b/src/shared/remote-runtime-shared-control-reconnect.ts @@ -75,6 +75,10 @@ export class SharedControlReconnectScheduler { // Why: OS resume / browser online should advance an already-scheduled reconnect, not start a new one. retryNow(): boolean { + // Why: a wake or network change is fresh evidence, so the next failure starts again from the short + // delays instead of the 5-minute idle cap earned during the outage — also when an open is already + // in flight and no timer is pending (#9092). + this.attempt = 0 if (!this.timer || !this.pendingOpen) { return false } diff --git a/src/shared/remote-runtime-socket-liveness.ts b/src/shared/remote-runtime-socket-liveness.ts index 8a2355a8286..30572fd0042 100644 --- a/src/shared/remote-runtime-socket-liveness.ts +++ b/src/shared/remote-runtime-socket-liveness.ts @@ -18,13 +18,33 @@ export type RemoteRuntimeSocketLivenessOptions = { livenessTimeoutMs?: number } +// Why: after sleep or a network change a socket can read OPEN while its path is gone; the normal +// cadence needs ~30-40 s to prove that. A resume probe settles it in one short, explicit window. +export const REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS = 8_000 + export type RemoteRuntimeSocketLivenessMonitor = { noteActivity: () => void + /** Ping now; declare the socket dead unless anything arrives within `deadlineMs`. */ + probeNow: (deadlineMs?: number) => void stop: () => void } +const activeMonitors = new Set() + +/** Probe every live remote-runtime socket in this process, e.g. on OS resume or network change. */ +export function probeAllRemoteRuntimeSocketsNow( + deadlineMs = REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS +): number { + const monitors = Array.from(activeMonitors) + for (const monitor of monitors) { + monitor.probeNow(deadlineMs) + } + return monitors.length +} + export function startRemoteRuntimeSocketLiveness(args: { - ping: () => void + /** Returns false when nothing was sent, e.g. the socket is still connecting. */ + ping: () => boolean onDead: () => void options?: RemoteRuntimeSocketLivenessOptions now?: () => number @@ -35,6 +55,8 @@ export function startRemoteRuntimeSocketLiveness(args: { args.options?.livenessTimeoutMs ?? REMOTE_RUNTIME_SOCKET_LIVENESS_TIMEOUT_MS let lastTickAt = now() let probeSentAt: number | null = null + let activitySinceResumeProbe = false + let resumeProbeTimer: ReturnType | null = null let stopped = false const timer = setInterval(() => { @@ -72,20 +94,64 @@ export function startRemoteRuntimeSocketLiveness(args: { } stopped = true clearInterval(timer) + if (resumeProbeTimer !== null) { + clearTimeout(resumeProbeTimer) + resumeProbeTimer = null + } + activeMonitors.delete(monitor) } - function tryPing(): void { - try { - args.ping() - } catch { - // Why: ping() can throw while a socket is mid-teardown; the probe deadline still settles it. + function probeNow(deadlineMs = REMOTE_RUNTIME_SOCKET_RESUME_PROBE_DEADLINE_MS): void { + if (stopped || resumeProbeTimer !== null) { + return + } + activitySinceResumeProbe = false + // Why: a socket still connecting has no path to probe yet; a deadline with no ping could kill it mid-handshake. + if (!tryPing()) { + return + } + const probedAt = now() + resumeProbeTimer = setTimeout(() => { + resumeProbeTimer = null + if (stopped || activitySinceResumeProbe) { + return + } + // Why: a late timer means this process slept or stalled, so replies may still be queued; probe afresh. + const elapsedMs = now() - probedAt + if (elapsedMs < 0 || elapsedMs > deadlineMs * 1.5) { + probeNow(deadlineMs) + return + } + stop() + args.onDead() + }, deadlineMs) + // Why: mobile typechecks shared code with DOM timer types where unref is absent. + if ( + typeof resumeProbeTimer === 'object' && + 'unref' in resumeProbeTimer && + typeof resumeProbeTimer.unref === 'function' + ) { + resumeProbeTimer.unref() } } - return { + function tryPing(): boolean { + try { + return args.ping() + } catch { + // Why: ping() can throw while a socket is mid-teardown; the probe deadline still settles it. + return true + } + } + + const monitor: RemoteRuntimeSocketLivenessMonitor = { noteActivity: () => { probeSentAt = null + activitySinceResumeProbe = true }, + probeNow, stop } + activeMonitors.add(monitor) + return monitor } diff --git a/src/shared/remote-runtime-socket-resume-probe.test.ts b/src/shared/remote-runtime-socket-resume-probe.test.ts new file mode 100644 index 00000000000..0bc3ff5751d --- /dev/null +++ b/src/shared/remote-runtime-socket-resume-probe.test.ts @@ -0,0 +1,149 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + probeAllRemoteRuntimeSocketsNow, + startRemoteRuntimeSocketLiveness +} from './remote-runtime-socket-liveness' +import { SharedControlReconnectScheduler } from './remote-runtime-shared-control-reconnect' + +// Regression for #9092: after sleep a socket can still read OPEN while its path is gone, and the +// normal cadence took ~30-40 s to notice; resume now probes every socket with a short deadline. +describe('remote runtime resume probe', () => { + const monitors: { stop: () => void }[] = [] + + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + for (const monitor of monitors.splice(0)) { + monitor.stop() + } + vi.useRealTimers() + }) + + function start(onDead = vi.fn(), ping = vi.fn(() => true)) { + const monitor = startRemoteRuntimeSocketLiveness({ + ping, + onDead, + options: { pingIntervalMs: 10_000, livenessTimeoutMs: 25_000 } + }) + monitors.push(monitor) + return { monitor, onDead, ping } + } + + it('declares a silent socket dead within the resume deadline', async () => { + const { onDead, ping } = start() + + expect(probeAllRemoteRuntimeSocketsNow(2_000)).toBe(1) + expect(ping).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1_999) + expect(onDead).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(onDead).toHaveBeenCalledTimes(1) + }) + + it('probes afresh instead of failing a socket when the process stalled through the deadline', async () => { + let clock = 0 + const onDead = vi.fn() + const ping = vi.fn(() => true) + monitors.push( + startRemoteRuntimeSocketLiveness({ + ping, + onDead, + now: () => clock, + options: { pingIntervalMs: 60_000, livenessTimeoutMs: 120_000 } + }) + ) + + probeAllRemoteRuntimeSocketsNow(2_000) + clock += 20_000 + await vi.advanceTimersByTimeAsync(2_000) + expect(onDead).not.toHaveBeenCalled() + expect(ping).toHaveBeenCalledTimes(2) + + clock += 2_000 + await vi.advanceTimersByTimeAsync(2_000) + expect(onDead).toHaveBeenCalledTimes(1) + }) + + it('keeps a socket that answers the probe', async () => { + const { monitor, onDead } = start() + + probeAllRemoteRuntimeSocketsNow(2_000) + await vi.advanceTimersByTimeAsync(500) + monitor.noteActivity() + await vi.advanceTimersByTimeAsync(5_000) + expect(onDead).not.toHaveBeenCalled() + }) + + it('skips stopped monitors and coalesces a repeated resume into one probe', async () => { + const stopped = start() + stopped.monitor.stop() + const live = start() + + expect(probeAllRemoteRuntimeSocketsNow(2_000)).toBe(1) + probeAllRemoteRuntimeSocketsNow(2_000) + expect(stopped.ping).not.toHaveBeenCalled() + expect(live.ping).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(2_000) + expect(live.onDead).toHaveBeenCalledTimes(1) + expect(stopped.onDead).not.toHaveBeenCalled() + }) + + it('does not arm a deadline for a socket that is still connecting', async () => { + let open = false + const ping = vi.fn(() => open) + const { onDead } = start(vi.fn(), ping) + + probeAllRemoteRuntimeSocketsNow(2_000) + await vi.advanceTimersByTimeAsync(2_000) + expect(onDead).not.toHaveBeenCalled() + + // Once open, the next resume probes it normally. + open = true + probeAllRemoteRuntimeSocketsNow(2_000) + await vi.advanceTimersByTimeAsync(2_000) + expect(onDead).toHaveBeenCalledTimes(1) + }) +}) + +describe('SharedControlReconnectScheduler.retryNow after an outage', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('restarts the backoff ladder so a failure right after wake retries quickly', () => { + const scheduler = new SharedControlReconnectScheduler() + const open = vi.fn() + for (let attempt = 0; attempt < 6; attempt += 1) { + scheduler.scheduleWithIdleBackoff(false, open) + vi.runOnlyPendingTimers() + } + expect(scheduler.attemptCount).toBeGreaterThanOrEqual(6) + + scheduler.scheduleWithIdleBackoff(false, open) + expect(scheduler.retryNow()).toBe(true) + expect(scheduler.attemptCount).toBe(0) + }) + + it('restarts the ladder even when the wake lands while a reconnect is already opening', () => { + const scheduler = new SharedControlReconnectScheduler() + const open = vi.fn() + for (let attempt = 0; attempt < 6; attempt += 1) { + scheduler.scheduleWithIdleBackoff(false, open) + vi.runOnlyPendingTimers() + } + // The last open() is in flight, so no timer is pending for retryNow to advance. + expect(scheduler.isScheduled).toBe(false) + expect(scheduler.retryNow()).toBe(false) + expect(scheduler.attemptCount).toBe(0) + }) + + it('does nothing when no reconnect is pending', () => { + const scheduler = new SharedControlReconnectScheduler() + expect(scheduler.retryNow()).toBe(false) + expect(scheduler.attemptCount).toBe(0) + }) +}) diff --git a/src/shared/remote-runtime-subscription-transport.ts b/src/shared/remote-runtime-subscription-transport.ts index 74dfa46d354..4c4be5cfcb1 100644 --- a/src/shared/remote-runtime-subscription-transport.ts +++ b/src/shared/remote-runtime-subscription-transport.ts @@ -282,9 +282,11 @@ export async function subscribeRemoteRuntimeTransport( const monitoredWs = ws liveness = startRemoteRuntimeSocketLiveness({ ping: () => { - if (monitoredWs.readyState === WebSocket.OPEN) { - monitoredWs.ping() + if (monitoredWs.readyState !== WebSocket.OPEN) { + return false } + monitoredWs.ping() + return true }, onDead: () => { fail( diff --git a/tests/e2e/paired-remote-terminal-network-outage.spec.ts b/tests/e2e/paired-remote-terminal-network-outage.spec.ts new file mode 100644 index 00000000000..caa33e1c313 --- /dev/null +++ b/tests/e2e/paired-remote-terminal-network-outage.spec.ts @@ -0,0 +1,254 @@ +/** + * A paired client's open remote terminal must answer typing again on its own after the network to + * its host goes away for 30 s (a lid close, a Tailscale re-key, a subway tunnel) and comes back. + * Neither process restarts and nobody clicks Reconnect. + * + * Run: + * ORCA_BACKGROUND_LAUNCH=1 pnpm exec playwright test \ + * tests/e2e/paired-remote-terminal-network-outage.spec.ts \ + * --config tests/playwright.config.ts --project electron-headless --workers=1 + */ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id' +import { decodePairingOffer, encodePairingOffer } from '../../src/shared/pairing' +import { quotePosixShell } from '../../src/shared/wsl-login-shell-command' +import { expect, test } from './helpers/orca-app' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' +import { cleanupE2EDaemons } from './helpers/electron-process-shutdown' +import { + launchPairedElectronClient, + type PairedElectronClient +} from './helpers/paired-electron-client' +import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy' + +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-terminal-outage-')) +const fixturePath = path.join(scratch, 'outage-terminal.mjs') + +writeFileSync( + fixturePath, + [ + "import { appendFileSync } from 'node:fs'", + 'const sink = process.argv[2]', + "process.stdout.write('READY\\r\\n')", + "process.stdin.setEncoding('utf8')", + "let pending = ''", + "process.stdin.on('data', (data) => {", + ' pending += data', + ' const lines = pending.split(/\\r\\n|\\r|\\n/)', + " pending = lines.pop() ?? ''", + ' for (const line of lines) {', + ' appendFileSync(sink, `${line}\\n`)', + ' process.stdout.write(`LIVE:${line}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +function fixtureCommand(sinkPath: string): string { + const command = [process.execPath, fixturePath, sinkPath] + return process.platform === 'win32' + ? `& ${command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')}` + : command.map(quotePosixShell).join(' ') +} + +function readText(filePath: string): string { + try { + return readFileSync(filePath, 'utf8') + } catch { + return '' + } +} + +async function callRuntime( + page: Page, + environmentId: string, + method: string, + params: unknown +): Promise { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: test-only RPC result shape. + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params + }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { environmentId, method, params } + ) as Promise +} + +type PaneProbe = { content: string; ptyId: string | null; recovery: string | null } + +async function readPane(page: Page, webTabId: string): Promise { + return page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return { + content: pane?.serializeAddon?.serialize?.() ?? '', + ptyId: pane?.container.dataset.ptyId ?? null, + recovery: pane?.container.dataset.ptyRecoveryState ?? null + } + }, webTabId) +} + +async function typeIntoPane(page: Page, webTabId: string, text: string): Promise { + await page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + if (!pane) { + throw new Error(`No pane mounted for ${id}`) + } + pane.terminal.focus() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: xterm always renders its helper textarea. + const textarea = pane.container.querySelector('.xterm-helper-textarea') as HTMLTextAreaElement + textarea.focus() + }, webTabId) + await page.keyboard.type(text) + await page.keyboard.press('Enter') +} + +const OUTAGES = [ + { mode: 'silent', ms: 30_000, label: '30 s' }, + { mode: 'reset', ms: 30_000, label: '30 s' }, + // Why: outlasts the pane's ~3 min auto-recovery window, with no `online` event when it returns. + { mode: 'silent', ms: 200_000, label: 'long' } +] as const + +for (const { mode, ms, label } of OUTAGES) { + test(`an open remote terminal answers typing again after a ${label} ${mode} network outage`, async ({ + testRepoPath + }, testInfo) => { + test.setTimeout(ms + 330_000) + const host = await launchHeadlessPairedRuntimeHost() + let client: PairedElectronClient | null = null + let proxy: FreezableTcpProxy | null = null + try { + await host.client.call('repo.add', { path: testRepoPath, kind: 'git' }) + const decoded = decodePairingOffer(host.offer.pairingUrl) + if (!('endpoint' in decoded)) { + throw new Error('expected a direct pairing offer') + } + const endpoint = new URL(decoded.endpoint) + proxy = await startFreezableTcpProxy(endpoint.hostname, Number(endpoint.port)) + endpoint.port = String(proxy.port) + client = await launchPairedElectronClient( + { pairingUrl: encodePairingOffer({ ...decoded, endpoint: endpoint.toString() }) }, + testInfo, + `terminal ${mode} outage` + ) + const page = client.page + const environmentId = client.environmentId + await expect + .poll(() => page.evaluate(() => window.__store?.getState().allWorktrees()[0]?.id ?? null), { + timeout: 60_000 + }) + .not.toBeNull() + const worktreeId = await page.evaluate(() => window.__store!.getState().allWorktrees()[0]!.id) + const sinkPath = path.join(scratch, `${mode}.log`) + const created = await callRuntime<{ tab: { parentTabId: string; terminal: string | null } }>( + page, + environmentId, + 'session.tabs.createTerminal', + { + worktree: `id:${worktreeId}`, + command: fixtureCommand(sinkPath), + activate: false, + select: false, + navigation: 'caller' + } + ) + expect(created.tab.terminal).toBeTruthy() + const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId) + await expect + .poll( + () => + page.evaluate( + ({ worktreeId, webTabId }) => + (window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some( + (tab) => tab.id === webTabId + ), + { worktreeId, webTabId } + ), + { timeout: 60_000 } + ) + .toBe(true) + await page.evaluate( + ({ worktreeId, webTabId }) => { + const state = window.__store!.getState() + state.setActiveView('terminal') + state.setActiveWorktree(worktreeId) + state.setActiveTab(webTabId) + state.setActiveTabType('terminal', worktreeId) + }, + { worktreeId, webTabId } + ) + await expect + .poll(async () => (await readPane(page, webTabId)).content, { timeout: 60_000 }) + .toContain('READY') + await typeIntoPane(page, webTabId, 'before') + await expect + .poll(async () => (await readPane(page, webTabId)).content, { timeout: 15_000 }) + .toContain('LIVE:before') + const ptyBefore = (await readPane(page, webTabId)).ptyId + + proxy.cut(mode) + await page.waitForTimeout(ms) + proxy.restore() + + const marker = `after_${mode}_${Date.now()}` + // Why retyped: typing during recovery may be dropped by design; the pane must answer once back. + try { + await expect + .poll( + async () => { + if (!readText(sinkPath).includes(marker)) { + await typeIntoPane(page, webTabId, marker) + } + return (await readPane(page, webTabId)).content + }, + { timeout: 120_000, intervals: [5_000] } + ) + .toContain(`LIVE:${marker}`) + } catch (error) { + const probe = await readPane(page, webTabId) + const evidence = { + ...probe, + content: probe.content.slice(-600), + ptyBefore, + sink: readText(sinkPath), + sharedControl: await page.evaluate( + async (id) => + (await window.api.runtimeEnvironments.list()).find((entry) => entry.id === id) ?? + null, + environmentId + ), + proxyEvents: proxy.events + } + console.log('[outage] evidence', JSON.stringify(evidence)) + await page.screenshot({ path: testInfo.outputPath('after-outage.png') }) + throw error + } + expect((await readPane(page, webTabId)).ptyId).toBe(ptyBefore) + } finally { + if (client) { + await cleanupE2EDaemons(client.userDataDir).catch(() => undefined) + await client.dispose() + } + await proxy?.close() + await host.dispose() + } + }) +} diff --git a/tests/e2e/ssh-orcad-terminal-network-outage.spec.ts b/tests/e2e/ssh-orcad-terminal-network-outage.spec.ts new file mode 100644 index 00000000000..120a5cbbb13 --- /dev/null +++ b/tests/e2e/ssh-orcad-terminal-network-outage.spec.ts @@ -0,0 +1,89 @@ +/** + * #25783 on the managed-server path: an open pane on an orcad-managed SSH host answers typing again + * on its own after the network to the host goes away for 30 s and comes back. Nobody clicks + * Reconnect, and the pane keeps its original shell. + */ +import type { Page } from '@stablyai/playwright-test' +import { expect, test } from './helpers/orca-app' +import { readPairedPaneContent } from './helpers/paired-host-terminal' +import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy' +import { + echoMarker, + expectPaneText, + expectTerminalAnswers, + launchManagedWorkspace, + openTerminalTab, + skipUnlessManagedDockerHost, + typeInTerminal, + withManagedHost +} from './helpers/orcad-managed-workspace' + +skipUnlessManagedDockerHost() +test.skip(process.platform === 'win32', 'The Docker SSH host uses POSIX tooling') + +const OUTAGES = [ + { mode: 'silent', ms: 30_000, label: '30 s' }, + { mode: 'reset', ms: 30_000, label: '30 s' } +] as const + +function sshStatus(page: Page, targetId: string): Promise { + return page.evaluate( + (id) => window.__store?.getState().sshConnectionStates.get(id)?.status ?? null, + targetId + ) +} + +for (const { mode, ms, label } of OUTAGES) { + test(`a managed pane answers typing again after a ${label} ${mode} network outage`, async (// oxlint-disable-next-line no-empty-pattern -- Owns its app launch through a restart session. + {}, testInfo) => { + test.setTimeout(ms + 10 * 60_000) + let proxy: FreezableTcpProxy | null = null + try { + await withManagedHost(testInfo, async (fixture) => { + const { host } = fixture + proxy = await startFreezableTcpProxy(host.input.host, host.input.port ?? 22) + // Why the proxy: the profile reaches sshd only through it, so cutting it is the network. + host.input = { ...host.input, port: proxy.port } + const { page, seeded } = await launchManagedWorkspace(fixture) + const tabId = await openTerminalTab(page, seeded.worktreeId) + // Quoted apart so the typed command line itself never matches. + await typeInTerminal(page, tabId, "echo SHELL''_PID=$$") + await expect + .poll(async () => /SHELL_PID=(\d+)/.exec(await readPairedPaneContent(page, tabId))?.[1]) + .toBeTruthy() + const shellPid = /SHELL_PID=(\d+)/.exec(await readPairedPaneContent(page, tabId))![1] + const marker = await echoMarker(page, tabId, 'BEFORE_OUTAGE') + + proxy.cut(mode) + await page.waitForTimeout(ms) + proxy.restore() + + try { + await expectTerminalAnswers(page, tabId, 180_000) + } catch (error) { + console.log( + '[outage] evidence', + JSON.stringify({ + sshStatus: await sshStatus(page, seeded.targetId), + pane: (await readPairedPaneContent(page, tabId)).slice(-600), + hostShellAlive: host.exec!(`kill -0 ${shellPid} && echo alive || echo gone`), + proxyEvents: proxy.events + }) + ) + await page.screenshot({ path: testInfo.outputPath('after-outage.png') }) + throw error + } + await expectPaneText(page, tabId, marker) + await typeInTerminal(page, tabId, "echo SHELL''_PID=$$") + await expect + .poll( + async () => + (await readPairedPaneContent(page, tabId)).split(`SHELL_PID=${shellPid}`).length + ) + .toBeGreaterThan(2) + }) + } finally { + await proxy?.close() + } + }) +}