From 1751433b2118fd15aaa1afc0e3b40fc0fb276674 Mon Sep 17 00:00:00 2001 From: Neil Date: Thu, 10 Sep 2026 02:41:24 -0700 Subject: [PATCH] fix(runtime): keep a live host verdict when a status probe can't reach it (#19647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A status.get dials its own fresh socket (sendRemoteRuntimeRequest), so on a half-open link it hangs and fails with the client-minted `runtime_unavailable` while established terminal/agent flows keep delivering. The status recheck ladder and the refresh wrapper caught that failure and published `status: null` over a recorded live verdict. That single write dropped the environment out of getReachableRuntimeSessionMirrorTargets (mirror teardown #1), and the next successful probe advanced the connection generation and rebuilt it (teardown #2) — one transient probe failure, two full session-tabs mirror teardowns, and the client demanding a reconnect while the agent kept working. Treat that failure as unverifiable, per docs/reference/ssh-execution-boundary.md: loss of contact is never evidence the host exited. A status.get that fails with `runtime_unavailable` (client-minted, never on the wire — safe under remote-wire-compatibility Rules 1 and 3) no longer overwrites a recorded live verdict. The live status stays, the mirror is never retired, the connection generation never bumps, and the recheck ladder keeps probing until a real answer. First-contact failures (no prior live verdict) still record null so host coverage completes. Carries forward the reader-side fixes from #19163 (filed against #16516): the recheck ladder re-asks a status: null host on its backoff and cancels on manual disconnect, and NoticeHostGlyph / use-worktree-card-foundation use the shared runtimeHostConnectionState derivation so a never-probed host is not painted the destructive red of one a probe found unreachable. Measured with a new paired-client e2e rig (stall-new TCP hop): baseline nulls the status and bumps the generation 0->1 on a link flap; fixed keeps nullStatus=false and the generation stable. Two adjacent defects found while measuring are filed separately: #19871 (multiplexer socket dropped on last-stream close) and #19872 (un-park banner claims auto-retry while parkRetry arms no timer). Co-authored-by: Omar Shahine <10343873+omarshahine@users.noreply.github.com> Co-authored-by: ASDFGHoney <86656987+ASDFGHoney@users.noreply.github.com> --- .../sidebar/NoticeHostGlyph.test.tsx | 14 +- .../components/sidebar/NoticeHostGlyph.tsx | 10 +- ...WorktreeCard.ssh-reconnect-prompt.test.tsx | 23 +- .../sidebar/use-worktree-card-foundation.ts | 13 +- .../runtime-host-connection-state.test.ts | 54 ++ .../runtime/runtime-host-connection-state.ts | 25 + .../slices/runtime-status-recheck.test.ts | 127 +++- .../store/slices/runtime-status-recheck.ts | 52 +- .../src/store/slices/runtime-status.test.ts | 41 +- .../src/store/slices/runtime-status.ts | 10 + .../helpers/runtime-endpoint-link-fault.ts | 16 + ...mote-terminal-tab-switch-reconnect.spec.ts | 543 ++++++++++++++++++ ...ar-runtime-host-disconnected-glyph.spec.ts | 124 ++++ 13 files changed, 1014 insertions(+), 38 deletions(-) create mode 100644 tests/e2e/paired-remote-terminal-tab-switch-reconnect.spec.ts create mode 100644 tests/e2e/sidebar-runtime-host-disconnected-glyph.spec.ts diff --git a/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx b/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx index 46bf7cd371f..efd47b976ec 100644 --- a/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx +++ b/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx @@ -83,7 +83,8 @@ describe('NoticeHostGlyph', () => { ) }) - it('marks a paired runtime with no live status as disconnected', async () => { + it('marks a paired runtime a probe found unreachable as disconnected', async () => { + runtimeStatusByEnvironmentId.set('openclaw-env', { status: null }) const container = await render('runtime:openclaw-env') expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe( @@ -91,6 +92,17 @@ describe('NoticeHostGlyph', () => { ) }) + it('does not call a host disconnected before its first probe answers', async () => { + // No entry means "not asked yet", not "asked and unreachable" — collapsing the two + // painted every remote row destructive between launch and the first probe. + const container = await render('runtime:openclaw-env') + + expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe( + 'Project on openclaw' + ) + expect(container.querySelector('svg')?.getAttribute('class')).not.toContain('text-destructive') + }) + it('gives the local host the monitor glyph the run-target rows use', async () => { const container = await render('local', 'Local Mac') diff --git a/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx b/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx index db1616203dd..7c070450e7d 100644 --- a/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx +++ b/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx @@ -5,6 +5,10 @@ import { HostRowIcon } from '../host-row-icon' import { useAppStore } from '@/store' import { parseExecutionHostId, type ExecutionHostId } from '../../../../shared/execution-host' import { translate } from '@/i18n/i18n' +import { + isDisconnectedRuntimeHostState, + runtimeHostConnectionStateForEntry +} from '@/runtime/runtime-host-connection-state' type NoticeHostGlyphProps = { hostId: ExecutionHostId @@ -26,11 +30,15 @@ export default function NoticeHostGlyph({ keyboardFocusable }: NoticeHostGlyphProps): React.JSX.Element | null { const host = parseExecutionHostId(hostId) + // Why the shared derivation, not raw truthiness: an absent entry means "not probed yet", + // which is not the same verdict as a probe that came back unreachable. const isDisconnected = useAppStore((s) => { if (host?.kind !== 'runtime') { return false } - return !s.runtimeStatusByEnvironmentId.get(host.environmentId)?.status + return isDisconnectedRuntimeHostState( + runtimeHostConnectionStateForEntry(s.runtimeStatusByEnvironmentId.get(host.environmentId)) + ) }) if (!host) { diff --git a/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx b/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx index 1fe2f4b9530..55ef3263ad4 100644 --- a/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx +++ b/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx @@ -218,18 +218,35 @@ describe('WorktreeCard SSH reconnect prompt', () => { expect(markup).not.toContain('Retry SSH connection') }) - it('marks a runtime-host worktree disconnected when its environment has no status', () => { + it('marks a runtime-host worktree disconnected once a probe finds it unreachable', () => { + runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }] + runtimeStatusByEnvironmentId.set('env-1', { status: null }) + const runtimeRepo: Repo = { + ...makeRepo(), + connectionId: undefined, + executionHostId: 'runtime:env-1' + } + const markup = renderToStaticMarkup( + + ) + expect(markup).toContain('Remote Mac disconnected') + }) + + // Why: "not probed yet" is not "probed and unreachable" — collapsing them painted every + // remote card destructive and dimmed between launch and the first probe answering. + it('leaves a runtime-host worktree undimmed before its first probe answers', () => { runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }] const runtimeRepo: Repo = { ...makeRepo(), connectionId: undefined, executionHostId: 'runtime:env-1' } - // No status entry for env-1 → host is disconnected. const markup = renderToStaticMarkup( ) - expect(markup).toContain('Remote Mac disconnected') + expect(markup).not.toContain('Remote Mac disconnected') + expect(markup).toContain('Project on Remote Mac') + expect(markup).not.toContain('opacity-60') }) it('distinguishes connected worktrees on different Orca servers', () => { diff --git a/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts b/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts index fc339bbe33f..d8ec20bb45a 100644 --- a/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts +++ b/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts @@ -10,6 +10,10 @@ import { getHostDisplayLabelOverrides } from '../../../../shared/host-setting-ov import { getWorkspacePortsByWorktreeId } from '@/lib/workspace-port-groups' import { getExplicitRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner' import { hydrateRuntimeEnvironmentSshState } from '@/runtime/runtime-environment-ssh-state' +import { + isDisconnectedRuntimeHostState, + runtimeHostConnectionStateForEntry +} from '@/runtime/runtime-host-connection-state' import { useAppStore } from '@/store' import { selectRuntimeAwareSshStatus, @@ -177,12 +181,17 @@ export function useWorktreeCardFoundation({ const runtimeHostLabel = runtimeHostId ? (getHostDisplayLabelOverrides(settings).get(runtimeHostId) ?? runtimeEnvironmentName) : null - // Why: runtime ("Orca server") hosts get the same disconnected dimming as SSH when their environment has no live status. + // Why the shared derivation, not raw truthiness: an absent entry means "not probed yet", + // which is not the same verdict as a probe that came back unreachable. const isRuntimeDisconnected = useAppStore((s) => { if (!runtimeOwnerEnvironmentId) { return false } - return !s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId)?.status + return isDisconnectedRuntimeHostState( + runtimeHostConnectionStateForEntry( + s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId) + ) + ) }) const [titleRenaming, setTitleRenaming] = useState(false) const [showRenameErrorDialog, setShowRenameErrorDialog] = useState(false) diff --git a/src/renderer/src/runtime/runtime-host-connection-state.test.ts b/src/renderer/src/runtime/runtime-host-connection-state.test.ts index 8db0c0eb010..2e739feb9f2 100644 --- a/src/renderer/src/runtime/runtime-host-connection-state.test.ts +++ b/src/renderer/src/runtime/runtime-host-connection-state.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from 'vitest' import type { RuntimeStatus } from '../../../shared/runtime-types' import { isConnectedRuntimeHostState, + isDisconnectedRuntimeHostState, runtimeHostConnectionState, + runtimeHostConnectionStateForEntry, runtimeStatusForOverall } from './runtime-host-connection-state' @@ -177,3 +179,55 @@ describe('runtime host connection state', () => { ).toBe('disconnected') }) }) + +describe('runtime host connection state for a recorded status entry', () => { + it('separates a host that was never probed from one a probe found unreachable', () => { + // The sidebar read raw truthiness, which collapsed these two into the same red glyph. + expect(runtimeHostConnectionStateForEntry(undefined)).toBe('checking') + expect(runtimeHostConnectionStateForEntry({ status: null })).toBe('disconnected') + }) + + it('reads the remote-control diagnostics recorded beside a failed probe', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: null, + remoteControl: remoteControl('reconnecting') + }) + ).toBe('reconnecting') + }) + + it('agrees with the status bar that a closed control channel is disconnected', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: makeStatus({ remoteControl: remoteControl('closed') }) + }) + ).toBe('disconnected') + }) + + it('names only the disconnected verdict as disconnected', () => { + expect(isDisconnectedRuntimeHostState('disconnected')).toBe(true) + for (const state of [ + 'connected', + 'checking', + 'reconnecting', + 'runtime-unavailable', + 'workspace-window-closed' + ] as const) { + expect(isDisconnectedRuntimeHostState(state)).toBe(false) + } + }) +}) + +function remoteControl( + state: NonNullable['state'] +): NonNullable { + return { + state, + pendingRequestCount: 0, + subscriptionCount: 0, + reconnectAttempt: 1, + lastConnectedAt: null, + lastClose: null, + lastError: null + } +} diff --git a/src/renderer/src/runtime/runtime-host-connection-state.ts b/src/renderer/src/runtime/runtime-host-connection-state.ts index 6094c995804..49114c2ee6d 100644 --- a/src/renderer/src/runtime/runtime-host-connection-state.ts +++ b/src/renderer/src/runtime/runtime-host-connection-state.ts @@ -97,3 +97,28 @@ export function isConnectedRuntimeHostState(state: RuntimeHostConnectionState): state === 'connected' || state === 'runtime-unavailable' || state === 'workspace-window-closed' ) } + +/** + * Only this verdict earns the destructive glyph. 'checking' and 'reconnecting' are + * unverifiable, not down, per docs/reference/ssh-execution-boundary.md. + */ +export function isDisconnectedRuntimeHostState(state: RuntimeHostConnectionState): boolean { + return state === 'disconnected' +} + +/** The same derivation, read straight off a recorded status entry. */ +export function runtimeHostConnectionStateForEntry( + entry: + | { + status: RuntimeStatus | null + remoteControl?: RuntimeStatus['remoteControl'] | null + } + | null + | undefined +): RuntimeHostConnectionState { + return runtimeHostConnectionState({ + hasStatusEntry: Boolean(entry), + status: entry?.status ?? null, + remoteControl: entry?.remoteControl ?? entry?.status?.remoteControl ?? null + }) +} diff --git a/src/renderer/src/store/slices/runtime-status-recheck.test.ts b/src/renderer/src/store/slices/runtime-status-recheck.test.ts index 9f09580c390..2c1f140f99d 100644 --- a/src/renderer/src/store/slices/runtime-status-recheck.test.ts +++ b/src/renderer/src/store/slices/runtime-status-recheck.test.ts @@ -66,7 +66,7 @@ describe('runtime status recheck', () => { expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.checkedAt).toBe(1) }) - it('cancels on removal, capability loss, and null without probing again', async () => { + it('cancels on removal and capability loss without probing again', async () => { const getStatus = vi.fn() const store = createStore(getStatus) store.getState().setRuntimeEnvironmentStatus('env-a', { @@ -148,7 +148,75 @@ describe('runtime status recheck', () => { }) }) - it('keeps setter side effects when a recheck discovers disconnection', async () => { + it('re-probes a host recorded unreachable until it answers again', async () => { + // A boot probe that failed while the host was asleep must not outlive the outage: + // nothing else re-asks, because the client-event subscription set is gated on a truthy status. + const getStatus = vi + .fn() + .mockResolvedValueOnce(unavailableResponse()) + .mockResolvedValue(response(status('ready'))) + const store = createStore(getStatus) + + store.getState().setRuntimeEnvironmentStatus('env-a', { status: null, checkedAt: 1 }) + + await vi.advanceTimersByTimeAsync(3_000) + expect(getStatus).toHaveBeenCalledWith({ + selector: 'env-a', + timeoutMs: 10_000, + observeOnly: true + }) + expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBeNull() + + await vi.advanceTimersByTimeAsync(6_000) + expect( + store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl + ).toMatchObject({ state: 'ready' }) + + const callsAtRecovery = getStatus.mock.calls.length + await vi.advanceTimersByTimeAsync(300_000) + expect(getStatus).toHaveBeenCalledTimes(callsAtRecovery) + }) + + it('does not re-toast while the ladder keeps confirming the same outage', async () => { + // The ladder republishes null on every failed retry; only a real truthy -> null + // transition is news, so the warning must not pop once per retry. + const getStatus = vi.fn().mockResolvedValue(unavailableResponse()) + const store = createStore(getStatus) + + store.getState().setRuntimeEnvironmentStatus('env-a', { status: null, checkedAt: 1 }) + await vi.advanceTimersByTimeAsync(3_000 + 6_000 + 12_000 + 30_000) + + expect(getStatus).toHaveBeenCalledTimes(4) + expect(toast.warning).not.toHaveBeenCalled() + }) + + it('stops re-probing a manually disconnected host', async () => { + // The probe short-circuits locally for these, so retrying only burns a timer forever. + const getStatus = vi.fn().mockResolvedValue({ + id: 'runtime.manualDisconnect', + ok: false, + error: { + code: 'runtime_manually_disconnected', + message: 'Runtime environment is manually disconnected.' + }, + _meta: { runtimeId: 'rt' } + }) + const store = createStore(getStatus) + + store.getState().setRuntimeEnvironmentStatus('env-a', { status: null, checkedAt: 1 }) + + await vi.advanceTimersByTimeAsync(3_000) + expect(getStatus).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(300_000) + expect(getStatus).toHaveBeenCalledOnce() + }) + + it('preserves a live verdict when an unverifiable probe cannot reach the host (#19647)', async () => { + // A failed status.get dials its own fresh socket, so its runtime_unavailable answer is + // unverifiable — the client could not ask. Nulling the recorded live status here would drop + // the environment out of the session-tabs mirror targets and dim its sidebar rows even though + // its established flows are still delivering. The verdict must survive; only its diagnostics + // are refreshed to reconnecting so the ladder keeps probing. const getStatus = vi.fn().mockResolvedValue({ id: 'status.get', ok: false, @@ -167,13 +235,45 @@ describe('runtime status recheck', () => { await vi.advanceTimersByTimeAsync(3_000) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBeNull() - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.remoteControl).toMatchObject( - { - state: 'reconnecting' - } - ) - expect(toast.warning).toHaveBeenCalledOnce() + const entry = store.getState().runtimeStatusByEnvironmentId.get('env-a') + expect(entry?.status).not.toBeNull() + expect(entry?.status?.remoteControl).toMatchObject({ state: 'reconnecting' }) + // No truthy -> null transition, so the disconnect toast must not fire on an unverifiable probe. + expect(toast.warning).not.toHaveBeenCalled() + }) + + it('does not advance the connection generation when recovering from an unverifiable probe (#19647)', async () => { + // The double-teardown: nulling a live status retires the mirror once, and the null -> truthy + // recovery advances the connection generation, rebuilding it a second time. Preserving the + // verdict across the outage keeps the generation stable, so recovery is not a reconnect. + const getStatus = vi + .fn() + .mockResolvedValueOnce({ + id: 'status.get', + ok: false, + error: { + code: 'runtime_unavailable', + message: 'offline', + data: { remoteControl: status('reconnecting').remoteControl } + }, + _meta: { runtimeId: 'rt' } + }) + .mockResolvedValue(response(status('ready'))) + const store = createStore(getStatus) + store.getState().setRuntimeEnvironmentStatus('env-a', { + status: status('awaiting_ready'), + checkedAt: 1 + }) + const generationBefore = store + .getState() + .runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration + + await vi.advanceTimersByTimeAsync(3_000) + await vi.advanceTimersByTimeAsync(6_000) + + const entry = store.getState().runtimeStatusByEnvironmentId.get('env-a') + expect(entry?.status?.remoteControl).toMatchObject({ state: 'ready' }) + expect(entry?.connectionGeneration).toBe(generationBefore) }) }) @@ -212,6 +312,15 @@ function status( } as RuntimeStatus } +function unavailableResponse() { + return { + id: 'status.get', + ok: false as const, + error: { code: 'runtime_unavailable', message: 'offline' }, + _meta: { runtimeId: 'rt' } + } +} + function response(result: RuntimeStatus) { return { id: 'status.get', ok: true as const, result, _meta: { runtimeId: result.runtimeId } } } diff --git a/src/renderer/src/store/slices/runtime-status-recheck.ts b/src/renderer/src/store/slices/runtime-status-recheck.ts index 303e5b4318d..b462c56b8a0 100644 --- a/src/renderer/src/store/slices/runtime-status-recheck.ts +++ b/src/renderer/src/store/slices/runtime-status-recheck.ts @@ -1,6 +1,6 @@ import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' import type { RuntimeStatus } from '../../../../shared/runtime-types' -import { unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client' +import { hasRuntimeRpcErrorCode, unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client' import { extractRuntimeTransportDiagnostics } from '@/runtime/runtime-status-probe-diagnostics' import type { RuntimeEnvironmentStatus } from './runtime-status' @@ -14,11 +14,13 @@ type RecheckState = { connectionGeneration: number environmentExists: () => boolean getConnectionGeneration: () => number + getCurrentStatus: () => RuntimeEnvironmentStatus | undefined publish: (status: RuntimeEnvironmentStatus) => void } type RuntimeStatusStore = { runtimeEnvironments: readonly { id: string }[] + runtimeStatusByEnvironmentId: ReadonlyMap setRuntimeEnvironmentStatus: (environmentId: string, status: RuntimeEnvironmentStatus) => void } @@ -30,6 +32,7 @@ export function reconcileRuntimeStatusRecheck(args: { connectionGeneration: number environmentExists: () => boolean getConnectionGeneration: () => number + getCurrentStatus: () => RuntimeEnvironmentStatus | undefined publish: (status: RuntimeEnvironmentStatus) => void }): void { if (!shouldRecheck(args.status)) { @@ -50,6 +53,7 @@ export function reconcileRuntimeStatusRecheck(args: { connectionGeneration: args.connectionGeneration, environmentExists: args.environmentExists, getConnectionGeneration: args.getConnectionGeneration, + getCurrentStatus: args.getCurrentStatus, publish: args.publish } rechecks.set(args.environmentId, state) @@ -57,6 +61,7 @@ export function reconcileRuntimeStatusRecheck(args: { state.connectionGeneration = args.connectionGeneration state.environmentExists = args.environmentExists state.getConnectionGeneration = args.getConnectionGeneration + state.getCurrentStatus = args.getCurrentStatus state.publish = args.publish } armRuntimeStatusRecheck(args.environmentId, state) @@ -75,6 +80,7 @@ export function reconcileRuntimeStatusForSlice( environmentExists: () => get().runtimeEnvironments.some((environment) => environment.id === environmentId), getConnectionGeneration, + getCurrentStatus: () => get().runtimeStatusByEnvironmentId.get(environmentId), publish: (nextStatus) => get().setRuntimeEnvironmentStatus(environmentId, nextStatus) }) } @@ -101,9 +107,19 @@ export function clearRuntimeStatusRechecksForTests(): void { cancelRuntimeStatusRechecks([...rechecks.keys()]) } +/** + * Whether this recorded verdict is one the ladder must keep re-asking. + * + * Null qualifies because a host recorded unreachable is excluded from the client-event + * subscription set (that set is gated on a truthy status), so no reconnect signal can + * ever clear it and one failed boot probe otherwise outlives the outage. #16516 + */ function shouldRecheck(status: RuntimeStatus | null): boolean { + if (status === null) { + return true + } return Boolean( - status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) && + status.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) && status.remoteControl && status.remoteControl.state !== 'ready' ) @@ -147,12 +163,34 @@ async function fireRuntimeStatusRecheck( }) nextEntry = { status: unwrapRuntimeRpcResult(response), checkedAt: Date.now() } } catch (error: unknown) { - const remoteControl = extractRuntimeTransportDiagnostics(error) - nextEntry = { - status: null, - ...(remoteControl ? { remoteControl } : {}), - checkedAt: Date.now() + // The probe short-circuits locally for a manually disconnected host, so retrying only + // burns a timer against an answer the user already chose. + if (hasRuntimeRpcErrorCode(error, 'runtime_manually_disconnected')) { + cancelRuntimeStatusRecheck(environmentId) + return } + const remoteControl = extractRuntimeTransportDiagnostics(error) + const current = state.getCurrentStatus() + // A failed status.get dials its own fresh socket (sendRemoteRuntimeRequest), so its + // failure is unverifiable — main mints `runtime_unavailable` for a transport error it + // never received an answer to, and per docs/reference/ssh-execution-boundary.md loss of + // contact is never evidence the host exited. Nulling a live verdict here would retire the + // host's session-tabs mirror (it drops out of getReachableRuntimeSessionMirrorTargets) and + // then, on the next successful probe, rebuild it a second time via the connection-generation + // bump in setRuntimeEnvironmentStatus. Keep the live status and only refresh its diagnostics + // so the ladder keeps probing without a teardown. #19647 + nextEntry = + hasRuntimeRpcErrorCode(error, 'runtime_unavailable') && current?.status != null + ? { + ...current, + status: remoteControl ? { ...current.status, remoteControl } : current.status, + checkedAt: Date.now() + } + : { + status: null, + ...(remoteControl ? { remoteControl } : {}), + checkedAt: Date.now() + } } state.inFlight = false if ( diff --git a/src/renderer/src/store/slices/runtime-status.test.ts b/src/renderer/src/store/slices/runtime-status.test.ts index 8e3260957bf..1b4945144c4 100644 --- a/src/renderer/src/store/slices/runtime-status.test.ts +++ b/src/renderer/src/store/slices/runtime-status.test.ts @@ -710,9 +710,8 @@ describe('runtime-status slice', () => { clearRuntimeCompatibilityCacheForTests() }) - // Both directions of the failure-publication policy, from one failing probe. A user-initiated - // check publishes the outage it just observed; a caller holding live transport evidence must - // not, because status.get dials its own socket and its failure is unverifiable, not exited. + // First-contact failure (no recorded verdict yet): a default check records the outage so host + // coverage completes; a caller holding live transport evidence records nothing (unverifiable). it.each([ { name: 'a user-initiated check', options: undefined, publishes: true }, { name: 'publishUnreachable defaulted', options: {}, publishes: true }, @@ -721,22 +720,34 @@ describe('runtime-status slice', () => { options: { publishUnreachable: false }, publishes: false } - ])('records null and returns false when a runtime refresh fails: $name', async (scenario) => { - const getStatus = vi.fn().mockRejectedValue(new Error('closed')) - stubRuntimeEnvironmentApi({ getStatus }) + ])( + 'records null and returns false when a first-contact refresh fails: $name', + async (scenario) => { + stubRuntimeEnvironmentApi({ getStatus: vi.fn().mockRejectedValue(new Error('closed')) }) + const store = createSliceStore() + + // The dial-answered contract the bridge's bounded retry chain reads is policy-independent. + const reachable = await store + .getState() + .refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options) + expect(reachable).toBe(false) + expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe( + scenario.publishes ? null : undefined + ) + } + ) + + // #19647: a failed status.get dials its own fresh socket, so a default (non-opted-out) refresh + // must not overwrite a recorded live verdict with null — that retires the host's session-tabs + // mirror and dims its still-live rows on a fault the client could not even ask through. + it('preserves a recorded live verdict when a default refresh probe fails', async () => { + stubRuntimeEnvironmentApi({ getStatus: vi.fn().mockRejectedValue(new Error('closed')) }) const store = createSliceStore() const cached = makeStatus() store.getState().setRuntimeEnvironmentStatus('env-a', { status: cached, checkedAt: 1 }) - const reachable = await store - .getState() - .refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options) - - // The dial-answered contract the bridge's bounded retry chain reads is policy-independent. - expect(reachable).toBe(false) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe( - scenario.publishes ? null : cached - ) + expect(await store.getState().refreshRuntimeEnvironmentStatus('env-a')).toBe(false) + expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(cached) }) it('hydrates saved environments through the single-environment refresh path', async () => { diff --git a/src/renderer/src/store/slices/runtime-status.ts b/src/renderer/src/store/slices/runtime-status.ts index 10985f389d8..192dbf6c21a 100644 --- a/src/renderer/src/store/slices/runtime-status.ts +++ b/src/renderer/src/store/slices/runtime-status.ts @@ -236,6 +236,7 @@ export const createRuntimeStatusSlice: StateCreator environment.id === environmentId), getConnectionGeneration: () => runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration(environmentId), + getCurrentStatus: () => get().runtimeStatusByEnvironmentId.get(environmentId), publish: (entry) => get().setRuntimeEnvironmentStatus(environmentId, entry) }) if (runtimeRestarted) { @@ -301,6 +302,15 @@ export const createRuntimeStatusSlice: StateCreator void + /** + * Severs every established flow once, leaving `mode` in force for what dials next. + * With `stall-new` this is the link flap that puts the shared-control socket into + * `reconnecting` while its replacement dial hangs. + */ + dropEstablished: () => number /** Connections accepted since the last reset — proves a dial actually reached the hop. */ acceptedConnectionCount: () => number stalledConnectionCount: () => number @@ -83,6 +89,16 @@ export async function startRuntimeEndpointLinkFault( stalled.clear() } }, + dropEstablished: () => { + let dropped = 0 + for (const socket of live) { + if (!stalled.has(socket)) { + socket.destroy() + dropped += 1 + } + } + return dropped + }, acceptedConnectionCount: () => accepted, stalledConnectionCount: () => stalledTotal, resetCounters: () => { diff --git a/tests/e2e/paired-remote-terminal-tab-switch-reconnect.spec.ts b/tests/e2e/paired-remote-terminal-tab-switch-reconnect.spec.ts new file mode 100644 index 00000000000..1b798747a27 --- /dev/null +++ b/tests/e2e/paired-remote-terminal-tab-switch-reconnect.spec.ts @@ -0,0 +1,543 @@ +/** + * #19647 "Orca has to reconnect every time I switch tabs". + * + * TOPOLOGY: real Orca desktop app as the remote server + a separate real Orca + * desktop client paired to it, with every client->host byte routed through a TCP + * hop whose fault mode this test controls. That hop is the reporter's Tailscale + * link; `stall-new` reproduces its characteristic failure, where established + * flows keep delivering (their agent kept working) while a freshly dialed + * connection hangs. + * + * Exploratory-first: the `[tab-switch-repro]` census and timeline lines are the + * diagnosis. The hard gate is the #19647 writer contract — a status.get the client + * could not push through must not null a recorded live verdict, and recovery must not + * advance the connection generation. Whether the pane repaints within budget is logged, + * NOT asserted: that is gated on the un-park latch (#19872) and the multiplexer + * cold-handshake (#19871), both filed separately. A green run is NOT a clean bill of health. + * + * Run: + * pnpm exec playwright test \ + * tests/e2e/paired-remote-terminal-tab-switch-reconnect.spec.ts \ + * --config tests/playwright.config.ts --project electron-headless --workers=1 + */ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { randomUUID } from 'node:crypto' +import os from 'node:os' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import { + HOST_TERMINAL_SURFACE_SEPARATOR, + toWebTerminalSurfaceTabId +} from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient +} from './helpers/paired-electron-client' +import { + readPairingEndpoint, + repointPairingUrl, + startRuntimeEndpointLinkFault +} from './helpers/runtime-endpoint-link-fault' +import { waitForTabParked } from './helpers/terminal-hidden-parking' + +const PARK_DELAY_MS = 2_000 +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-tab-switch-reconnect-')) +const fixturePath = path.join(scratch, 'tab-switch-terminal.mjs') +writeFileSync( + fixturePath, + [ + "import { appendFileSync } from 'node:fs'", + 'const sink = process.argv[2]', + 'const record = (line) => appendFileSync(sink, `${line}\\n`)', + "record('READY')", + "process.stdout.write('READY\\r\\n')", + "process.stdin.setEncoding('utf8')", + "let pending = ''", + "process.stdin.on('data', (data) => {", + ' pending += data', + ' const lines = pending.split(/\\r\\n|\\r|\\n/)', + " pending = lines.pop() ?? ''", + ' for (const line of lines) {', + ' record(`LINE:${line}`)', + ' process.stdout.write(`LINE:${line}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(sinkPath: string): string { + const command = [process.execPath, fixturePath, sinkPath] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +function readSink(sinkPath: string): string { + try { + return readFileSync(sinkPath, 'utf8') + } catch { + return '' + } +} + +async function callEnvironment( + page: Page, + environmentId: string, + method: string, + params: unknown +): Promise { + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params + }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { environmentId, method, params } + ) as Promise +} + +type HostTerminal = { + hostTabId: string + sinkPath: string + terminal: string + webTabId: string +} + +async function createHostTerminal( + page: Page, + environmentId: string, + worktreeId: string +): Promise { + const sinkPath = path.join(scratch, `sink-${randomUUID()}.log`) + const result = await callEnvironment<{ + tab: { id: string; terminal: string | null } + }>(page, environmentId, 'session.tabs.createTerminal', { + worktree: `id:${worktreeId}`, + command: fixtureCommand(sinkPath), + activate: false, + select: false, + navigation: 'caller' + }) + if (!result.tab.terminal) { + throw new Error('host session terminal was not created') + } + const hostTabId = result.tab.id.split(HOST_TERMINAL_SURFACE_SEPARATOR)[0] + return { + hostTabId, + sinkPath, + terminal: result.tab.terminal, + webTabId: toWebTerminalSurfaceTabId(hostTabId) + } +} + +async function waitForMirroredTab(page: Page, worktreeId: string, webTabId: string): Promise { + await expect + .poll( + () => + page.evaluate( + (id) => (window.__store?.getState().tabsByWorktree[id] ?? []).map((tab) => tab.id), + worktreeId + ), + { + timeout: 60_000, + message: `client never mirrored host tab ${webTabId}` + } + ) + .toContain(webTabId) +} + +async function selectClientTab(page: Page, worktreeId: string, webTabId: string): Promise { + await page.evaluate( + ({ webTabId, worktreeId }) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(worktreeId) + state?.setActiveTab(webTabId) + state?.setActiveTabType('terminal') + }, + { webTabId, worktreeId } + ) +} + +async function openClientTab(page: Page, worktreeId: string, webTabId: string): Promise { + await waitForMirroredTab(page, worktreeId, webTabId) + await selectClientTab(page, worktreeId, webTabId) + await expect + .poll(() => page.evaluate((id) => window.__paneManagers?.has(id) ?? false, webTabId), { + timeout: 60_000, + message: `client pane for ${webTabId} did not mount` + }) + .toBe(true) +} + +type MultiplexCensus = { + activeStreamCount: number + transportSubscribeCount: number + transportUnsubscribeCount: number + streamSubscribeCount: number + streamUnsubscribeCount: number +} + +async function readMultiplexCensus(page: Page): Promise { + return page.evaluate(() => { + const snapshot = ( + window as Window & { + __remoteTerminalMultiplexAckGate?: { + snapshot: () => { + activeStreams: unknown[] + transportSubscribeCount: number + transportUnsubscribeCount: number + streamSubscribeCount: number + streamUnsubscribeCount: number + } + } + } + ).__remoteTerminalMultiplexAckGate?.snapshot() + return { + activeStreamCount: snapshot?.activeStreams.length ?? -1, + transportSubscribeCount: snapshot?.transportSubscribeCount ?? -1, + transportUnsubscribeCount: snapshot?.transportUnsubscribeCount ?? -1, + streamSubscribeCount: snapshot?.streamSubscribeCount ?? -1, + streamUnsubscribeCount: snapshot?.streamUnsubscribeCount ?? -1 + } + }) +} + +type PaneObservation = { + atMs: number + banner: string | null + recoveryState: string | null + mounted: boolean + nullStatusEnvironments: number + connectionGeneration: number | null + remoteControlState: string | null +} + +async function observePane( + page: Page, + webTabId: string, + environmentId: string, + startedAt: number +): Promise { + const observation = await page.evaluate( + ({ id, environmentId }) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + const banner = document.querySelector('[data-terminal-remote-runtime-reconnect-banner]') + const statuses = window.__store?.getState().runtimeStatusByEnvironmentId + let nullStatusEnvironments = 0 + for (const entry of statuses?.values() ?? []) { + if (entry.status === null) { + nullStatusEnvironments += 1 + } + } + return { + banner: banner?.getAttribute('data-terminal-remote-runtime-reconnect-banner') ?? null, + recoveryState: pane?.container?.dataset?.ptyRecoveryState ?? null, + mounted: Boolean(manager), + nullStatusEnvironments, + connectionGeneration: statuses?.get(environmentId)?.connectionGeneration ?? null, + remoteControlState: + statuses?.get(environmentId)?.status?.remoteControl?.state ?? + statuses?.get(environmentId)?.remoteControl?.state ?? + null + } + }, + { id: webTabId, environmentId } + ) + return { atMs: Date.now() - startedAt, ...observation } +} + +async function readPaneContent(page: Page, webTabId: string): Promise { + return page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane?.serializeAddon?.serialize?.() ?? '' + }, webTabId) +} + +type RevealRecord = { + timeline: PaneObservation[] + paintedAtMs: number | null + sawBanner: boolean + sawNullStatus: boolean +} + +/** Samples the revealed pane for `budgetMs`, stopping early once `marker` paints. */ +async function recordRevealTimeline( + page: Page, + webTabId: string, + environmentId: string, + marker: string, + budgetMs: number +): Promise { + const startedAt = Date.now() + const timeline: PaneObservation[] = [] + let paintedAtMs: number | null = null + let sawBanner = false + let sawNullStatus = false + let previous = '' + while (Date.now() - startedAt < budgetMs) { + const observation = await observePane(page, webTabId, environmentId, startedAt) + sawBanner ||= observation.banner !== null + sawNullStatus ||= observation.nullStatusEnvironments > 0 + const key = `${observation.banner}|${observation.recoveryState}|${observation.mounted}|${observation.nullStatusEnvironments}|${observation.connectionGeneration}|${observation.remoteControlState}` + if (key !== previous) { + timeline.push(observation) + previous = key + } + if ((await readPaneContent(page, webTabId)).includes(marker)) { + paintedAtMs = Date.now() - startedAt + timeline.push(await observePane(page, webTabId, environmentId, startedAt)) + break + } + await new Promise((resolve) => setTimeout(resolve, 250)) + } + return { timeline, paintedAtMs, sawBanner, sawNullStatus } +} + +test('paired client tab switch does not reconnect the remote runtime', async ({ + orcaPage +}, testInfo) => { + test.setTimeout(900_000) + const rawOffer = await createRuntimeDesktopPairingOffer(orcaPage) + const link = await startRuntimeEndpointLinkFault(readPairingEndpoint(rawOffer.pairingUrl)) + const offer = { + ...rawOffer, + pairingUrl: repointPairingUrl(rawOffer.pairingUrl, link.endpoint) + } + + const client = await launchPairedElectronClient(offer, testInfo, 'tab-switch-reconnect', { + extraEnv: { ORCA_E2E_TERMINAL_PARKING_DELAY_MS: String(PARK_DELAY_MS) } + }) + const createdTerminals: string[] = [] + try { + const worktreeId = await orcaPage.evaluate(() => { + const id = window.__store?.getState().activeWorktreeId + if (!id) { + throw new Error('headed host has no active worktree') + } + return id + }) + await expect + .poll( + () => + client.page.evaluate( + (id) => + window.__store + ?.getState() + .allWorktrees() + .some((worktree) => worktree.id === id) ?? false, + worktreeId + ), + { + timeout: 60_000, + message: 'paired client never saw the host worktree' + } + ) + .toBe(true) + + const target = await createHostTerminal(client.page, client.environmentId, worktreeId) + const decoys = [ + await createHostTerminal(client.page, client.environmentId, worktreeId), + await createHostTerminal(client.page, client.environmentId, worktreeId) + ] + createdTerminals.push(target.terminal, ...decoys.map((decoy) => decoy.terminal)) + + await openClientTab(client.page, worktreeId, target.webTabId) + await expect + .poll(() => readPaneContent(client.page, target.webTabId), { + timeout: 60_000, + message: 'target terminal never painted READY' + }) + .toContain('READY') + + const censusAfterFirstOpen = await readMultiplexCensus(client.page) + console.log( + `[tab-switch-repro] census after first open: ${JSON.stringify(censusAfterFirstOpen)}` + ) + + // ---- Arm A: plain tab switching on a healthy link. No fault involved: if the + // transport subscribe count climbs here, tab switching alone tears down and + // re-establishes the environment's control channel. + for (let round = 0; round < 3; round += 1) { + await openClientTab(client.page, worktreeId, decoys[0].webTabId) + await openClientTab(client.page, worktreeId, decoys[1].webTabId) + await waitForTabParked(client.page, target.webTabId, { + parkDelayMs: PARK_DELAY_MS + }) + await openClientTab(client.page, worktreeId, target.webTabId) + } + const censusAfterHealthySwitching = await readMultiplexCensus(client.page) + console.log( + `[tab-switch-repro] census after healthy switching: ${JSON.stringify(censusAfterHealthySwitching)}` + ) + console.log( + `[tab-switch-repro] ARM A transportSubscribeCount delta over 3 healthy park/reveal rounds: ${censusAfterHealthySwitching.transportSubscribeCount - censusAfterFirstOpen.transportSubscribeCount} (unsubscribe delta ${censusAfterHealthySwitching.transportUnsubscribeCount - censusAfterFirstOpen.transportUnsubscribeCount})` + ) + + // ---- Arm B: park every terminal pane for this environment, so nothing holds + // the multiplexed control channel open, then reveal one under a link whose + // established flows are fine but whose new dials hang. + // Why not a plain view switch: cold-park exempts the single most-recently-hidden + // tab, so one pane (and its stream) keeps the multiplexer alive, and the exemption + // migrates to a parked tab the moment the exempt one closes. So: make a decoy the + // exempt one, park the rest, put the link into stall-new, and only then close that + // decoy on the host (over the established control flow). The multiplexer idles out + // and nothing can re-dial it on a healthy path before the reveal. + await openClientTab(client.page, worktreeId, decoys[1].webTabId) + await client.page.evaluate(() => { + window.__store?.getState().setActiveView('changes') + }) + await waitForTabParked(client.page, target.webTabId, { parkDelayMs: PARK_DELAY_MS }) + await waitForTabParked(client.page, decoys[0].webTabId, { parkDelayMs: PARK_DELAY_MS }) + const censusBeforeRelease = await readMultiplexCensus(client.page) + console.log( + `[tab-switch-repro] census with only the exempt decoy mounted: ${JSON.stringify(censusBeforeRelease)}` + ) + link.resetCounters() + link.setMode('stall-new') + await callEnvironment(client.page, client.environmentId, 'terminal.closeTab', { + terminal: decoys[1].terminal + }) + createdTerminals.splice(createdTerminals.indexOf(decoys[1].terminal), 1) + await expect + .poll(async () => (await readMultiplexCensus(client.page)).transportUnsubscribeCount, { + timeout: 30_000, + message: 'the terminal multiplexer was never released after its last stream closed' + }) + .toBeGreaterThan(censusBeforeRelease.transportUnsubscribeCount) + const censusAfterFullPark = await readMultiplexCensus(client.page) + console.log( + `[tab-switch-repro] census after every pane parked and the multiplexer released: ${JSON.stringify(censusAfterFullPark)}` + ) + // Setup invariant, not a claim about the bug: the reveal below has to pay a fresh + // dial, or Arm B degrades into "reveal a tab whose transport is still up". + expect( + censusAfterFullPark.transportUnsubscribeCount, + 'Arm B precondition: the terminal multiplexer must have been released before the stalled reveal' + ).toBeGreaterThan(censusBeforeRelease.transportUnsubscribeCount) + + await selectClientTab(client.page, worktreeId, target.webTabId) + const stalled = await recordRevealTimeline( + client.page, + target.webTabId, + client.environmentId, + 'READY', + 40_000 + ) + console.log( + `[tab-switch-repro] stalled reveal: painted=${String(stalled.paintedAtMs)} banner=${stalled.sawBanner} nullStatus=${stalled.sawNullStatus} timeline=${JSON.stringify(stalled.timeline)}` + ) + console.log( + `[tab-switch-repro] dials at the hop: accepted=${link.acceptedConnectionCount()} stalled=${link.stalledConnectionCount()}` + ) + + // ---- Arm B2: the reporter's "reconnecting" state. Sever the established flows + // once while new dials still hang, so the shared-control socket enters + // `reconnecting` and the status recheck ladder arms. What the store does with + // the resulting failed status.get is the writer under test: a live verdict must + // not become `status: null` because the client could not ask. + const generationBeforeFlap = ( + await observePane(client.page, target.webTabId, client.environmentId, 0) + ).connectionGeneration + const dropped = link.dropEstablished() + const flapped = await recordRevealTimeline( + client.page, + target.webTabId, + client.environmentId, + 'READY', + 45_000 + ) + console.log( + `[tab-switch-repro] link flap under stall-new: dropped=${dropped} banner=${flapped.sawBanner} nullStatus=${flapped.sawNullStatus} timeline=${JSON.stringify(flapped.timeline)}` + ) + + // ---- Arm C: restore the link and measure time-to-recovery. + link.setMode('pass') + // Drive the documented recovery path. A pane whose attach timed out parks a retry that + // arms no timer (#19872): it waits for online/resume/manual Reconnect, so within a bounded + // budget the pane only comes back when one of those fires. Dispatching 'online' is exactly + // that trigger — the same one a real network-return raises — so this arm exercises genuine + // recovery, not the ~180s latch. + await client.page.evaluate(() => window.dispatchEvent(new Event('online'))) + const recovered = await recordRevealTimeline( + client.page, + target.webTabId, + client.environmentId, + 'READY', + 30_000 + ) + console.log( + `[tab-switch-repro] recovery after link restored: painted=${String(recovered.paintedAtMs)} timeline=${JSON.stringify(recovered.timeline)}` + ) + console.log( + `[tab-switch-repro] census after recovery: ${JSON.stringify(await readMultiplexCensus(client.page))}` + ) + console.log(`[tab-switch-repro] target sink: ${readSink(target.sinkPath).slice(0, 200)}`) + const generationAfterRecovery = ( + await observePane(client.page, target.webTabId, client.environmentId, 0) + ).connectionGeneration + console.log( + `[tab-switch-repro] connection generation across the flap: ${String(generationBeforeFlap)} -> ${String(generationAfterRecovery)}` + ) + + // Writer contract (#19647), the fix under test: a status.get that could not reach the host is + // unverifiable, so the recorded live verdict survives the transport fault (no null published) + // and recovery is not a second connection (the connection generation never advances). These + // are the hard gate; both fail on the unfixed writer. + expect( + flapped.sawNullStatus, + 'a failed status.get over a stalled link published status: null over a live verdict (#19647)' + ).toBe(false) + expect( + generationAfterRecovery, + 'recovery advanced the connection generation, so the session-tabs mirror was rebuilt (#19647)' + ).toBe(generationBeforeFlap) + // The fix keeps the environment revivable rather than retiring it: the pane is never disposed + // and its host is never dropped from the mirror targets, so a later trigger can bring it back. + const finalObservation = await observePane( + client.page, + target.webTabId, + client.environmentId, + 0 + ) + expect( + finalObservation.mounted, + 'the revealed pane was disposed instead of kept revivable' + ).toBe(true) + // Recovery TIMELINE is diagnostic, NOT a pass/fail gate: whether the pane actually repaints + // within budget depends on the un-park latch (#19872) and the multiplexer cold-handshake + // (#19871), both filed separately and out of scope here. A green run is not a clean bill of + // health — read the [tab-switch-repro] timeline above. This value is unchanged by this PR + // (the unfixed writer left it null too), so it is logged, not asserted. + console.log( + `[tab-switch-repro] recovery-contract observation (gated on #19872/#19871): painted=${String(recovered.paintedAtMs)}` + ) + } finally { + link.setMode('pass') + for (const terminal of createdTerminals) { + await callEnvironment(client.page, client.environmentId, 'terminal.closeTab', { + terminal + }).catch(() => undefined) + } + await client.dispose() + await link.close() + } +}) diff --git a/tests/e2e/sidebar-runtime-host-disconnected-glyph.spec.ts b/tests/e2e/sidebar-runtime-host-disconnected-glyph.spec.ts new file mode 100644 index 00000000000..08d202214c8 --- /dev/null +++ b/tests/e2e/sidebar-runtime-host-disconnected-glyph.spec.ts @@ -0,0 +1,124 @@ +import type { Locator, Page, TestInfo } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { waitForActiveWorktree, waitForSessionReady } from './helpers/store' + +const ENVIRONMENT_ID = 'e2e-remote-host' +const HOST_LABEL = 'Remote Mac' + +type RecordedState = 'no-entry' | 'probed-unreachable' | 'probed-reachable' + +/** Put the seeded worktree's repo on a runtime host and record one status verdict for it. */ +async function seedRuntimeHost(page: Page, recorded: RecordedState): Promise { + return page.evaluate( + ({ environmentId, hostLabel, recorded }) => { + const store = window.__store + if (!store) { + throw new Error('window.__store is not available') + } + const state = store.getState() + const worktree = Object.values(state.worktreesByRepo).flat()[0] + if (!worktree) { + throw new Error('no seeded worktree to place on a runtime host') + } + state.setRuntimeEnvironments([ + { + id: environmentId, + name: hostLabel, + createdAt: 1, + updatedAt: 1, + lastUsedAt: null, + runtimeId: 'e2e-runtime', + endpoints: [ + { id: 'ws', kind: 'websocket', label: 'WebSocket', endpoint: 'ws://127.0.0.1:1' } + ], + preferredEndpointId: 'ws' + } + ]) + store.setState({ + repos: state.repos.map((repo) => + repo.id === worktree.repoId + ? { ...repo, connectionId: undefined, executionHostId: `runtime:${environmentId}` } + : repo + ) + }) + if (recorded === 'probed-unreachable') { + state.setRuntimeEnvironmentStatus(environmentId, { status: null, checkedAt: Date.now() }) + } + if (recorded === 'probed-reachable') { + state.setRuntimeEnvironmentStatus(environmentId, { + status: { + runtimeId: 'e2e-runtime', + rendererGraphEpoch: 1, + graphStatus: 'ready', + authoritativeWindowId: 1, + desktopWindowStatus: 'available', + liveTabCount: 0, + liveLeafCount: 0 + }, + checkedAt: Date.now() + }) + } + return worktree.id + }, + { environmentId: ENVIRONMENT_ID, hostLabel: HOST_LABEL, recorded } + ) +} + +async function captureCard(card: Locator, testInfo: TestInfo, name: string): Promise { + const shot = testInfo.outputPath(name) + await card.screenshot({ path: shot, animations: 'disabled' }) + await testInfo.attach(name, { path: shot, contentType: 'image/png' }) +} + +test.describe('sidebar runtime host glyph', () => { + test.beforeEach(async ({ orcaPage }) => { + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + }) + + // Why: an absent entry means "not probed yet". Painting it destructive made every + // remote card red and dimmed between launch and the first probe answering. + test('does not call a runtime host disconnected before its first probe answers', async ({ + orcaPage + }, testInfo) => { + await seedRuntimeHost(orcaPage, 'no-entry') + const card = orcaPage.locator(`[data-worktree-card-surface="true"]`).first() + await expect(card).toBeVisible() + + // Captured before the assertions so a regression run still yields the evidence image. + await captureCard(card, testInfo, 'runtime-host-glyph-before-probe.png') + + await expect(card.locator('svg.lucide-server')).toBeVisible() + await expect(card.locator('svg.lucide-server-off')).toHaveCount(0) + await expect(card).not.toHaveClass(/opacity-60/) + }) + + // The deliberate no-change case: a probe actually reported this host unreachable. + test('still marks a runtime host disconnected once a probe finds it unreachable', async ({ + orcaPage + }, testInfo) => { + await seedRuntimeHost(orcaPage, 'probed-unreachable') + const card = orcaPage.locator(`[data-worktree-card-surface="true"]`).first() + await expect(card).toBeVisible() + + await captureCard(card, testInfo, 'runtime-host-glyph-disconnected.png') + + await expect(card.locator('svg.lucide-server-off')).toBeVisible() + await expect(card).toHaveClass(/opacity-60/) + }) + + test('clears the disconnected glyph when a later probe reaches the host', async ({ + orcaPage + }, testInfo) => { + await seedRuntimeHost(orcaPage, 'probed-unreachable') + const card = orcaPage.locator(`[data-worktree-card-surface="true"]`).first() + await expect(card.locator('svg.lucide-server-off')).toBeVisible() + + await seedRuntimeHost(orcaPage, 'probed-reachable') + await captureCard(card, testInfo, 'runtime-host-glyph-recovered.png') + + await expect(card.locator('svg.lucide-server')).toBeVisible() + await expect(card.locator('svg.lucide-server-off')).toHaveCount(0) + await expect(card).not.toHaveClass(/opacity-60/) + }) +})