mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
merge: PR #19873 into integration stack
This commit is contained in:
@@ -83,7 +83,8 @@ describe('NoticeHostGlyph', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('marks a paired runtime with no live status as disconnected', async () => {
|
||||
it('marks a paired runtime a probe found unreachable as disconnected', async () => {
|
||||
runtimeStatusByEnvironmentId.set('openclaw-env', { status: null })
|
||||
const container = await render('runtime:openclaw-env')
|
||||
|
||||
expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe(
|
||||
@@ -91,6 +92,17 @@ describe('NoticeHostGlyph', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('does not call a host disconnected before its first probe answers', async () => {
|
||||
// No entry means "not asked yet", not "asked and unreachable" — collapsing the two
|
||||
// painted every remote row destructive between launch and the first probe.
|
||||
const container = await render('runtime:openclaw-env')
|
||||
|
||||
expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe(
|
||||
'Project on openclaw'
|
||||
)
|
||||
expect(container.querySelector('svg')?.getAttribute('class')).not.toContain('text-destructive')
|
||||
})
|
||||
|
||||
it('gives the local host the monitor glyph the run-target rows use', async () => {
|
||||
const container = await render('local', 'Local Mac')
|
||||
|
||||
|
||||
@@ -5,6 +5,10 @@ import { HostRowIcon } from '../host-row-icon'
|
||||
import { useAppStore } from '@/store'
|
||||
import { parseExecutionHostId, type ExecutionHostId } from '../../../../shared/execution-host'
|
||||
import { translate } from '@/i18n/i18n'
|
||||
import {
|
||||
isDisconnectedRuntimeHostState,
|
||||
runtimeHostConnectionStateForEntry
|
||||
} from '@/runtime/runtime-host-connection-state'
|
||||
|
||||
type NoticeHostGlyphProps = {
|
||||
hostId: ExecutionHostId
|
||||
@@ -26,11 +30,15 @@ export default function NoticeHostGlyph({
|
||||
keyboardFocusable
|
||||
}: NoticeHostGlyphProps): React.JSX.Element | null {
|
||||
const host = parseExecutionHostId(hostId)
|
||||
// Why the shared derivation, not raw truthiness: an absent entry means "not probed yet",
|
||||
// which is not the same verdict as a probe that came back unreachable.
|
||||
const isDisconnected = useAppStore((s) => {
|
||||
if (host?.kind !== 'runtime') {
|
||||
return false
|
||||
}
|
||||
return !s.runtimeStatusByEnvironmentId.get(host.environmentId)?.status
|
||||
return isDisconnectedRuntimeHostState(
|
||||
runtimeHostConnectionStateForEntry(s.runtimeStatusByEnvironmentId.get(host.environmentId))
|
||||
)
|
||||
})
|
||||
|
||||
if (!host) {
|
||||
|
||||
@@ -218,18 +218,35 @@ describe('WorktreeCard SSH reconnect prompt', () => {
|
||||
expect(markup).not.toContain('Retry SSH connection')
|
||||
})
|
||||
|
||||
it('marks a runtime-host worktree disconnected when its environment has no status', () => {
|
||||
it('marks a runtime-host worktree disconnected once a probe finds it unreachable', () => {
|
||||
runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }]
|
||||
runtimeStatusByEnvironmentId.set('env-1', { status: null })
|
||||
const runtimeRepo: Repo = {
|
||||
...makeRepo(),
|
||||
connectionId: undefined,
|
||||
executionHostId: 'runtime:env-1'
|
||||
}
|
||||
const markup = renderToStaticMarkup(
|
||||
<WorktreeCard worktree={makeWorktree()} repo={runtimeRepo} isActive={false} />
|
||||
)
|
||||
expect(markup).toContain('Remote Mac disconnected')
|
||||
})
|
||||
|
||||
// Why: "not probed yet" is not "probed and unreachable" — collapsing them painted every
|
||||
// remote card destructive and dimmed between launch and the first probe answering.
|
||||
it('leaves a runtime-host worktree undimmed before its first probe answers', () => {
|
||||
runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }]
|
||||
const runtimeRepo: Repo = {
|
||||
...makeRepo(),
|
||||
connectionId: undefined,
|
||||
executionHostId: 'runtime:env-1'
|
||||
}
|
||||
// No status entry for env-1 → host is disconnected.
|
||||
const markup = renderToStaticMarkup(
|
||||
<WorktreeCard worktree={makeWorktree()} repo={runtimeRepo} isActive={false} />
|
||||
)
|
||||
expect(markup).toContain('Remote Mac disconnected')
|
||||
expect(markup).not.toContain('Remote Mac disconnected')
|
||||
expect(markup).toContain('Project on Remote Mac')
|
||||
expect(markup).not.toContain('opacity-60')
|
||||
})
|
||||
|
||||
it('distinguishes connected worktrees on different Orca servers', () => {
|
||||
|
||||
@@ -10,6 +10,10 @@ import { getHostDisplayLabelOverrides } from '../../../../shared/host-setting-ov
|
||||
import { getWorkspacePortsByWorktreeId } from '@/lib/workspace-port-groups'
|
||||
import { getExplicitRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner'
|
||||
import { hydrateRuntimeEnvironmentSshState } from '@/runtime/runtime-environment-ssh-state'
|
||||
import {
|
||||
isDisconnectedRuntimeHostState,
|
||||
runtimeHostConnectionStateForEntry
|
||||
} from '@/runtime/runtime-host-connection-state'
|
||||
import { useAppStore } from '@/store'
|
||||
import {
|
||||
selectRuntimeAwareSshStatus,
|
||||
@@ -177,12 +181,17 @@ export function useWorktreeCardFoundation({
|
||||
const runtimeHostLabel = runtimeHostId
|
||||
? (getHostDisplayLabelOverrides(settings).get(runtimeHostId) ?? runtimeEnvironmentName)
|
||||
: null
|
||||
// Why: runtime ("Orca server") hosts get the same disconnected dimming as SSH when their environment has no live status.
|
||||
// Why the shared derivation, not raw truthiness: an absent entry means "not probed yet",
|
||||
// which is not the same verdict as a probe that came back unreachable.
|
||||
const isRuntimeDisconnected = useAppStore((s) => {
|
||||
if (!runtimeOwnerEnvironmentId) {
|
||||
return false
|
||||
}
|
||||
return !s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId)?.status
|
||||
return isDisconnectedRuntimeHostState(
|
||||
runtimeHostConnectionStateForEntry(
|
||||
s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId)
|
||||
)
|
||||
)
|
||||
})
|
||||
const [titleRenaming, setTitleRenaming] = useState(false)
|
||||
const [showRenameErrorDialog, setShowRenameErrorDialog] = useState(false)
|
||||
|
||||
@@ -2,7 +2,9 @@ import { describe, expect, it } from 'vitest'
|
||||
import type { RuntimeStatus } from '../../../shared/runtime-types'
|
||||
import {
|
||||
isConnectedRuntimeHostState,
|
||||
isDisconnectedRuntimeHostState,
|
||||
runtimeHostConnectionState,
|
||||
runtimeHostConnectionStateForEntry,
|
||||
runtimeStatusForOverall
|
||||
} from './runtime-host-connection-state'
|
||||
|
||||
@@ -177,3 +179,55 @@ describe('runtime host connection state', () => {
|
||||
).toBe('disconnected')
|
||||
})
|
||||
})
|
||||
|
||||
describe('runtime host connection state for a recorded status entry', () => {
|
||||
it('separates a host that was never probed from one a probe found unreachable', () => {
|
||||
// The sidebar read raw truthiness, which collapsed these two into the same red glyph.
|
||||
expect(runtimeHostConnectionStateForEntry(undefined)).toBe('checking')
|
||||
expect(runtimeHostConnectionStateForEntry({ status: null })).toBe('disconnected')
|
||||
})
|
||||
|
||||
it('reads the remote-control diagnostics recorded beside a failed probe', () => {
|
||||
expect(
|
||||
runtimeHostConnectionStateForEntry({
|
||||
status: null,
|
||||
remoteControl: remoteControl('reconnecting')
|
||||
})
|
||||
).toBe('reconnecting')
|
||||
})
|
||||
|
||||
it('agrees with the status bar that a closed control channel is disconnected', () => {
|
||||
expect(
|
||||
runtimeHostConnectionStateForEntry({
|
||||
status: makeStatus({ remoteControl: remoteControl('closed') })
|
||||
})
|
||||
).toBe('disconnected')
|
||||
})
|
||||
|
||||
it('names only the disconnected verdict as disconnected', () => {
|
||||
expect(isDisconnectedRuntimeHostState('disconnected')).toBe(true)
|
||||
for (const state of [
|
||||
'connected',
|
||||
'checking',
|
||||
'reconnecting',
|
||||
'runtime-unavailable',
|
||||
'workspace-window-closed'
|
||||
] as const) {
|
||||
expect(isDisconnectedRuntimeHostState(state)).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
function remoteControl(
|
||||
state: NonNullable<RuntimeStatus['remoteControl']>['state']
|
||||
): NonNullable<RuntimeStatus['remoteControl']> {
|
||||
return {
|
||||
state,
|
||||
pendingRequestCount: 0,
|
||||
subscriptionCount: 0,
|
||||
reconnectAttempt: 1,
|
||||
lastConnectedAt: null,
|
||||
lastClose: null,
|
||||
lastError: null
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,3 +97,28 @@ export function isConnectedRuntimeHostState(state: RuntimeHostConnectionState):
|
||||
state === 'connected' || state === 'runtime-unavailable' || state === 'workspace-window-closed'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Only this verdict earns the destructive glyph. 'checking' and 'reconnecting' are
|
||||
* unverifiable, not down, per docs/reference/ssh-execution-boundary.md.
|
||||
*/
|
||||
export function isDisconnectedRuntimeHostState(state: RuntimeHostConnectionState): boolean {
|
||||
return state === 'disconnected'
|
||||
}
|
||||
|
||||
/** The same derivation, read straight off a recorded status entry. */
|
||||
export function runtimeHostConnectionStateForEntry(
|
||||
entry:
|
||||
| {
|
||||
status: RuntimeStatus | null
|
||||
remoteControl?: RuntimeStatus['remoteControl'] | null
|
||||
}
|
||||
| null
|
||||
| undefined
|
||||
): RuntimeHostConnectionState {
|
||||
return runtimeHostConnectionState({
|
||||
hasStatusEntry: Boolean(entry),
|
||||
status: entry?.status ?? null,
|
||||
remoteControl: entry?.remoteControl ?? entry?.status?.remoteControl ?? null
|
||||
})
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ describe('runtime status recheck', () => {
|
||||
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.checkedAt).toBe(1)
|
||||
})
|
||||
|
||||
it('cancels on removal, capability loss, and null without probing again', async () => {
|
||||
it('cancels on removal and capability loss without probing again', async () => {
|
||||
const getStatus = vi.fn()
|
||||
const store = createStore(getStatus)
|
||||
store.getState().setRuntimeEnvironmentStatus('env-a', {
|
||||
@@ -148,7 +148,75 @@ describe('runtime status recheck', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps setter side effects when a recheck discovers disconnection', async () => {
|
||||
it('re-probes a host recorded unreachable until it answers again', async () => {
|
||||
// A boot probe that failed while the host was asleep must not outlive the outage:
|
||||
// nothing else re-asks, because the client-event subscription set is gated on a truthy status.
|
||||
const getStatus = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(unavailableResponse())
|
||||
.mockResolvedValue(response(status('ready')))
|
||||
const store = createStore(getStatus)
|
||||
|
||||
store.getState().setRuntimeEnvironmentStatus('env-a', { status: null, checkedAt: 1 })
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3_000)
|
||||
expect(getStatus).toHaveBeenCalledWith({
|
||||
selector: 'env-a',
|
||||
timeoutMs: 10_000,
|
||||
observeOnly: true
|
||||
})
|
||||
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBeNull()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(6_000)
|
||||
expect(
|
||||
store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl
|
||||
).toMatchObject({ state: 'ready' })
|
||||
|
||||
const callsAtRecovery = getStatus.mock.calls.length
|
||||
await vi.advanceTimersByTimeAsync(300_000)
|
||||
expect(getStatus).toHaveBeenCalledTimes(callsAtRecovery)
|
||||
})
|
||||
|
||||
it('does not re-toast while the ladder keeps confirming the same outage', async () => {
|
||||
// The ladder republishes null on every failed retry; only a real truthy -> null
|
||||
// transition is news, so the warning must not pop once per retry.
|
||||
const getStatus = vi.fn().mockResolvedValue(unavailableResponse())
|
||||
const store = createStore(getStatus)
|
||||
|
||||
store.getState().setRuntimeEnvironmentStatus('env-a', { status: null, checkedAt: 1 })
|
||||
await vi.advanceTimersByTimeAsync(3_000 + 6_000 + 12_000 + 30_000)
|
||||
|
||||
expect(getStatus).toHaveBeenCalledTimes(4)
|
||||
expect(toast.warning).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops re-probing a manually disconnected host', async () => {
|
||||
// The probe short-circuits locally for these, so retrying only burns a timer forever.
|
||||
const getStatus = vi.fn().mockResolvedValue({
|
||||
id: 'runtime.manualDisconnect',
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'runtime_manually_disconnected',
|
||||
message: 'Runtime environment is manually disconnected.'
|
||||
},
|
||||
_meta: { runtimeId: 'rt' }
|
||||
})
|
||||
const store = createStore(getStatus)
|
||||
|
||||
store.getState().setRuntimeEnvironmentStatus('env-a', { status: null, checkedAt: 1 })
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3_000)
|
||||
expect(getStatus).toHaveBeenCalledOnce()
|
||||
await vi.advanceTimersByTimeAsync(300_000)
|
||||
expect(getStatus).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('preserves a live verdict when an unverifiable probe cannot reach the host (#19647)', async () => {
|
||||
// A failed status.get dials its own fresh socket, so its runtime_unavailable answer is
|
||||
// unverifiable — the client could not ask. Nulling the recorded live status here would drop
|
||||
// the environment out of the session-tabs mirror targets and dim its sidebar rows even though
|
||||
// its established flows are still delivering. The verdict must survive; only its diagnostics
|
||||
// are refreshed to reconnecting so the ladder keeps probing.
|
||||
const getStatus = vi.fn().mockResolvedValue({
|
||||
id: 'status.get',
|
||||
ok: false,
|
||||
@@ -167,13 +235,45 @@ describe('runtime status recheck', () => {
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3_000)
|
||||
|
||||
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBeNull()
|
||||
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.remoteControl).toMatchObject(
|
||||
{
|
||||
state: 'reconnecting'
|
||||
}
|
||||
)
|
||||
expect(toast.warning).toHaveBeenCalledOnce()
|
||||
const entry = store.getState().runtimeStatusByEnvironmentId.get('env-a')
|
||||
expect(entry?.status).not.toBeNull()
|
||||
expect(entry?.status?.remoteControl).toMatchObject({ state: 'reconnecting' })
|
||||
// No truthy -> null transition, so the disconnect toast must not fire on an unverifiable probe.
|
||||
expect(toast.warning).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not advance the connection generation when recovering from an unverifiable probe (#19647)', async () => {
|
||||
// The double-teardown: nulling a live status retires the mirror once, and the null -> truthy
|
||||
// recovery advances the connection generation, rebuilding it a second time. Preserving the
|
||||
// verdict across the outage keeps the generation stable, so recovery is not a reconnect.
|
||||
const getStatus = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
id: 'status.get',
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'runtime_unavailable',
|
||||
message: 'offline',
|
||||
data: { remoteControl: status('reconnecting').remoteControl }
|
||||
},
|
||||
_meta: { runtimeId: 'rt' }
|
||||
})
|
||||
.mockResolvedValue(response(status('ready')))
|
||||
const store = createStore(getStatus)
|
||||
store.getState().setRuntimeEnvironmentStatus('env-a', {
|
||||
status: status('awaiting_ready'),
|
||||
checkedAt: 1
|
||||
})
|
||||
const generationBefore = store
|
||||
.getState()
|
||||
.runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3_000)
|
||||
await vi.advanceTimersByTimeAsync(6_000)
|
||||
|
||||
const entry = store.getState().runtimeStatusByEnvironmentId.get('env-a')
|
||||
expect(entry?.status?.remoteControl).toMatchObject({ state: 'ready' })
|
||||
expect(entry?.connectionGeneration).toBe(generationBefore)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -212,6 +312,15 @@ function status(
|
||||
} as RuntimeStatus
|
||||
}
|
||||
|
||||
function unavailableResponse() {
|
||||
return {
|
||||
id: 'status.get',
|
||||
ok: false as const,
|
||||
error: { code: 'runtime_unavailable', message: 'offline' },
|
||||
_meta: { runtimeId: 'rt' }
|
||||
}
|
||||
}
|
||||
|
||||
function response(result: RuntimeStatus) {
|
||||
return { id: 'status.get', ok: true as const, result, _meta: { runtimeId: result.runtimeId } }
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version'
|
||||
import type { RuntimeStatus } from '../../../../shared/runtime-types'
|
||||
import { unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client'
|
||||
import { hasRuntimeRpcErrorCode, unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client'
|
||||
import { extractRuntimeTransportDiagnostics } from '@/runtime/runtime-status-probe-diagnostics'
|
||||
import type { RuntimeEnvironmentStatus } from './runtime-status'
|
||||
|
||||
@@ -14,11 +14,13 @@ type RecheckState = {
|
||||
connectionGeneration: number
|
||||
environmentExists: () => boolean
|
||||
getConnectionGeneration: () => number
|
||||
getCurrentStatus: () => RuntimeEnvironmentStatus | undefined
|
||||
publish: (status: RuntimeEnvironmentStatus) => void
|
||||
}
|
||||
|
||||
type RuntimeStatusStore = {
|
||||
runtimeEnvironments: readonly { id: string }[]
|
||||
runtimeStatusByEnvironmentId: ReadonlyMap<string, RuntimeEnvironmentStatus>
|
||||
setRuntimeEnvironmentStatus: (environmentId: string, status: RuntimeEnvironmentStatus) => void
|
||||
}
|
||||
|
||||
@@ -30,6 +32,7 @@ export function reconcileRuntimeStatusRecheck(args: {
|
||||
connectionGeneration: number
|
||||
environmentExists: () => boolean
|
||||
getConnectionGeneration: () => number
|
||||
getCurrentStatus: () => RuntimeEnvironmentStatus | undefined
|
||||
publish: (status: RuntimeEnvironmentStatus) => void
|
||||
}): void {
|
||||
if (!shouldRecheck(args.status)) {
|
||||
@@ -50,6 +53,7 @@ export function reconcileRuntimeStatusRecheck(args: {
|
||||
connectionGeneration: args.connectionGeneration,
|
||||
environmentExists: args.environmentExists,
|
||||
getConnectionGeneration: args.getConnectionGeneration,
|
||||
getCurrentStatus: args.getCurrentStatus,
|
||||
publish: args.publish
|
||||
}
|
||||
rechecks.set(args.environmentId, state)
|
||||
@@ -57,6 +61,7 @@ export function reconcileRuntimeStatusRecheck(args: {
|
||||
state.connectionGeneration = args.connectionGeneration
|
||||
state.environmentExists = args.environmentExists
|
||||
state.getConnectionGeneration = args.getConnectionGeneration
|
||||
state.getCurrentStatus = args.getCurrentStatus
|
||||
state.publish = args.publish
|
||||
}
|
||||
armRuntimeStatusRecheck(args.environmentId, state)
|
||||
@@ -75,6 +80,7 @@ export function reconcileRuntimeStatusForSlice(
|
||||
environmentExists: () =>
|
||||
get().runtimeEnvironments.some((environment) => environment.id === environmentId),
|
||||
getConnectionGeneration,
|
||||
getCurrentStatus: () => get().runtimeStatusByEnvironmentId.get(environmentId),
|
||||
publish: (nextStatus) => get().setRuntimeEnvironmentStatus(environmentId, nextStatus)
|
||||
})
|
||||
}
|
||||
@@ -101,9 +107,19 @@ export function clearRuntimeStatusRechecksForTests(): void {
|
||||
cancelRuntimeStatusRechecks([...rechecks.keys()])
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this recorded verdict is one the ladder must keep re-asking.
|
||||
*
|
||||
* Null qualifies because a host recorded unreachable is excluded from the client-event
|
||||
* subscription set (that set is gated on a truthy status), so no reconnect signal can
|
||||
* ever clear it and one failed boot probe otherwise outlives the outage. #16516
|
||||
*/
|
||||
function shouldRecheck(status: RuntimeStatus | null): boolean {
|
||||
if (status === null) {
|
||||
return true
|
||||
}
|
||||
return Boolean(
|
||||
status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) &&
|
||||
status.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) &&
|
||||
status.remoteControl &&
|
||||
status.remoteControl.state !== 'ready'
|
||||
)
|
||||
@@ -147,12 +163,34 @@ async function fireRuntimeStatusRecheck(
|
||||
})
|
||||
nextEntry = { status: unwrapRuntimeRpcResult<RuntimeStatus>(response), checkedAt: Date.now() }
|
||||
} catch (error: unknown) {
|
||||
const remoteControl = extractRuntimeTransportDiagnostics(error)
|
||||
nextEntry = {
|
||||
status: null,
|
||||
...(remoteControl ? { remoteControl } : {}),
|
||||
checkedAt: Date.now()
|
||||
// The probe short-circuits locally for a manually disconnected host, so retrying only
|
||||
// burns a timer against an answer the user already chose.
|
||||
if (hasRuntimeRpcErrorCode(error, 'runtime_manually_disconnected')) {
|
||||
cancelRuntimeStatusRecheck(environmentId)
|
||||
return
|
||||
}
|
||||
const remoteControl = extractRuntimeTransportDiagnostics(error)
|
||||
const current = state.getCurrentStatus()
|
||||
// A failed status.get dials its own fresh socket (sendRemoteRuntimeRequest), so its
|
||||
// failure is unverifiable — main mints `runtime_unavailable` for a transport error it
|
||||
// never received an answer to, and per docs/reference/ssh-execution-boundary.md loss of
|
||||
// contact is never evidence the host exited. Nulling a live verdict here would retire the
|
||||
// host's session-tabs mirror (it drops out of getReachableRuntimeSessionMirrorTargets) and
|
||||
// then, on the next successful probe, rebuild it a second time via the connection-generation
|
||||
// bump in setRuntimeEnvironmentStatus. Keep the live status and only refresh its diagnostics
|
||||
// so the ladder keeps probing without a teardown. #19647
|
||||
nextEntry =
|
||||
hasRuntimeRpcErrorCode(error, 'runtime_unavailable') && current?.status != null
|
||||
? {
|
||||
...current,
|
||||
status: remoteControl ? { ...current.status, remoteControl } : current.status,
|
||||
checkedAt: Date.now()
|
||||
}
|
||||
: {
|
||||
status: null,
|
||||
...(remoteControl ? { remoteControl } : {}),
|
||||
checkedAt: Date.now()
|
||||
}
|
||||
}
|
||||
state.inFlight = false
|
||||
if (
|
||||
|
||||
@@ -710,9 +710,8 @@ describe('runtime-status slice', () => {
|
||||
clearRuntimeCompatibilityCacheForTests()
|
||||
})
|
||||
|
||||
// Both directions of the failure-publication policy, from one failing probe. A user-initiated
|
||||
// check publishes the outage it just observed; a caller holding live transport evidence must
|
||||
// not, because status.get dials its own socket and its failure is unverifiable, not exited.
|
||||
// First-contact failure (no recorded verdict yet): a default check records the outage so host
|
||||
// coverage completes; a caller holding live transport evidence records nothing (unverifiable).
|
||||
it.each([
|
||||
{ name: 'a user-initiated check', options: undefined, publishes: true },
|
||||
{ name: 'publishUnreachable defaulted', options: {}, publishes: true },
|
||||
@@ -721,22 +720,34 @@ describe('runtime-status slice', () => {
|
||||
options: { publishUnreachable: false },
|
||||
publishes: false
|
||||
}
|
||||
])('records null and returns false when a runtime refresh fails: $name', async (scenario) => {
|
||||
const getStatus = vi.fn().mockRejectedValue(new Error('closed'))
|
||||
stubRuntimeEnvironmentApi({ getStatus })
|
||||
])(
|
||||
'records null and returns false when a first-contact refresh fails: $name',
|
||||
async (scenario) => {
|
||||
stubRuntimeEnvironmentApi({ getStatus: vi.fn().mockRejectedValue(new Error('closed')) })
|
||||
const store = createSliceStore()
|
||||
|
||||
// The dial-answered contract the bridge's bounded retry chain reads is policy-independent.
|
||||
const reachable = await store
|
||||
.getState()
|
||||
.refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options)
|
||||
expect(reachable).toBe(false)
|
||||
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(
|
||||
scenario.publishes ? null : undefined
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
// #19647: a failed status.get dials its own fresh socket, so a default (non-opted-out) refresh
|
||||
// must not overwrite a recorded live verdict with null — that retires the host's session-tabs
|
||||
// mirror and dims its still-live rows on a fault the client could not even ask through.
|
||||
it('preserves a recorded live verdict when a default refresh probe fails', async () => {
|
||||
stubRuntimeEnvironmentApi({ getStatus: vi.fn().mockRejectedValue(new Error('closed')) })
|
||||
const store = createSliceStore()
|
||||
const cached = makeStatus()
|
||||
store.getState().setRuntimeEnvironmentStatus('env-a', { status: cached, checkedAt: 1 })
|
||||
|
||||
const reachable = await store
|
||||
.getState()
|
||||
.refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options)
|
||||
|
||||
// The dial-answered contract the bridge's bounded retry chain reads is policy-independent.
|
||||
expect(reachable).toBe(false)
|
||||
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(
|
||||
scenario.publishes ? null : cached
|
||||
)
|
||||
expect(await store.getState().refreshRuntimeEnvironmentStatus('env-a')).toBe(false)
|
||||
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(cached)
|
||||
})
|
||||
|
||||
it('hydrates saved environments through the single-environment refresh path', async () => {
|
||||
|
||||
@@ -236,6 +236,7 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
|
||||
get().runtimeEnvironments.some((environment) => environment.id === environmentId),
|
||||
getConnectionGeneration: () =>
|
||||
runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration(environmentId),
|
||||
getCurrentStatus: () => get().runtimeStatusByEnvironmentId.get(environmentId),
|
||||
publish: (entry) => get().setRuntimeEnvironmentStatus(environmentId, entry)
|
||||
})
|
||||
if (runtimeRestarted) {
|
||||
@@ -301,6 +302,15 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
|
||||
// Unverifiable, not exited: leave the cached verdict for the caller's retry to settle.
|
||||
return
|
||||
}
|
||||
// A null entry here only ever comes from a failed status.get, which dials its own fresh
|
||||
// socket — an unverifiable transport failure, never a host-answered "gone". Overwriting a
|
||||
// recorded live verdict with it retires the host's session-tabs mirror and dims its sidebar
|
||||
// rows for a host whose established flows are still delivering. Keep the live verdict; the
|
||||
// status recheck ladder keeps probing and settles it on a real answer. A first-contact
|
||||
// failure (no prior live status) still records null so host coverage completes. #19647
|
||||
if (entry.status === null && get().runtimeStatusByEnvironmentId.get(environmentId)?.status) {
|
||||
return
|
||||
}
|
||||
// Why: setRuntimeEnvironmentStatus drops any stale compat failure on a non-null
|
||||
// (reachable) status, so a recovered host's reuse-flagged refetches re-probe.
|
||||
get().setRuntimeEnvironmentStatus(environmentId, entry)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import net from 'node:net'
|
||||
import { decodePairingOffer, encodePairingOffer } from '../../../src/shared/pairing'
|
||||
|
||||
/**
|
||||
* A TCP hop in front of a paired runtime's WebSocket endpoint whose fault mode can
|
||||
* change mid-test.
|
||||
*
|
||||
* `stall-new` is the Tailscale-shaped fault this exists for: already-established
|
||||
* flows keep delivering while a freshly dialed connection hangs unanswered. That
|
||||
* asymmetry is what separates "the control plane could not ask" from "the runtime
|
||||
* is gone", and neither killing the runtime nor `runtimeEnvironments.disconnect`
|
||||
* can produce it — both take the two halves down together.
|
||||
*/
|
||||
export type RuntimeLinkFaultMode = 'pass' | 'stall-new'
|
||||
|
||||
export type RuntimeEndpointLinkFault = {
|
||||
/** ws:// endpoint that routes through this hop. */
|
||||
endpoint: string
|
||||
setMode: (mode: RuntimeLinkFaultMode) => void
|
||||
/**
|
||||
* Severs every established flow once, leaving `mode` in force for what dials next.
|
||||
* With `stall-new` this is the link flap that puts the shared-control socket into
|
||||
* `reconnecting` while its replacement dial hangs.
|
||||
*/
|
||||
dropEstablished: () => number
|
||||
/** Connections accepted since the last reset — proves a dial actually reached the hop. */
|
||||
acceptedConnectionCount: () => number
|
||||
stalledConnectionCount: () => number
|
||||
resetCounters: () => void
|
||||
close: () => Promise<void>
|
||||
}
|
||||
|
||||
function parseWsEndpoint(endpoint: string): { host: string; port: number } {
|
||||
const url = new URL(endpoint)
|
||||
return { host: url.hostname, port: Number(url.port) }
|
||||
}
|
||||
|
||||
export async function startRuntimeEndpointLinkFault(
|
||||
upstreamEndpoint: string
|
||||
): Promise<RuntimeEndpointLinkFault> {
|
||||
const upstream = parseWsEndpoint(upstreamEndpoint)
|
||||
let mode: RuntimeLinkFaultMode = 'pass'
|
||||
let accepted = 0
|
||||
let stalledTotal = 0
|
||||
const stalled = new Set<net.Socket>()
|
||||
const live = new Set<net.Socket>()
|
||||
|
||||
const server = net.createServer((client) => {
|
||||
accepted += 1
|
||||
live.add(client)
|
||||
client.on('close', () => live.delete(client))
|
||||
client.on('error', () => client.destroy())
|
||||
if (mode === 'stall-new') {
|
||||
// Accept the TCP handshake and answer nothing: the dialer waits out its own
|
||||
// timeout, as it does on a half-open path, instead of failing fast on reset.
|
||||
stalledTotal += 1
|
||||
stalled.add(client)
|
||||
client.on('close', () => stalled.delete(client))
|
||||
return
|
||||
}
|
||||
const upstreamSocket = net.connect(upstream.port, upstream.host)
|
||||
live.add(upstreamSocket)
|
||||
upstreamSocket.on('close', () => live.delete(upstreamSocket))
|
||||
upstreamSocket.on('error', () => client.destroy())
|
||||
client.pipe(upstreamSocket)
|
||||
upstreamSocket.pipe(client)
|
||||
})
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once('error', reject)
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
server.removeListener('error', reject)
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
const address = server.address()
|
||||
if (address === null || typeof address === 'string') {
|
||||
throw new Error('runtime endpoint link fault did not bind a TCP port')
|
||||
}
|
||||
|
||||
return {
|
||||
endpoint: `ws://127.0.0.1:${address.port}`,
|
||||
setMode: (next) => {
|
||||
mode = next
|
||||
if (next === 'pass') {
|
||||
for (const socket of stalled) {
|
||||
socket.destroy()
|
||||
}
|
||||
stalled.clear()
|
||||
}
|
||||
},
|
||||
dropEstablished: () => {
|
||||
let dropped = 0
|
||||
for (const socket of live) {
|
||||
if (!stalled.has(socket)) {
|
||||
socket.destroy()
|
||||
dropped += 1
|
||||
}
|
||||
}
|
||||
return dropped
|
||||
},
|
||||
acceptedConnectionCount: () => accepted,
|
||||
stalledConnectionCount: () => stalledTotal,
|
||||
resetCounters: () => {
|
||||
accepted = 0
|
||||
stalledTotal = 0
|
||||
},
|
||||
close: async () => {
|
||||
for (const socket of live) {
|
||||
socket.destroy()
|
||||
}
|
||||
live.clear()
|
||||
stalled.clear()
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-points a pairing offer at `endpoint` without touching its keys or device token. */
|
||||
export function repointPairingUrl(pairingUrl: string, endpoint: string): string {
|
||||
return encodePairingOffer({ ...decodePairingOffer(pairingUrl), endpoint })
|
||||
}
|
||||
|
||||
export function readPairingEndpoint(pairingUrl: string): string {
|
||||
return decodePairingOffer(pairingUrl).endpoint
|
||||
}
|
||||
@@ -0,0 +1,543 @@
|
||||
/**
|
||||
* #19647 "Orca has to reconnect every time I switch tabs".
|
||||
*
|
||||
* TOPOLOGY: real Orca desktop app as the remote server + a separate real Orca
|
||||
* desktop client paired to it, with every client->host byte routed through a TCP
|
||||
* hop whose fault mode this test controls. That hop is the reporter's Tailscale
|
||||
* link; `stall-new` reproduces its characteristic failure, where established
|
||||
* flows keep delivering (their agent kept working) while a freshly dialed
|
||||
* connection hangs.
|
||||
*
|
||||
* Exploratory-first: the `[tab-switch-repro]` census and timeline lines are the
|
||||
* diagnosis. The hard gate is the #19647 writer contract — a status.get the client
|
||||
* could not push through must not null a recorded live verdict, and recovery must not
|
||||
* advance the connection generation. Whether the pane repaints within budget is logged,
|
||||
* NOT asserted: that is gated on the un-park latch (#19872) and the multiplexer
|
||||
* cold-handshake (#19871), both filed separately. A green run is NOT a clean bill of health.
|
||||
*
|
||||
* Run:
|
||||
* pnpm exec playwright test \
|
||||
* tests/e2e/paired-remote-terminal-tab-switch-reconnect.spec.ts \
|
||||
* --config tests/playwright.config.ts --project electron-headless --workers=1
|
||||
*/
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import {
|
||||
HOST_TERMINAL_SURFACE_SEPARATOR,
|
||||
toWebTerminalSurfaceTabId
|
||||
} from '../../src/shared/terminal-surface-id'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import {
|
||||
createRuntimeDesktopPairingOffer,
|
||||
launchPairedElectronClient
|
||||
} from './helpers/paired-electron-client'
|
||||
import {
|
||||
readPairingEndpoint,
|
||||
repointPairingUrl,
|
||||
startRuntimeEndpointLinkFault
|
||||
} from './helpers/runtime-endpoint-link-fault'
|
||||
import { waitForTabParked } from './helpers/terminal-hidden-parking'
|
||||
|
||||
const PARK_DELAY_MS = 2_000
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-tab-switch-reconnect-'))
|
||||
const fixturePath = path.join(scratch, 'tab-switch-terminal.mjs')
|
||||
writeFileSync(
|
||||
fixturePath,
|
||||
[
|
||||
"import { appendFileSync } from 'node:fs'",
|
||||
'const sink = process.argv[2]',
|
||||
'const record = (line) => appendFileSync(sink, `${line}\\n`)',
|
||||
"record('READY')",
|
||||
"process.stdout.write('READY\\r\\n')",
|
||||
"process.stdin.setEncoding('utf8')",
|
||||
"let pending = ''",
|
||||
"process.stdin.on('data', (data) => {",
|
||||
' pending += data',
|
||||
' const lines = pending.split(/\\r\\n|\\r|\\n/)',
|
||||
" pending = lines.pop() ?? ''",
|
||||
' for (const line of lines) {',
|
||||
' record(`LINE:${line}`)',
|
||||
' process.stdout.write(`LINE:${line}\\r\\n`)',
|
||||
' }',
|
||||
'})',
|
||||
'process.stdin.resume()'
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
test.afterAll(() => {
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`
|
||||
}
|
||||
|
||||
function fixtureCommand(sinkPath: string): string {
|
||||
const command = [process.execPath, fixturePath, sinkPath]
|
||||
return process.platform === 'win32'
|
||||
? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')
|
||||
: command.map(shellQuote).join(' ')
|
||||
}
|
||||
|
||||
function readSink(sinkPath: string): string {
|
||||
try {
|
||||
return readFileSync(sinkPath, 'utf8')
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
async function callEnvironment<TResult>(
|
||||
page: Page,
|
||||
environmentId: string,
|
||||
method: string,
|
||||
params: unknown
|
||||
): Promise<TResult> {
|
||||
return page.evaluate(
|
||||
async ({ environmentId, method, params }) => {
|
||||
const response = await window.api.runtimeEnvironments.call({
|
||||
selector: environmentId,
|
||||
method,
|
||||
params
|
||||
})
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ environmentId, method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
type HostTerminal = {
|
||||
hostTabId: string
|
||||
sinkPath: string
|
||||
terminal: string
|
||||
webTabId: string
|
||||
}
|
||||
|
||||
async function createHostTerminal(
|
||||
page: Page,
|
||||
environmentId: string,
|
||||
worktreeId: string
|
||||
): Promise<HostTerminal> {
|
||||
const sinkPath = path.join(scratch, `sink-${randomUUID()}.log`)
|
||||
const result = await callEnvironment<{
|
||||
tab: { id: string; terminal: string | null }
|
||||
}>(page, environmentId, 'session.tabs.createTerminal', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
command: fixtureCommand(sinkPath),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
if (!result.tab.terminal) {
|
||||
throw new Error('host session terminal was not created')
|
||||
}
|
||||
const hostTabId = result.tab.id.split(HOST_TERMINAL_SURFACE_SEPARATOR)[0]
|
||||
return {
|
||||
hostTabId,
|
||||
sinkPath,
|
||||
terminal: result.tab.terminal,
|
||||
webTabId: toWebTerminalSurfaceTabId(hostTabId)
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForMirroredTab(page: Page, worktreeId: string, webTabId: string): Promise<void> {
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
page.evaluate(
|
||||
(id) => (window.__store?.getState().tabsByWorktree[id] ?? []).map((tab) => tab.id),
|
||||
worktreeId
|
||||
),
|
||||
{
|
||||
timeout: 60_000,
|
||||
message: `client never mirrored host tab ${webTabId}`
|
||||
}
|
||||
)
|
||||
.toContain(webTabId)
|
||||
}
|
||||
|
||||
async function selectClientTab(page: Page, worktreeId: string, webTabId: string): Promise<void> {
|
||||
await page.evaluate(
|
||||
({ webTabId, worktreeId }) => {
|
||||
const state = window.__store?.getState()
|
||||
state?.setActiveView('terminal')
|
||||
state?.setActiveWorktree(worktreeId)
|
||||
state?.setActiveTab(webTabId)
|
||||
state?.setActiveTabType('terminal')
|
||||
},
|
||||
{ webTabId, worktreeId }
|
||||
)
|
||||
}
|
||||
|
||||
async function openClientTab(page: Page, worktreeId: string, webTabId: string): Promise<void> {
|
||||
await waitForMirroredTab(page, worktreeId, webTabId)
|
||||
await selectClientTab(page, worktreeId, webTabId)
|
||||
await expect
|
||||
.poll(() => page.evaluate((id) => window.__paneManagers?.has(id) ?? false, webTabId), {
|
||||
timeout: 60_000,
|
||||
message: `client pane for ${webTabId} did not mount`
|
||||
})
|
||||
.toBe(true)
|
||||
}
|
||||
|
||||
type MultiplexCensus = {
|
||||
activeStreamCount: number
|
||||
transportSubscribeCount: number
|
||||
transportUnsubscribeCount: number
|
||||
streamSubscribeCount: number
|
||||
streamUnsubscribeCount: number
|
||||
}
|
||||
|
||||
async function readMultiplexCensus(page: Page): Promise<MultiplexCensus> {
|
||||
return page.evaluate(() => {
|
||||
const snapshot = (
|
||||
window as Window & {
|
||||
__remoteTerminalMultiplexAckGate?: {
|
||||
snapshot: () => {
|
||||
activeStreams: unknown[]
|
||||
transportSubscribeCount: number
|
||||
transportUnsubscribeCount: number
|
||||
streamSubscribeCount: number
|
||||
streamUnsubscribeCount: number
|
||||
}
|
||||
}
|
||||
}
|
||||
).__remoteTerminalMultiplexAckGate?.snapshot()
|
||||
return {
|
||||
activeStreamCount: snapshot?.activeStreams.length ?? -1,
|
||||
transportSubscribeCount: snapshot?.transportSubscribeCount ?? -1,
|
||||
transportUnsubscribeCount: snapshot?.transportUnsubscribeCount ?? -1,
|
||||
streamSubscribeCount: snapshot?.streamSubscribeCount ?? -1,
|
||||
streamUnsubscribeCount: snapshot?.streamUnsubscribeCount ?? -1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type PaneObservation = {
|
||||
atMs: number
|
||||
banner: string | null
|
||||
recoveryState: string | null
|
||||
mounted: boolean
|
||||
nullStatusEnvironments: number
|
||||
connectionGeneration: number | null
|
||||
remoteControlState: string | null
|
||||
}
|
||||
|
||||
async function observePane(
|
||||
page: Page,
|
||||
webTabId: string,
|
||||
environmentId: string,
|
||||
startedAt: number
|
||||
): Promise<PaneObservation> {
|
||||
const observation = await page.evaluate(
|
||||
({ id, environmentId }) => {
|
||||
const manager = window.__paneManagers?.get(id)
|
||||
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
|
||||
const banner = document.querySelector('[data-terminal-remote-runtime-reconnect-banner]')
|
||||
const statuses = window.__store?.getState().runtimeStatusByEnvironmentId
|
||||
let nullStatusEnvironments = 0
|
||||
for (const entry of statuses?.values() ?? []) {
|
||||
if (entry.status === null) {
|
||||
nullStatusEnvironments += 1
|
||||
}
|
||||
}
|
||||
return {
|
||||
banner: banner?.getAttribute('data-terminal-remote-runtime-reconnect-banner') ?? null,
|
||||
recoveryState: pane?.container?.dataset?.ptyRecoveryState ?? null,
|
||||
mounted: Boolean(manager),
|
||||
nullStatusEnvironments,
|
||||
connectionGeneration: statuses?.get(environmentId)?.connectionGeneration ?? null,
|
||||
remoteControlState:
|
||||
statuses?.get(environmentId)?.status?.remoteControl?.state ??
|
||||
statuses?.get(environmentId)?.remoteControl?.state ??
|
||||
null
|
||||
}
|
||||
},
|
||||
{ id: webTabId, environmentId }
|
||||
)
|
||||
return { atMs: Date.now() - startedAt, ...observation }
|
||||
}
|
||||
|
||||
async function readPaneContent(page: Page, webTabId: string): Promise<string> {
|
||||
return page.evaluate((id) => {
|
||||
const manager = window.__paneManagers?.get(id)
|
||||
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
|
||||
return pane?.serializeAddon?.serialize?.() ?? ''
|
||||
}, webTabId)
|
||||
}
|
||||
|
||||
type RevealRecord = {
|
||||
timeline: PaneObservation[]
|
||||
paintedAtMs: number | null
|
||||
sawBanner: boolean
|
||||
sawNullStatus: boolean
|
||||
}
|
||||
|
||||
/** Samples the revealed pane for `budgetMs`, stopping early once `marker` paints. */
|
||||
async function recordRevealTimeline(
|
||||
page: Page,
|
||||
webTabId: string,
|
||||
environmentId: string,
|
||||
marker: string,
|
||||
budgetMs: number
|
||||
): Promise<RevealRecord> {
|
||||
const startedAt = Date.now()
|
||||
const timeline: PaneObservation[] = []
|
||||
let paintedAtMs: number | null = null
|
||||
let sawBanner = false
|
||||
let sawNullStatus = false
|
||||
let previous = ''
|
||||
while (Date.now() - startedAt < budgetMs) {
|
||||
const observation = await observePane(page, webTabId, environmentId, startedAt)
|
||||
sawBanner ||= observation.banner !== null
|
||||
sawNullStatus ||= observation.nullStatusEnvironments > 0
|
||||
const key = `${observation.banner}|${observation.recoveryState}|${observation.mounted}|${observation.nullStatusEnvironments}|${observation.connectionGeneration}|${observation.remoteControlState}`
|
||||
if (key !== previous) {
|
||||
timeline.push(observation)
|
||||
previous = key
|
||||
}
|
||||
if ((await readPaneContent(page, webTabId)).includes(marker)) {
|
||||
paintedAtMs = Date.now() - startedAt
|
||||
timeline.push(await observePane(page, webTabId, environmentId, startedAt))
|
||||
break
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
return { timeline, paintedAtMs, sawBanner, sawNullStatus }
|
||||
}
|
||||
|
||||
test('paired client tab switch does not reconnect the remote runtime', async ({
|
||||
orcaPage
|
||||
}, testInfo) => {
|
||||
test.setTimeout(900_000)
|
||||
const rawOffer = await createRuntimeDesktopPairingOffer(orcaPage)
|
||||
const link = await startRuntimeEndpointLinkFault(readPairingEndpoint(rawOffer.pairingUrl))
|
||||
const offer = {
|
||||
...rawOffer,
|
||||
pairingUrl: repointPairingUrl(rawOffer.pairingUrl, link.endpoint)
|
||||
}
|
||||
|
||||
const client = await launchPairedElectronClient(offer, testInfo, 'tab-switch-reconnect', {
|
||||
extraEnv: { ORCA_E2E_TERMINAL_PARKING_DELAY_MS: String(PARK_DELAY_MS) }
|
||||
})
|
||||
const createdTerminals: string[] = []
|
||||
try {
|
||||
const worktreeId = await orcaPage.evaluate(() => {
|
||||
const id = window.__store?.getState().activeWorktreeId
|
||||
if (!id) {
|
||||
throw new Error('headed host has no active worktree')
|
||||
}
|
||||
return id
|
||||
})
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
(id) =>
|
||||
window.__store
|
||||
?.getState()
|
||||
.allWorktrees()
|
||||
.some((worktree) => worktree.id === id) ?? false,
|
||||
worktreeId
|
||||
),
|
||||
{
|
||||
timeout: 60_000,
|
||||
message: 'paired client never saw the host worktree'
|
||||
}
|
||||
)
|
||||
.toBe(true)
|
||||
|
||||
const target = await createHostTerminal(client.page, client.environmentId, worktreeId)
|
||||
const decoys = [
|
||||
await createHostTerminal(client.page, client.environmentId, worktreeId),
|
||||
await createHostTerminal(client.page, client.environmentId, worktreeId)
|
||||
]
|
||||
createdTerminals.push(target.terminal, ...decoys.map((decoy) => decoy.terminal))
|
||||
|
||||
await openClientTab(client.page, worktreeId, target.webTabId)
|
||||
await expect
|
||||
.poll(() => readPaneContent(client.page, target.webTabId), {
|
||||
timeout: 60_000,
|
||||
message: 'target terminal never painted READY'
|
||||
})
|
||||
.toContain('READY')
|
||||
|
||||
const censusAfterFirstOpen = await readMultiplexCensus(client.page)
|
||||
console.log(
|
||||
`[tab-switch-repro] census after first open: ${JSON.stringify(censusAfterFirstOpen)}`
|
||||
)
|
||||
|
||||
// ---- Arm A: plain tab switching on a healthy link. No fault involved: if the
|
||||
// transport subscribe count climbs here, tab switching alone tears down and
|
||||
// re-establishes the environment's control channel.
|
||||
for (let round = 0; round < 3; round += 1) {
|
||||
await openClientTab(client.page, worktreeId, decoys[0].webTabId)
|
||||
await openClientTab(client.page, worktreeId, decoys[1].webTabId)
|
||||
await waitForTabParked(client.page, target.webTabId, {
|
||||
parkDelayMs: PARK_DELAY_MS
|
||||
})
|
||||
await openClientTab(client.page, worktreeId, target.webTabId)
|
||||
}
|
||||
const censusAfterHealthySwitching = await readMultiplexCensus(client.page)
|
||||
console.log(
|
||||
`[tab-switch-repro] census after healthy switching: ${JSON.stringify(censusAfterHealthySwitching)}`
|
||||
)
|
||||
console.log(
|
||||
`[tab-switch-repro] ARM A transportSubscribeCount delta over 3 healthy park/reveal rounds: ${censusAfterHealthySwitching.transportSubscribeCount - censusAfterFirstOpen.transportSubscribeCount} (unsubscribe delta ${censusAfterHealthySwitching.transportUnsubscribeCount - censusAfterFirstOpen.transportUnsubscribeCount})`
|
||||
)
|
||||
|
||||
// ---- Arm B: park every terminal pane for this environment, so nothing holds
|
||||
// the multiplexed control channel open, then reveal one under a link whose
|
||||
// established flows are fine but whose new dials hang.
|
||||
// Why not a plain view switch: cold-park exempts the single most-recently-hidden
|
||||
// tab, so one pane (and its stream) keeps the multiplexer alive, and the exemption
|
||||
// migrates to a parked tab the moment the exempt one closes. So: make a decoy the
|
||||
// exempt one, park the rest, put the link into stall-new, and only then close that
|
||||
// decoy on the host (over the established control flow). The multiplexer idles out
|
||||
// and nothing can re-dial it on a healthy path before the reveal.
|
||||
await openClientTab(client.page, worktreeId, decoys[1].webTabId)
|
||||
await client.page.evaluate(() => {
|
||||
window.__store?.getState().setActiveView('changes')
|
||||
})
|
||||
await waitForTabParked(client.page, target.webTabId, { parkDelayMs: PARK_DELAY_MS })
|
||||
await waitForTabParked(client.page, decoys[0].webTabId, { parkDelayMs: PARK_DELAY_MS })
|
||||
const censusBeforeRelease = await readMultiplexCensus(client.page)
|
||||
console.log(
|
||||
`[tab-switch-repro] census with only the exempt decoy mounted: ${JSON.stringify(censusBeforeRelease)}`
|
||||
)
|
||||
link.resetCounters()
|
||||
link.setMode('stall-new')
|
||||
await callEnvironment(client.page, client.environmentId, 'terminal.closeTab', {
|
||||
terminal: decoys[1].terminal
|
||||
})
|
||||
createdTerminals.splice(createdTerminals.indexOf(decoys[1].terminal), 1)
|
||||
await expect
|
||||
.poll(async () => (await readMultiplexCensus(client.page)).transportUnsubscribeCount, {
|
||||
timeout: 30_000,
|
||||
message: 'the terminal multiplexer was never released after its last stream closed'
|
||||
})
|
||||
.toBeGreaterThan(censusBeforeRelease.transportUnsubscribeCount)
|
||||
const censusAfterFullPark = await readMultiplexCensus(client.page)
|
||||
console.log(
|
||||
`[tab-switch-repro] census after every pane parked and the multiplexer released: ${JSON.stringify(censusAfterFullPark)}`
|
||||
)
|
||||
// Setup invariant, not a claim about the bug: the reveal below has to pay a fresh
|
||||
// dial, or Arm B degrades into "reveal a tab whose transport is still up".
|
||||
expect(
|
||||
censusAfterFullPark.transportUnsubscribeCount,
|
||||
'Arm B precondition: the terminal multiplexer must have been released before the stalled reveal'
|
||||
).toBeGreaterThan(censusBeforeRelease.transportUnsubscribeCount)
|
||||
|
||||
await selectClientTab(client.page, worktreeId, target.webTabId)
|
||||
const stalled = await recordRevealTimeline(
|
||||
client.page,
|
||||
target.webTabId,
|
||||
client.environmentId,
|
||||
'READY',
|
||||
40_000
|
||||
)
|
||||
console.log(
|
||||
`[tab-switch-repro] stalled reveal: painted=${String(stalled.paintedAtMs)} banner=${stalled.sawBanner} nullStatus=${stalled.sawNullStatus} timeline=${JSON.stringify(stalled.timeline)}`
|
||||
)
|
||||
console.log(
|
||||
`[tab-switch-repro] dials at the hop: accepted=${link.acceptedConnectionCount()} stalled=${link.stalledConnectionCount()}`
|
||||
)
|
||||
|
||||
// ---- Arm B2: the reporter's "reconnecting" state. Sever the established flows
|
||||
// once while new dials still hang, so the shared-control socket enters
|
||||
// `reconnecting` and the status recheck ladder arms. What the store does with
|
||||
// the resulting failed status.get is the writer under test: a live verdict must
|
||||
// not become `status: null` because the client could not ask.
|
||||
const generationBeforeFlap = (
|
||||
await observePane(client.page, target.webTabId, client.environmentId, 0)
|
||||
).connectionGeneration
|
||||
const dropped = link.dropEstablished()
|
||||
const flapped = await recordRevealTimeline(
|
||||
client.page,
|
||||
target.webTabId,
|
||||
client.environmentId,
|
||||
'READY',
|
||||
45_000
|
||||
)
|
||||
console.log(
|
||||
`[tab-switch-repro] link flap under stall-new: dropped=${dropped} banner=${flapped.sawBanner} nullStatus=${flapped.sawNullStatus} timeline=${JSON.stringify(flapped.timeline)}`
|
||||
)
|
||||
|
||||
// ---- Arm C: restore the link and measure time-to-recovery.
|
||||
link.setMode('pass')
|
||||
// Drive the documented recovery path. A pane whose attach timed out parks a retry that
|
||||
// arms no timer (#19872): it waits for online/resume/manual Reconnect, so within a bounded
|
||||
// budget the pane only comes back when one of those fires. Dispatching 'online' is exactly
|
||||
// that trigger — the same one a real network-return raises — so this arm exercises genuine
|
||||
// recovery, not the ~180s latch.
|
||||
await client.page.evaluate(() => window.dispatchEvent(new Event('online')))
|
||||
const recovered = await recordRevealTimeline(
|
||||
client.page,
|
||||
target.webTabId,
|
||||
client.environmentId,
|
||||
'READY',
|
||||
30_000
|
||||
)
|
||||
console.log(
|
||||
`[tab-switch-repro] recovery after link restored: painted=${String(recovered.paintedAtMs)} timeline=${JSON.stringify(recovered.timeline)}`
|
||||
)
|
||||
console.log(
|
||||
`[tab-switch-repro] census after recovery: ${JSON.stringify(await readMultiplexCensus(client.page))}`
|
||||
)
|
||||
console.log(`[tab-switch-repro] target sink: ${readSink(target.sinkPath).slice(0, 200)}`)
|
||||
const generationAfterRecovery = (
|
||||
await observePane(client.page, target.webTabId, client.environmentId, 0)
|
||||
).connectionGeneration
|
||||
console.log(
|
||||
`[tab-switch-repro] connection generation across the flap: ${String(generationBeforeFlap)} -> ${String(generationAfterRecovery)}`
|
||||
)
|
||||
|
||||
// Writer contract (#19647), the fix under test: a status.get that could not reach the host is
|
||||
// unverifiable, so the recorded live verdict survives the transport fault (no null published)
|
||||
// and recovery is not a second connection (the connection generation never advances). These
|
||||
// are the hard gate; both fail on the unfixed writer.
|
||||
expect(
|
||||
flapped.sawNullStatus,
|
||||
'a failed status.get over a stalled link published status: null over a live verdict (#19647)'
|
||||
).toBe(false)
|
||||
expect(
|
||||
generationAfterRecovery,
|
||||
'recovery advanced the connection generation, so the session-tabs mirror was rebuilt (#19647)'
|
||||
).toBe(generationBeforeFlap)
|
||||
// The fix keeps the environment revivable rather than retiring it: the pane is never disposed
|
||||
// and its host is never dropped from the mirror targets, so a later trigger can bring it back.
|
||||
const finalObservation = await observePane(
|
||||
client.page,
|
||||
target.webTabId,
|
||||
client.environmentId,
|
||||
0
|
||||
)
|
||||
expect(
|
||||
finalObservation.mounted,
|
||||
'the revealed pane was disposed instead of kept revivable'
|
||||
).toBe(true)
|
||||
// Recovery TIMELINE is diagnostic, NOT a pass/fail gate: whether the pane actually repaints
|
||||
// within budget depends on the un-park latch (#19872) and the multiplexer cold-handshake
|
||||
// (#19871), both filed separately and out of scope here. A green run is not a clean bill of
|
||||
// health — read the [tab-switch-repro] timeline above. This value is unchanged by this PR
|
||||
// (the unfixed writer left it null too), so it is logged, not asserted.
|
||||
console.log(
|
||||
`[tab-switch-repro] recovery-contract observation (gated on #19872/#19871): painted=${String(recovered.paintedAtMs)}`
|
||||
)
|
||||
} finally {
|
||||
link.setMode('pass')
|
||||
for (const terminal of createdTerminals) {
|
||||
await callEnvironment(client.page, client.environmentId, 'terminal.closeTab', {
|
||||
terminal
|
||||
}).catch(() => undefined)
|
||||
}
|
||||
await client.dispose()
|
||||
await link.close()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,124 @@
|
||||
import type { Locator, Page, TestInfo } from '@stablyai/playwright-test'
|
||||
import { test, expect } from './helpers/orca-app'
|
||||
import { waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
|
||||
const ENVIRONMENT_ID = 'e2e-remote-host'
|
||||
const HOST_LABEL = 'Remote Mac'
|
||||
|
||||
type RecordedState = 'no-entry' | 'probed-unreachable' | 'probed-reachable'
|
||||
|
||||
/** Put the seeded worktree's repo on a runtime host and record one status verdict for it. */
|
||||
async function seedRuntimeHost(page: Page, recorded: RecordedState): Promise<string> {
|
||||
return page.evaluate(
|
||||
({ environmentId, hostLabel, recorded }) => {
|
||||
const store = window.__store
|
||||
if (!store) {
|
||||
throw new Error('window.__store is not available')
|
||||
}
|
||||
const state = store.getState()
|
||||
const worktree = Object.values(state.worktreesByRepo).flat()[0]
|
||||
if (!worktree) {
|
||||
throw new Error('no seeded worktree to place on a runtime host')
|
||||
}
|
||||
state.setRuntimeEnvironments([
|
||||
{
|
||||
id: environmentId,
|
||||
name: hostLabel,
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
lastUsedAt: null,
|
||||
runtimeId: 'e2e-runtime',
|
||||
endpoints: [
|
||||
{ id: 'ws', kind: 'websocket', label: 'WebSocket', endpoint: 'ws://127.0.0.1:1' }
|
||||
],
|
||||
preferredEndpointId: 'ws'
|
||||
}
|
||||
])
|
||||
store.setState({
|
||||
repos: state.repos.map((repo) =>
|
||||
repo.id === worktree.repoId
|
||||
? { ...repo, connectionId: undefined, executionHostId: `runtime:${environmentId}` }
|
||||
: repo
|
||||
)
|
||||
})
|
||||
if (recorded === 'probed-unreachable') {
|
||||
state.setRuntimeEnvironmentStatus(environmentId, { status: null, checkedAt: Date.now() })
|
||||
}
|
||||
if (recorded === 'probed-reachable') {
|
||||
state.setRuntimeEnvironmentStatus(environmentId, {
|
||||
status: {
|
||||
runtimeId: 'e2e-runtime',
|
||||
rendererGraphEpoch: 1,
|
||||
graphStatus: 'ready',
|
||||
authoritativeWindowId: 1,
|
||||
desktopWindowStatus: 'available',
|
||||
liveTabCount: 0,
|
||||
liveLeafCount: 0
|
||||
},
|
||||
checkedAt: Date.now()
|
||||
})
|
||||
}
|
||||
return worktree.id
|
||||
},
|
||||
{ environmentId: ENVIRONMENT_ID, hostLabel: HOST_LABEL, recorded }
|
||||
)
|
||||
}
|
||||
|
||||
async function captureCard(card: Locator, testInfo: TestInfo, name: string): Promise<void> {
|
||||
const shot = testInfo.outputPath(name)
|
||||
await card.screenshot({ path: shot, animations: 'disabled' })
|
||||
await testInfo.attach(name, { path: shot, contentType: 'image/png' })
|
||||
}
|
||||
|
||||
test.describe('sidebar runtime host glyph', () => {
|
||||
test.beforeEach(async ({ orcaPage }) => {
|
||||
await waitForSessionReady(orcaPage)
|
||||
await waitForActiveWorktree(orcaPage)
|
||||
})
|
||||
|
||||
// Why: an absent entry means "not probed yet". Painting it destructive made every
|
||||
// remote card red and dimmed between launch and the first probe answering.
|
||||
test('does not call a runtime host disconnected before its first probe answers', async ({
|
||||
orcaPage
|
||||
}, testInfo) => {
|
||||
await seedRuntimeHost(orcaPage, 'no-entry')
|
||||
const card = orcaPage.locator(`[data-worktree-card-surface="true"]`).first()
|
||||
await expect(card).toBeVisible()
|
||||
|
||||
// Captured before the assertions so a regression run still yields the evidence image.
|
||||
await captureCard(card, testInfo, 'runtime-host-glyph-before-probe.png')
|
||||
|
||||
await expect(card.locator('svg.lucide-server')).toBeVisible()
|
||||
await expect(card.locator('svg.lucide-server-off')).toHaveCount(0)
|
||||
await expect(card).not.toHaveClass(/opacity-60/)
|
||||
})
|
||||
|
||||
// The deliberate no-change case: a probe actually reported this host unreachable.
|
||||
test('still marks a runtime host disconnected once a probe finds it unreachable', async ({
|
||||
orcaPage
|
||||
}, testInfo) => {
|
||||
await seedRuntimeHost(orcaPage, 'probed-unreachable')
|
||||
const card = orcaPage.locator(`[data-worktree-card-surface="true"]`).first()
|
||||
await expect(card).toBeVisible()
|
||||
|
||||
await captureCard(card, testInfo, 'runtime-host-glyph-disconnected.png')
|
||||
|
||||
await expect(card.locator('svg.lucide-server-off')).toBeVisible()
|
||||
await expect(card).toHaveClass(/opacity-60/)
|
||||
})
|
||||
|
||||
test('clears the disconnected glyph when a later probe reaches the host', async ({
|
||||
orcaPage
|
||||
}, testInfo) => {
|
||||
await seedRuntimeHost(orcaPage, 'probed-unreachable')
|
||||
const card = orcaPage.locator(`[data-worktree-card-surface="true"]`).first()
|
||||
await expect(card.locator('svg.lucide-server-off')).toBeVisible()
|
||||
|
||||
await seedRuntimeHost(orcaPage, 'probed-reachable')
|
||||
await captureCard(card, testInfo, 'runtime-host-glyph-recovered.png')
|
||||
|
||||
await expect(card.locator('svg.lucide-server')).toBeVisible()
|
||||
await expect(card.locator('svg.lucide-server-off')).toHaveCount(0)
|
||||
await expect(card).not.toHaveClass(/opacity-60/)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user