mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 16:03:49 +00:00
wip: remote-sidebar-offline
This commit is contained in:
@@ -27,8 +27,23 @@ import {
|
||||
getRuntimeClientEventEnvironmentIds,
|
||||
invalidateRuntimeClientEventReplay
|
||||
} from './runtime-environment-subscription-selection'
|
||||
import { dialRuntimeStatusReprobe } from './runtime-status-reprobe'
|
||||
import type { WorktreeEventRuntime } from './worktree-event-runtime'
|
||||
|
||||
/** Backoff for re-asking status.get after a probe that failed on its own socket. */
|
||||
const RUNTIME_STATUS_PROBE_RETRY_DELAYS_MS = [2_000, 10_000]
|
||||
|
||||
/**
|
||||
* Why a request carries its reason: `reconnected` must re-ask even when the cache reads
|
||||
* reachable (a restart changes the runtimeId under an unchanged-looking status), while
|
||||
* `recordedUnreachable` is satisfied by any answer that clears the offline verdict.
|
||||
*/
|
||||
type RuntimeStatusProbeTrigger = 'reconnected' | 'recordedUnreachable'
|
||||
|
||||
function isRuntimeStatusRecordedUnreachable(environmentId: string): boolean {
|
||||
return useAppStore.getState().runtimeStatusByEnvironmentId?.get(environmentId)?.status === null
|
||||
}
|
||||
|
||||
export function registerRuntimeClientIpcBridge(
|
||||
unsubs: (() => void)[],
|
||||
worktreeRuntime: WorktreeEventRuntime
|
||||
@@ -134,6 +149,103 @@ export function registerRuntimeClientIpcBridge(
|
||||
})
|
||||
}
|
||||
|
||||
const inFlightRuntimeStatusProbes = new Set<string>()
|
||||
const trailingRuntimeStatusProbes = new Map<string, RuntimeStatusProbeTrigger>()
|
||||
const runtimeStatusProbeRetryTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
||||
let runtimeStatusProbesStopped = false
|
||||
// Why not the desired-subscription set alone: a host that is not the active environment
|
||||
// drops out of it the moment anything records its status unreachable, so gating the
|
||||
// retry on it cancels the retry exactly where recovery matters. Only removal
|
||||
// (or a still-unhydrated catalog behind an active id) decides whether to keep asking,
|
||||
// and the tombstone covers the window where settings still names a deleted host active.
|
||||
const shouldProbeRuntimeStatus = (environmentId: string): boolean => {
|
||||
const state = useAppStore.getState()
|
||||
if (state.removedRuntimeEnvironmentIds?.has(environmentId)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
(state.runtimeEnvironments ?? []).some((environment) => environment.id === environmentId) ||
|
||||
getRuntimeClientEventEnvironmentIds(state).includes(environmentId)
|
||||
)
|
||||
}
|
||||
// Why: concurrent probes resolve in arbitrary order, so a slow one can publish its
|
||||
// stale answer over a newer one and leave the sidebar naming a superseded runtime.
|
||||
const probeRuntimeStatus = (
|
||||
environmentId: string,
|
||||
attempt = 0,
|
||||
trigger: RuntimeStatusProbeTrigger = 'reconnected'
|
||||
): void => {
|
||||
if (runtimeStatusProbesStopped || !shouldProbeRuntimeStatus(environmentId)) {
|
||||
return
|
||||
}
|
||||
if (inFlightRuntimeStatusProbes.has(environmentId)) {
|
||||
// Serialize, don't drop: the in-flight answer predates this request, so a reconnect
|
||||
// that lands mid-probe would otherwise go unasked — and a probe that succeeds
|
||||
// schedules no retry to pick it up later. A reconnect outranks a queued
|
||||
// recorded-unreachable request, which the in-flight answer may already settle.
|
||||
if (trigger === 'reconnected' || !trailingRuntimeStatusProbes.has(environmentId)) {
|
||||
trailingRuntimeStatusProbes.set(environmentId, trigger)
|
||||
}
|
||||
return
|
||||
}
|
||||
const pendingRetry = runtimeStatusProbeRetryTimers.get(environmentId)
|
||||
if (pendingRetry !== undefined) {
|
||||
clearTimeout(pendingRetry)
|
||||
runtimeStatusProbeRetryTimers.delete(environmentId)
|
||||
}
|
||||
inFlightRuntimeStatusProbes.add(environmentId)
|
||||
void dialRuntimeStatusReprobe(environmentId)
|
||||
.catch(() => false)
|
||||
.then((reachable) => {
|
||||
inFlightRuntimeStatusProbes.delete(environmentId)
|
||||
const trailingTrigger = trailingRuntimeStatusProbes.get(environmentId)
|
||||
if (trailingTrigger !== undefined) {
|
||||
trailingRuntimeStatusProbes.delete(environmentId)
|
||||
// A newer reconnect asked while this one was dialing: restart the attempt chain.
|
||||
// A resubscribe only asked because the cache read unreachable, so skip the extra
|
||||
// socket + E2EE handshake when this answer already cleared that.
|
||||
if (
|
||||
trailingTrigger === 'reconnected' ||
|
||||
isRuntimeStatusRecordedUnreachable(environmentId)
|
||||
) {
|
||||
probeRuntimeStatus(environmentId, 0, trailingTrigger)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Why: status.get dials its own short-lived socket, so it can fail while the
|
||||
// control transport that just proved the host is up stays healthy. That failure
|
||||
// is unverifiable and publishes nothing, so no store transition, resubscribe or
|
||||
// further trigger follows — without this bounded retry one unlucky probe leaves
|
||||
// a host already recorded offline stranded until the next transport gap.
|
||||
const retryDelayMs = RUNTIME_STATUS_PROBE_RETRY_DELAYS_MS[attempt]
|
||||
if (
|
||||
reachable ||
|
||||
retryDelayMs === undefined ||
|
||||
runtimeStatusProbesStopped ||
|
||||
!shouldProbeRuntimeStatus(environmentId)
|
||||
) {
|
||||
return
|
||||
}
|
||||
runtimeStatusProbeRetryTimers.set(
|
||||
environmentId,
|
||||
setTimeout(() => {
|
||||
runtimeStatusProbeRetryTimers.delete(environmentId)
|
||||
probeRuntimeStatus(environmentId, attempt + 1)
|
||||
}, retryDelayMs)
|
||||
)
|
||||
})
|
||||
}
|
||||
unsubs.push(() => {
|
||||
// The flag, not just the timers: a probe still in flight at teardown would
|
||||
// otherwise schedule a fresh retry chain after the bridge is gone.
|
||||
runtimeStatusProbesStopped = true
|
||||
trailingRuntimeStatusProbes.clear()
|
||||
for (const retryTimer of runtimeStatusProbeRetryTimers.values()) {
|
||||
clearTimeout(retryTimer)
|
||||
}
|
||||
runtimeStatusProbeRetryTimers.clear()
|
||||
})
|
||||
|
||||
const runtimeClientEventsSync = createRuntimeClientEventsSync({
|
||||
getDesiredEnvironmentIds: () => getRuntimeClientEventEnvironmentIds(useAppStore.getState()),
|
||||
getSubscriptionKey: (environmentId) => buildRuntimeClientEventEnvironmentKey([environmentId]),
|
||||
@@ -141,7 +253,7 @@ export function registerRuntimeClientIpcBridge(
|
||||
const sshGeneration = getEnvironmentSshStateGeneration(environmentId)
|
||||
const runtimeGeneration = getRuntimeEnvironmentConnectionGeneration(environmentId)
|
||||
const runtimeRevision = getRuntimeEnvironmentRevision(environmentId)
|
||||
return subscribeRuntimeClientEvents(
|
||||
const subscription = subscribeRuntimeClientEvents(
|
||||
environmentId,
|
||||
(event) => {
|
||||
if (
|
||||
@@ -156,6 +268,7 @@ export function registerRuntimeClientIpcBridge(
|
||||
() => {
|
||||
invalidateRuntimeClientEventReplay({
|
||||
getSshStateReference: () => useAppStore.getState().sshStateByEnvironment,
|
||||
refreshRuntimeStatus: () => probeRuntimeStatus(environmentId),
|
||||
requestProjectRefresh: () => runtimeProjectRefreshScheduler.request(environmentId),
|
||||
markEnvironmentSshStateStale: () =>
|
||||
useAppStore.getState().markEnvironmentSshStateStale(environmentId),
|
||||
@@ -165,6 +278,20 @@ export function registerRuntimeClientIpcBridge(
|
||||
})
|
||||
}
|
||||
)
|
||||
// Why: only a reconnect of an already-ready transport replays with the tag above.
|
||||
// A connection whose first ready lands after the host recovered (app started, or
|
||||
// the env was added, while it was down) never replays, so the recorded-unreachable
|
||||
// verdict this subscribe just disproved has to be re-asked here. Kept off the
|
||||
// returned promise so subscription registration/teardown ordering is unchanged.
|
||||
void subscription.then(
|
||||
() => {
|
||||
if (isRuntimeStatusRecordedUnreachable(environmentId)) {
|
||||
probeRuntimeStatus(environmentId, 0, 'recordedUnreachable')
|
||||
}
|
||||
},
|
||||
() => {}
|
||||
)
|
||||
return subscription
|
||||
},
|
||||
onEvent: handleRuntimeClientEvent
|
||||
})
|
||||
|
||||
@@ -178,6 +178,7 @@ export function createRuntimeEnvironmentStoreSyncSubscriber(
|
||||
|
||||
type RuntimeClientEventReplayInvalidationDeps = {
|
||||
getSshStateReference: () => RuntimeEnvironmentStoreSyncState['sshStateByEnvironment']
|
||||
refreshRuntimeStatus: () => void
|
||||
requestProjectRefresh: () => void
|
||||
markEnvironmentSshStateStale: () => void
|
||||
hydrateEnvironmentSshState: () => Promise<unknown>
|
||||
@@ -193,6 +194,10 @@ type RuntimeClientEventReplayInvalidationDeps = {
|
||||
export function invalidateRuntimeClientEventReplay(
|
||||
deps: RuntimeClientEventReplayInvalidationDeps
|
||||
): void {
|
||||
// Why: nothing else re-probes status.get after a reconnect, so a host recorded
|
||||
// unreachable during the gap stayed 'disconnected' in the sidebar forever while
|
||||
// its terminals and RPCs worked again.
|
||||
deps.refreshRuntimeStatus()
|
||||
deps.requestProjectRefresh()
|
||||
const previousSshStateReference = deps.getSshStateReference()
|
||||
deps.markEnvironmentSshStateStale()
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { toast } from 'sonner'
|
||||
import { useAppStore } from '@/store'
|
||||
import { buildExecutionHostRegistry } from '../../../../shared/execution-host-registry'
|
||||
import { toRuntimeExecutionHostId } from '../../../../shared/execution-host'
|
||||
import { createCompatibleRuntimeStatusResponse } from '@/runtime/runtime-compatibility-test-fixture'
|
||||
import { tagRuntimeSubscriptionReplayResponse } from '../../../../shared/runtime-subscription-replay'
|
||||
import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments'
|
||||
import type { RuntimeStatus } from '../../../../shared/runtime-types'
|
||||
import type { WorktreeEventRuntime } from './worktree-event-runtime'
|
||||
import { registerRuntimeClientIpcBridge } from './runtime-client-ipc-bridge'
|
||||
|
||||
vi.mock('sonner', () => ({
|
||||
toast: Object.assign(vi.fn(), {
|
||||
warning: vi.fn(),
|
||||
error: vi.fn(),
|
||||
dismiss: vi.fn()
|
||||
})
|
||||
}))
|
||||
|
||||
const ENVIRONMENT_ID = 'env-devbox'
|
||||
|
||||
function environment(): PublicKnownRuntimeEnvironment {
|
||||
return {
|
||||
id: ENVIRONMENT_ID,
|
||||
name: 'Remote devbox',
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
lastUsedAt: null,
|
||||
runtimeId: null,
|
||||
endpoints: [{ id: 'ws-a', kind: 'websocket', label: 'WebSocket', endpoint: 'ws://x' }],
|
||||
preferredEndpointId: 'ws-a'
|
||||
} as PublicKnownRuntimeEnvironment
|
||||
}
|
||||
|
||||
/** What the sidebar host header renders from: 'available' is online, 'disconnected' is the offline GUI. */
|
||||
function sidebarHostHealth(): string | undefined {
|
||||
const state = useAppStore.getState()
|
||||
return buildExecutionHostRegistry({
|
||||
repos: [],
|
||||
settings: state.settings,
|
||||
runtimeEnvironments: state.runtimeEnvironments,
|
||||
runtimeStatusByEnvironmentId: state.runtimeStatusByEnvironmentId
|
||||
}).find((host) => host.id === toRuntimeExecutionHostId(ENVIRONMENT_ID))?.health
|
||||
}
|
||||
|
||||
function liveRuntimeStatus(): RuntimeStatus {
|
||||
const response = createCompatibleRuntimeStatusResponse()
|
||||
if (!response.ok) {
|
||||
throw new Error('fixture must be a successful status response')
|
||||
}
|
||||
return response.result
|
||||
}
|
||||
|
||||
async function settle(): Promise<void> {
|
||||
for (let index = 0; index < 60; index += 1) {
|
||||
await Promise.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
describe('remote Orca server reconnect', () => {
|
||||
let unsubs: (() => void)[] = []
|
||||
let stopBridge: (() => void) | null = null
|
||||
let subscriptionResponders: {
|
||||
selector: string
|
||||
onResponse: (response: unknown) => void
|
||||
}[] = []
|
||||
let liveRuntimeId = 'remote-runtime'
|
||||
let failingStatusProbes = 0
|
||||
let blockNextStatusProbe = false
|
||||
let releaseBlockedStatusProbe: (() => void) | null = null
|
||||
|
||||
beforeEach(() => {
|
||||
subscriptionResponders = []
|
||||
unsubs = []
|
||||
liveRuntimeId = 'remote-runtime'
|
||||
failingStatusProbes = 0
|
||||
blockNextStatusProbe = false
|
||||
releaseBlockedStatusProbe = null
|
||||
// Module-level sonner double: without this a toast from an earlier test leaks into
|
||||
// the assertions below.
|
||||
vi.mocked(toast.warning).mockClear()
|
||||
vi.stubGlobal('window', {
|
||||
api: {
|
||||
runtimeEnvironments: {
|
||||
list: vi.fn(async () => [environment()]),
|
||||
getStatus: vi.fn(async () => {
|
||||
// Captured before the block so a probe that is still dialing answers with the
|
||||
// runtime it was dispatched against, not with whatever restarted meanwhile.
|
||||
const dispatchedRuntimeId = liveRuntimeId
|
||||
if (blockNextStatusProbe) {
|
||||
blockNextStatusProbe = false
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseBlockedStatusProbe = resolve
|
||||
})
|
||||
}
|
||||
if (failingStatusProbes > 0) {
|
||||
failingStatusProbes -= 1
|
||||
// status.get dials its own socket; it can fail while the control transport is up.
|
||||
return {
|
||||
id: 'status.get',
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'runtime_unavailable',
|
||||
message: 'probe socket refused'
|
||||
},
|
||||
_meta: { runtimeId: null }
|
||||
}
|
||||
}
|
||||
return createCompatibleRuntimeStatusResponse(dispatchedRuntimeId)
|
||||
}),
|
||||
call: vi.fn(async () => ({ id: 'x', ok: true, result: [] })),
|
||||
subscribe: vi.fn(
|
||||
async (
|
||||
args: { selector: string },
|
||||
callbacks: { onResponse: (response: unknown) => void }
|
||||
): Promise<{ unsubscribe: () => void }> => {
|
||||
subscriptionResponders.push({
|
||||
selector: args.selector,
|
||||
onResponse: callbacks.onResponse
|
||||
})
|
||||
return { unsubscribe: vi.fn() }
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
useAppStore.setState({
|
||||
settings: { activeRuntimeEnvironmentId: ENVIRONMENT_ID } as never,
|
||||
runtimeEnvironments: [environment()],
|
||||
// Cleared explicitly: the removal case leaves a tombstone in this module-level
|
||||
// store, which would silently suppress every later test's probes.
|
||||
removedRuntimeEnvironmentIds: new Set(),
|
||||
runtimeStatusByEnvironmentId: new Map([
|
||||
[ENVIRONMENT_ID, { status: liveRuntimeStatus(), checkedAt: Date.now() }]
|
||||
]) as never
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
stopBridge?.()
|
||||
stopBridge = null
|
||||
for (const unsub of unsubs.splice(0)) {
|
||||
unsub()
|
||||
}
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
/** Delivers the replay-tagged first response a shared-control reconnect produces. */
|
||||
function replaySubscription(selector = ENVIRONMENT_ID): void {
|
||||
const responder = subscriptionResponders.findLast((entry) => entry.selector === selector)
|
||||
if (!responder) {
|
||||
throw new Error(`no client-event subscription for ${selector}`)
|
||||
}
|
||||
responder.onResponse(
|
||||
tagRuntimeSubscriptionReplayResponse({
|
||||
id: 'sub',
|
||||
ok: true,
|
||||
result: { type: 'ready', snapshot: { sshStates: [] } }
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function startBridge(): void {
|
||||
stopBridge = registerRuntimeClientIpcBridge(unsubs, {
|
||||
worktreeChangeRefreshQueue: { enqueue: vi.fn() },
|
||||
activateNotifiedWorktree: vi.fn()
|
||||
} as unknown as WorktreeEventRuntime)
|
||||
}
|
||||
|
||||
it('re-probes a replayed subscription while the cached status still looks reachable', async () => {
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
|
||||
// The gap was short enough that nothing probed during it, so the cached status still
|
||||
// names the pre-restart runtime and the replay tag is the only evidence it is stale.
|
||||
liveRuntimeId = 'remote-runtime-restarted'
|
||||
replaySubscription()
|
||||
await settle()
|
||||
|
||||
expect(
|
||||
useAppStore.getState().runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status?.runtimeId
|
||||
).toBe('remote-runtime-restarted')
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
})
|
||||
|
||||
it('returns the sidebar host to online when the first subscription lands untagged', async () => {
|
||||
// A connection that was never ready does not replay: nothing tags its first
|
||||
// response, so a client that booted while the host was down has only the
|
||||
// successful subscribe as evidence that the recorded verdict is stale.
|
||||
useAppStore.setState({
|
||||
runtimeStatusByEnvironmentId: new Map([
|
||||
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
|
||||
]) as never
|
||||
})
|
||||
expect(sidebarHostHealth()).toBe('disconnected')
|
||||
|
||||
startBridge()
|
||||
await settle()
|
||||
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
expect(subscriptionResponders.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('re-asks after a probe that failed while the transport stayed up', async () => {
|
||||
vi.useFakeTimers()
|
||||
// Already recorded unreachable, so the failing probe's `null` is an unchanged
|
||||
// re-publication: it writes nothing and leaves no store transition for the
|
||||
// resubscribe path to key off. Only a retry can still recover this host.
|
||||
useAppStore.setState({
|
||||
runtimeStatusByEnvironmentId: new Map([
|
||||
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
|
||||
]) as never
|
||||
})
|
||||
failingStatusProbes = 1
|
||||
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(sidebarHostHealth()).toBe('disconnected')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
await settle()
|
||||
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
})
|
||||
|
||||
it('stops re-asking a host that keeps refusing, instead of polling it forever', async () => {
|
||||
vi.useFakeTimers()
|
||||
useAppStore.setState({
|
||||
runtimeStatusByEnvironmentId: new Map([
|
||||
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
|
||||
]) as never
|
||||
})
|
||||
failingStatusProbes = Number.POSITIVE_INFINITY
|
||||
|
||||
startBridge()
|
||||
await settle()
|
||||
await vi.advanceTimersByTimeAsync(120_000)
|
||||
await settle()
|
||||
|
||||
// One probe on the successful subscribe plus the two bounded retries.
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('stops probing once the bridge is torn down mid-probe', async () => {
|
||||
vi.useFakeTimers()
|
||||
useAppStore.setState({
|
||||
runtimeStatusByEnvironmentId: new Map([
|
||||
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
|
||||
]) as never
|
||||
})
|
||||
failingStatusProbes = Number.POSITIVE_INFINITY
|
||||
blockNextStatusProbe = true
|
||||
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
|
||||
// Teardown clears scheduled retries, but this probe has not answered yet.
|
||||
stopBridge?.()
|
||||
stopBridge = null
|
||||
for (const unsub of unsubs.splice(0)) {
|
||||
unsub()
|
||||
}
|
||||
releaseBlockedStatusProbe?.()
|
||||
await settle()
|
||||
await vi.advanceTimersByTimeAsync(120_000)
|
||||
await settle()
|
||||
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('re-asks for a saved host that is not the active environment', async () => {
|
||||
vi.useFakeTimers()
|
||||
// A non-active host is only in the desired-subscription set while its status is non-null,
|
||||
// so gating the retry on that set would strand it offline with its subscription already
|
||||
// torn down the moment anything else (an explicit disconnect, the toast's own retry)
|
||||
// records the same outage.
|
||||
useAppStore.setState({
|
||||
settings: { activeRuntimeEnvironmentId: 'env-laptop' } as never
|
||||
})
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
|
||||
failingStatusProbes = 1
|
||||
replaySubscription()
|
||||
await settle()
|
||||
useAppStore.getState().setRuntimeEnvironmentStatus(ENVIRONMENT_ID, {
|
||||
status: null,
|
||||
checkedAt: Date.now()
|
||||
})
|
||||
expect(sidebarHostHealth()).toBe('disconnected')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
await settle()
|
||||
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
})
|
||||
|
||||
it('re-asks after a reconnect that lands while an earlier probe is still dialing', async () => {
|
||||
blockNextStatusProbe = true
|
||||
startBridge()
|
||||
await settle()
|
||||
replaySubscription()
|
||||
await settle()
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
|
||||
// The host restarts while that probe is still on its own socket: the transport drops,
|
||||
// reconnects and replays again. Serializing is right, dropping the request is not —
|
||||
// the in-flight answer predates the restart this replay is reporting.
|
||||
liveRuntimeId = 'remote-runtime-restarted'
|
||||
replaySubscription()
|
||||
await settle()
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
|
||||
releaseBlockedStatusProbe?.()
|
||||
await settle()
|
||||
|
||||
expect(
|
||||
useAppStore.getState().runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status?.runtimeId
|
||||
).toBe('remote-runtime-restarted')
|
||||
})
|
||||
|
||||
it('does not resurrect a host removed while a retry was pending', async () => {
|
||||
vi.useFakeTimers()
|
||||
useAppStore.setState({
|
||||
runtimeStatusByEnvironmentId: new Map([
|
||||
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
|
||||
]) as never
|
||||
})
|
||||
failingStatusProbes = Number.POSITIVE_INFINITY
|
||||
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
|
||||
// The user deletes the remote server before the first retry fires. Settings can still
|
||||
// name it as active until the next settings read, so removal is what has to stop this.
|
||||
useAppStore.getState().setRuntimeEnvironments([])
|
||||
await settle()
|
||||
expect(useAppStore.getState().runtimeStatusByEnvironmentId.has(ENVIRONMENT_ID)).toBe(false)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(120_000)
|
||||
await settle()
|
||||
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
// buildExecutionHostRegistry enumerates the status map, so a re-published entry for a
|
||||
// deleted id puts that host back in the sidebar under its raw id.
|
||||
expect(useAppStore.getState().runtimeStatusByEnvironmentId.has(ENVIRONMENT_ID)).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps a live cached status when the replay-triggered probe fails on its own socket', async () => {
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
// Asserted over every write, not just the end state: a demotion that a later probe
|
||||
// undoes still flashed the sidebar offline and still fired the toast.
|
||||
let recordedUnreachable = false
|
||||
unsubs.push(
|
||||
useAppStore.subscribe((state) => {
|
||||
recordedUnreachable ||=
|
||||
state.runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status === null
|
||||
})
|
||||
)
|
||||
|
||||
// status.get dials its own short-lived socket, so its failure is unverifiable — and the
|
||||
// transport that just replayed is proof the host is up. Recording it as offline would
|
||||
// manufacture the stuck-offline sidebar this re-probe exists to cure.
|
||||
failingStatusProbes = 1
|
||||
replaySubscription()
|
||||
await settle()
|
||||
|
||||
expect(recordedUnreachable).toBe(false)
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
expect(toast.warning).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns a stuck-offline host to online when the replayed probe answers', async () => {
|
||||
// The reported bug: the sidebar stayed offline after the connection recovered. The first
|
||||
// probe failing keeps the recorded verdict unreachable, so only the replay recovers it
|
||||
// (its retry chain is still parked behind a 2s timer this test never advances).
|
||||
useAppStore.setState({
|
||||
runtimeStatusByEnvironmentId: new Map([
|
||||
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
|
||||
]) as never
|
||||
})
|
||||
failingStatusProbes = 1
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(sidebarHostHealth()).toBe('disconnected')
|
||||
|
||||
replaySubscription()
|
||||
await settle()
|
||||
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
})
|
||||
|
||||
it('does not dial a second status socket when the in-flight probe already answered', async () => {
|
||||
startBridge()
|
||||
await settle()
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(0)
|
||||
|
||||
// A reconnect re-probes unconditionally, and that probe is still on its own socket.
|
||||
blockNextStatusProbe = true
|
||||
replaySubscription()
|
||||
await settle()
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
|
||||
// Meanwhile the outage's own failed probe is recorded, which resubscribes; that
|
||||
// resubscribe resolves against a cache that still reads unreachable.
|
||||
useAppStore.getState().setRuntimeEnvironmentStatus(ENVIRONMENT_ID, {
|
||||
status: null,
|
||||
checkedAt: Date.now()
|
||||
})
|
||||
await settle()
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
|
||||
releaseBlockedStatusProbe?.()
|
||||
await settle()
|
||||
|
||||
// The resubscribe only wanted an answer for a host the cache called unreachable, and
|
||||
// the probe it waited on gave one: a second status.get is a whole extra socket dial.
|
||||
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
|
||||
expect(sidebarHostHealth()).toBe('available')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay'
|
||||
import { ensureBrowserClientHostsForRestoredPages } from '@/runtime/restored-client-hosted-browser-host-attach'
|
||||
import { refreshRuntimeEnvironmentStatus } from '@/store/slices/runtime-status-refresh'
|
||||
import { useAppStore } from '../../store'
|
||||
|
||||
/** Same wait as the store's status action, so a re-probe is no more impatient than any status.get. */
|
||||
const RUNTIME_STATUS_REPROBE_TIMEOUT_MS = 10_000
|
||||
|
||||
/**
|
||||
* Re-asks `status.get` after a reconnect and publishes only what it actually verified.
|
||||
*
|
||||
* The store's `refreshRuntimeEnvironmentStatus` records a failed probe as `null`, which is right
|
||||
* for a user-initiated check but wrong here: this probe dials its own short-lived socket with a
|
||||
* fresh handshake (`src/shared/remote-runtime-request-socket.ts`), so it can fail while the
|
||||
* control transport that just replayed is demonstrably alive. That failure is `unverifiable`, not
|
||||
* evidence the host is down — publishing it over a live cached verdict demotes the sidebar to
|
||||
* offline behind a "Can't reach" toast, which is the exact symptom this re-probe exists to cure.
|
||||
* A failed probe therefore leaves the cached verdict untouched and the caller's retry chain settles
|
||||
* it. Resolves to whether the host answered, so callers can drive that chain.
|
||||
*/
|
||||
export function dialRuntimeStatusReprobe(environmentId: string): Promise<boolean> {
|
||||
return refreshRuntimeEnvironmentStatus(
|
||||
environmentId,
|
||||
RUNTIME_STATUS_REPROBE_TIMEOUT_MS,
|
||||
(status) => {
|
||||
if (status === null) {
|
||||
return
|
||||
}
|
||||
useAppStore
|
||||
.getState()
|
||||
.setRuntimeEnvironmentStatus(environmentId, { status, checkedAt: Date.now() })
|
||||
// Mirrors the store action's recovery follow-ups: the host that just answered may be holding
|
||||
// pages this desktop hosts, and closes it never heard must not be lost to the reattach.
|
||||
void ensureBrowserClientHostsForRestoredPages(useAppStore.getState())
|
||||
void replayClientHostedBrowserCloseIntents(environmentId, useAppStore.getState())
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -303,6 +303,7 @@ describe('createRuntimeEnvironmentStoreSyncSubscriber', () => {
|
||||
describe('invalidateRuntimeClientEventReplay', () => {
|
||||
it('explicitly syncs an advanced SSH generation when stale publication is a no-op and hydration fails', async () => {
|
||||
const sshStateReference = new Map()
|
||||
const refreshRuntimeStatus = vi.fn()
|
||||
const requestProjectRefresh = vi.fn()
|
||||
const markEnvironmentSshStateStale = vi.fn()
|
||||
const sync = vi.fn()
|
||||
@@ -312,6 +313,7 @@ describe('invalidateRuntimeClientEventReplay', () => {
|
||||
|
||||
invalidateRuntimeClientEventReplay({
|
||||
getSshStateReference: () => sshStateReference,
|
||||
refreshRuntimeStatus,
|
||||
requestProjectRefresh,
|
||||
markEnvironmentSshStateStale,
|
||||
hydrateEnvironmentSshState,
|
||||
@@ -319,6 +321,7 @@ describe('invalidateRuntimeClientEventReplay', () => {
|
||||
})
|
||||
await Promise.resolve()
|
||||
|
||||
expect(refreshRuntimeStatus).toHaveBeenCalledOnce()
|
||||
expect(requestProjectRefresh).toHaveBeenCalledOnce()
|
||||
expect(markEnvironmentSshStateStale).toHaveBeenCalledOnce()
|
||||
expect(sync).toHaveBeenCalledOnce()
|
||||
@@ -331,6 +334,7 @@ describe('invalidateRuntimeClientEventReplay', () => {
|
||||
|
||||
invalidateRuntimeClientEventReplay({
|
||||
getSshStateReference: () => sshStateReference,
|
||||
refreshRuntimeStatus: vi.fn(),
|
||||
requestProjectRefresh: vi.fn(),
|
||||
markEnvironmentSshStateStale: () => {
|
||||
sshStateReference = new Map()
|
||||
|
||||
Reference in New Issue
Block a user