fix(runtime): let connections own host status recovery (#20003)

* fix(runtime): let connections own host status recovery

Verify runtime status after authenticated connection recovery and publish
ordered snapshots to desktop and browser viewers. Consolidate failed-status
retries in the connection owner and remove renderer retry/diagnostics merging.

Adapt sidebar host-state derivation and regression coverage from Omar
Shahine's original fix in https://github.com/stablyai/orca/pull/19163.

Co-authored-by: Omar Shahine <10343873+omarshahine@users.noreply.github.com>

* fix(runtime): show blocked hosts honestly and remove obsolete status options

* fix(runtime): preserve timeout guidance and update IPC test fixtures

* fix(runtime): preserve status evidence and address review gaps

* test(sidebar): assert workspace host icons dimming and recovery tooltips

* fix(palette): require available hosts before adding implicit badges

* fix: retain disconnected host snapshots for new renderers

---------

Co-authored-by: Omar Shahine <10343873+omarshahine@users.noreply.github.com>
This commit is contained in:
Jinwoo Hong
2026-09-11 03:19:01 -04:00
committed by GitHub
co-authored by Omar Shahine
parent b6e4457552
commit 3b82d8de64
76 changed files with 2315 additions and 1612 deletions
+75
View File
@@ -10,6 +10,81 @@
}
},
"gates": [
{
"id": "runtime.connection-owned-host-status",
"title": "Host status recovers with its owning connection",
"maturity": "experimental",
"protection": "partial",
"owner": "runtime",
"layer": "service-integration-and-e2e",
"surfaces": [
"sidebar host status",
"desktop runtime connection",
"browser primary connection"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["remote-runtime"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["remote-runtime"],
"coverageNotes": "Real authenticated sockets plus isolated desktop and headless hosts with desktop and browser viewers; deterministic lifecycle tests cover stale results and reader deadlines.",
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19163"],
"invariant": "Failed bootstrap and authenticated reconnect converge without UI triggers; one connection owner publishes verified status, with no independent healthy status polling.",
"oracle": "Observe automatic recovery, retained runtime identity on failure, ordered publications, exact request counts, isolated viewer outages, and retirement on disconnect.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm test src/shared/runtime-host-status-owner.test.ts src/main/ipc/runtime-environment-status-recovery.test.ts src/main/ipc/runtime-environment-status-connection.test.ts src/renderer/src/store/slices/runtime-status-snapshot.test.ts src/renderer/src/web/web-runtime-status-owner.test.ts",
"ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1"
],
"testFiles": [
"src/shared/runtime-host-status-owner.test.ts",
"src/main/ipc/runtime-environment-status-recovery.test.ts",
"src/main/ipc/runtime-environment-status-connection.test.ts",
"src/renderer/src/store/slices/runtime-status-snapshot.test.ts",
"src/renderer/src/web/web-runtime-status-owner.test.ts",
"tests/e2e/runtime-host-status-recovery.spec.ts"
],
"assertionRefs": [
{
"file": "src/main/ipc/runtime-environment-status-recovery.test.ts",
"assertions": [
"recovers a saved host after its first status check fails, without another UI request"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-10",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"result": "passed",
"summary": "Desktop-host and headless-host journeys passed with desktop and browser viewers.",
"durationSeconds": 31.3
}
],
"runtimeBudget": {
"p95Seconds": 180,
"scope": "Target excluding builds; measured p95 not established."
},
"flakeHistory": {
"status": "soaking",
"evidence": "Local candidate runs passed; no sustained CI history yet."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "First-status-failure oracle failed on main 58ff95becb40 (one request instead of two) and passes on the candidate. E2E verifies candidate recovery, not a baseline comparison."
},
"performanceBudget": {
"required": false,
"evidence": "Deterministic tests assert one shared request and no healthy owner polling."
},
"promotionCriteria": ["Collect repeated CI runs without unexplained flakes."],
"knownGaps": [
"No live Linux, Windows, SSH, or mixed-version pair validation.",
"TCP interruption exercises reconnect, not a full real host process restart.",
"The outage begins on the first saved-host check, not by relaunching a preseeded desktop profile."
],
"demotionRule": "Keep experimental until repeated runs establish reliability; preserve request-count and lifecycle assertions."
},
{
"id": "mobile-push.headless-startup-and-policy",
"title": "Headless push lifecycle and mobile delivery policy",
@@ -1,4 +1,4 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { beforeEach, describe, expect, it } from 'vitest'
import type { PairingOffer } from '../../shared/pairing'
import {
advanceRuntimeEnvironmentCapabilityIncarnation,
@@ -17,7 +17,6 @@ describe('runtime environment capability evidence', () => {
it('accepts evidence by dispatch order instead of completion order', () => {
const older = captureRuntimeEnvironmentCapabilityEvidence('env', pairing())
const newer = captureRuntimeEnvironmentCapabilityEvidence('env', pairing())
const pause = vi.fn()
expect(
applyRuntimeEnvironmentCapabilityVerdict({
@@ -30,12 +29,10 @@ describe('runtime environment capability evidence', () => {
applyRuntimeEnvironmentCapabilityVerdict({
evidence: older,
verdict: 'absent',
runtimeId: 'runtime-old',
onAbsent: pause
runtimeId: 'runtime-old'
})
).toBe(false)
expect(pause).not.toHaveBeenCalled()
expect(isRuntimeEnvironmentCapabilityPaused('env')).toBe(false)
})
@@ -68,8 +68,6 @@ export function applyRuntimeEnvironmentCapabilityVerdict(args: {
evidence: RuntimeEnvironmentCapabilityEvidence
verdict: RuntimeEnvironmentCapabilityVerdict
runtimeId: string
onCapable?: () => void
onAbsent?: () => void
}): boolean {
const state = stateFor(args.evidence.environmentId)
if (
@@ -83,11 +81,6 @@ export function applyRuntimeEnvironmentCapabilityVerdict(args: {
verdict: args.verdict,
runtimeId: args.runtimeId
}
if (args.verdict === 'capable') {
args.onCapable?.()
} else {
args.onAbsent?.()
}
return true
}
@@ -20,6 +20,8 @@ import { verifyAndAddRuntimeEnvironmentFromPairingCode } from './runtime-environ
import { clearRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence'
import {
closeRemoteRuntimeRequestConnection,
getRuntimeEnvironmentStatusOwner,
getRuntimeEnvironmentStatusSnapshots,
retryRemoteRuntimeSharedControlConnectionNow
} from './runtime-environment-request-connections'
import {
@@ -29,7 +31,6 @@ import {
} from './runtime-environment-manual-disconnect'
import {
callRuntimeEnvironment,
clearSharedControlSupport,
getRuntimeEnvironmentStatus
} from './runtime-environment-transport-routing'
@@ -60,6 +61,9 @@ export function registerRuntimeEnvironmentConnectivityHandlers({
getUserDataPath,
invalidateTransport
}: ConnectivityHandlerOptions): void {
ipcMain.handle('runtimeEnvironments:getStatusSnapshots', () =>
getRuntimeEnvironmentStatusSnapshots()
)
ipcMain.handle('runtimeEnvironments:list', () =>
listEnvironments(getUserDataPath()).map(redactRuntimeEnvironment)
)
@@ -80,6 +84,12 @@ export function registerRuntimeEnvironmentConnectivityHandlers({
const result = await verifyAndAddRuntimeEnvironmentFromPairingCode(getUserDataPath(), args)
if (result.ok) {
clearRuntimeEnvironmentManualDisconnect(result.environment.id)
getRuntimeEnvironmentStatusOwner(getUserDataPath(), result.environment.id).acceptVerified({
id: 'status.get',
ok: true,
result: result.runtimeStatus,
_meta: { runtimeId: result.runtimeStatus.runtimeId }
})
}
return result
}
@@ -121,6 +131,8 @@ export function registerRuntimeEnvironmentConnectivityHandlers({
markRuntimeEnvironmentManuallyDisconnected(environment.id)
invalidateTransport(environment.id)
closeLegacySelectorTransport(args.selector, environment.id)
// Retain disconnected evidence for renderers that missed the teardown event.
getRuntimeEnvironmentStatusOwner(getUserDataPath(), environment.id)
return { disconnected: redactRuntimeEnvironment(environment) }
}
)
@@ -132,7 +144,9 @@ export function registerRuntimeEnvironmentConnectivityHandlers({
): Promise<RuntimeRpcResponse<RuntimeStatus>> => {
const environment = resolveEnvironment(getUserDataPath(), args.selector)
clearRuntimeEnvironmentManualDisconnect(environment.id)
return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs)
return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs, {
reconnect: true
})
}
)
ipcMain.handle(
@@ -156,7 +170,6 @@ function closeLegacySelectorTransport(selector: string, environmentId: string):
return
}
closeRemoteRuntimeRequestConnection(selector)
clearSharedControlSupport(selector)
}
function registerPassiveStatusHandler(getUserDataPath: () => string): void {
@@ -1,3 +1,5 @@
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } }))
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -20,14 +22,17 @@ vi.mock('../../shared/remote-runtime-client', () => ({
sendRemoteRuntimeRequest: sendRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: vi.fn(),
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
reconnectRemoteRuntimeSharedControlConnection: vi.fn(),
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn()
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: vi.fn(),
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
reconnectRemoteRuntimeSharedControlConnection: vi.fn(),
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn()
})
})
import {
callRuntimeEnvironment,
@@ -55,6 +60,7 @@ describe('federated read RPC transport routing', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -9,6 +9,7 @@ export const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [
'runtimeEnvironments:retryControlConnection',
'runtimeEnvironments:prepareBrowserClientHostPlacement',
'runtimeEnvironments:getStatus',
'runtimeEnvironments:getStatusSnapshots',
'runtimeEnvironments:call',
'runtimeEnvironments:subscribe',
'runtimeEnvironments:unsubscribe'
@@ -47,9 +47,9 @@ describe('runtime environment shared-control connection cache', () => {
applyRuntimeEnvironmentCapabilityVerdict({
evidence: absent,
verdict: 'absent',
runtimeId: 'runtime-test',
onAbsent: () => pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID)
runtimeId: 'runtime-test'
})
pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID)
expect(getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state).toBe('closed')
await delay(400)
expect(server.connectionCount()).toBe(1)
@@ -58,12 +58,10 @@ describe('runtime environment shared-control connection cache', () => {
applyRuntimeEnvironmentCapabilityVerdict({
evidence: capable,
verdict: 'capable',
runtimeId: 'runtime-test',
onCapable: () => {
ensureRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID, server.pairing)
reconnectRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID)
}
runtimeId: 'runtime-test'
})
ensureRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID, server.pairing)
reconnectRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID)
await waitFor(() => server.connectionCount() === 2)
})
@@ -119,9 +117,9 @@ describe('runtime environment shared-control connection cache', () => {
applyRuntimeEnvironmentCapabilityVerdict({
evidence,
verdict: 'absent',
runtimeId: 'runtime-test',
onAbsent: () => pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID)
runtimeId: 'runtime-test'
})
pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID)
expect(getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state).toBe('reconnecting')
await waitFor(() => server.connectionCount() === 2)
@@ -1,4 +1,9 @@
import type { PairingOffer } from '../../shared/pairing'
import { resolveEnvironment } from '../../shared/runtime-environment-store'
import { getPreferredPairingOffer } from '../../shared/runtime-environments'
import type { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner'
import type { RuntimeStatus } from '../../shared/runtime-types'
import { createRuntimeEnvironmentStatusOwner } from './runtime-environment-status-owner'
import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/protocol-version'
import type {
RuntimeOrchestrationEnvelope,
@@ -30,6 +35,56 @@ type CachedSharedControlConnection = {
const requestConnections = new Map<string, CachedRuntimeConnection>()
const sharedControlConnections = new Map<string, CachedSharedControlConnection>()
const statusOwners = new Map<string, { key: string; owner: RuntimeHostStatusOwner }>()
export function getRuntimeEnvironmentStatusOwner(
userDataPath: string,
selector: string
): RuntimeHostStatusOwner {
const environment = resolveEnvironment(userDataPath, selector)
const pairing = getPreferredPairingOffer(environment)
const key = `${userDataPath}\0${environment.pairingRevision ?? environment.createdAt}\0${getPairingKey(pairing)}`
let cached = statusOwners.get(environment.id)
if (!cached || cached.key !== key || cached.owner.read().retired) {
if (cached) {
closeRemoteRuntimeRequestConnection(environment.id)
}
const owner = createRuntimeEnvironmentStatusOwner(userDataPath, environment, {
isReady: () => getRemoteRuntimeSharedControlDiagnostics(environment.id)?.state === 'ready',
request: (signal) =>
sendRemoteRuntimeSharedControlRequest<RuntimeStatus>(
environment.id,
pairing,
'status.get',
undefined,
15_000,
undefined,
signal
),
establish: () => {
ensureRemoteRuntimeSharedControlConnection(environment.id, pairing)
reconnectRemoteRuntimeSharedControlConnection(environment.id)
},
pause: () => pauseRemoteRuntimeSharedControlRetry(environment.id)
})
cached = { key, owner }
statusOwners.set(environment.id, cached)
if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
owner.dispose()
}
}
return cached.owner
}
export function resetRuntimeEnvironmentStatusOwners(): void {
for (const id of statusOwners.keys()) {
closeRemoteRuntimeRequestConnection(id)
}
}
export function getRuntimeEnvironmentStatusSnapshots() {
return [...statusOwners.values()].map(({ owner }) => owner.read())
}
export function sendRemoteRuntimeConnectionRequest<TResult>(
environmentId: string,
@@ -56,6 +111,9 @@ export function sendRemoteRuntimeConnectionRequest<TResult>(
}
export function closeRemoteRuntimeRequestConnection(environmentId: string): void {
const status = statusOwners.get(environmentId)
statusOwners.delete(environmentId)
status?.owner.dispose()
const cached = requestConnections.get(environmentId)
requestConnections.delete(environmentId)
cached?.connection.close()
@@ -166,6 +224,16 @@ function getSharedControlConnection(
transportGeneration,
diagnostics
})
statusOwners
.get(environmentId)
?.owner.connectionChanged(
diagnostics.state === 'ready'
? 'ready'
: diagnostics.state === 'closed' || diagnostics.state === 'reconnecting'
? 'disconnected'
: 'connecting',
diagnostics
)
}
})
}
@@ -1,39 +1,23 @@
import {
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES,
REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY
} from '../../shared/protocol-version'
import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client'
import { markEnvironmentUsed } from '../../shared/runtime-environment-store'
import type {
getPreferredPairingOffer,
KnownRuntimeEnvironment
} from '../../shared/runtime-environments'
import type { RuntimeStatus } from '../../shared/runtime-types'
import { RemoteRuntimeClientError } from '../../shared/remote-runtime-client-error'
import {
applyRuntimeEnvironmentCapabilityVerdict,
captureRuntimeEnvironmentCapabilityEvidence,
getAcceptedRuntimeEnvironmentCapabilityOutcome,
isRuntimeEnvironmentCapabilityOutcomeCurrent,
runtimeEnvironmentCapabilityOutcome,
resetRuntimeEnvironmentCapabilityEvidence,
type RuntimeEnvironmentCapabilityOutcome
} from './runtime-environment-capability-evidence'
import { pauseRemoteRuntimeSharedControlRetry } from './runtime-environment-request-connections'
const sharedControlSupport = new Map<
string,
{ cacheKey: string; check: Promise<RuntimeEnvironmentCapabilityOutcome> }
>()
import {
getRuntimeEnvironmentStatusOwner,
resetRuntimeEnvironmentStatusOwners
} from './runtime-environment-request-connections'
export function resetSharedControlSupport(): void {
sharedControlSupport.clear()
resetRuntimeEnvironmentStatusOwners()
resetRuntimeEnvironmentCapabilityEvidence()
}
export function clearSharedControlSupport(environmentId: string): void {
sharedControlSupport.delete(environmentId)
}
export async function supportsSharedControl(
userDataPath: string,
environment: KnownRuntimeEnvironment,
@@ -48,85 +32,17 @@ export async function supportsSharedControl(
if (accepted) {
return accepted
}
const cacheKey = getSharedControlSupportCacheKey(environment, pairing)
const cached = sharedControlSupport.get(environment.id)
if (cached?.cacheKey === cacheKey) {
const outcome = await cached.check
if (isRuntimeEnvironmentCapabilityOutcomeCurrent(outcome)) {
return outcome
}
if (sharedControlSupport.get(environment.id)?.check === cached.check) {
sharedControlSupport.delete(environment.id)
}
return { kind: 'stale_incarnation' }
const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({
timeoutMs
})
if (!response.ok) {
throw new RemoteRuntimeClientError(response.error.code, response.error.message)
}
let resolvedCacheKey = cacheKey
const evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing)
const check = (async () => {
const response = await sendRemoteRuntimeRequest<RuntimeStatus>(
return (
getAcceptedRuntimeEnvironmentCapabilityOutcome(
environment.id,
pairing,
'status.get',
undefined,
timeoutMs,
undefined,
undefined,
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES
)
if (response.ok === true) {
const verdict = response.result.capabilities?.includes(
REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY
)
? 'capable'
: 'absent'
const acceptedEvidence = applyRuntimeEnvironmentCapabilityVerdict({
evidence,
verdict,
runtimeId: response._meta.runtimeId,
onAbsent: () => pauseRemoteRuntimeSharedControlRetry(environment.id)
})
if (!acceptedEvidence) {
return { kind: 'stale_incarnation' } as const
}
markEnvironmentUsed(userDataPath, environment.id, { runtimeId: response._meta.runtimeId })
resolvedCacheKey = getSharedControlSupportCacheKey(
environment,
pairing,
response._meta.runtimeId
)
return runtimeEnvironmentCapabilityOutcome(evidence, verdict, response._meta.runtimeId)
}
return runtimeEnvironmentCapabilityOutcome(
evidence,
'absent',
environment.runtimeId ?? 'unknown-runtime'
)
})()
// Why: support belongs to the saved pairing/runtime identity, not its mutable display name.
sharedControlSupport.set(environment.id, { cacheKey, check })
try {
const outcome = await check
const cachedAfterCheck = sharedControlSupport.get(environment.id)
if (cachedAfterCheck?.check === check && cachedAfterCheck.cacheKey !== resolvedCacheKey) {
sharedControlSupport.set(environment.id, { cacheKey: resolvedCacheKey, check })
}
return outcome
} catch (error) {
if (sharedControlSupport.get(environment.id)?.check === check) {
sharedControlSupport.delete(environment.id)
}
throw error
}
}
function getSharedControlSupportCacheKey(
environment: KnownRuntimeEnvironment,
pairing: ReturnType<typeof getPreferredPairingOffer>,
runtimeId = environment.runtimeId
): string {
return [
runtimeId ?? 'unknown-runtime',
pairing.endpoint,
pairing.deviceToken,
pairing.publicKeyB64
].join('\0')
response._meta.runtimeId
) ?? { kind: 'stale_incarnation' }
)
}
@@ -0,0 +1,65 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, expect, it, vi } from 'vitest'
import { encodePairingOffer } from '../../shared/pairing'
import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store'
import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version'
import {
createSharedControlTestServer,
closeSharedControlTestServers
} from '../../shared/remote-runtime-shared-control-test-server'
import { getRuntimeEnvironmentStatus } from './runtime-environment-transport-routing'
import {
getRuntimeEnvironmentStatusOwner,
resetRuntimeEnvironmentStatusOwners
} from './runtime-environment-request-connections'
vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } }))
const profiles: string[] = []
afterEach(async () => {
resetRuntimeEnvironmentStatusOwners()
await closeSharedControlTestServers()
profiles.splice(0).forEach((profile) => rmSync(profile, { recursive: true, force: true }))
})
it('publishes real same-socket verification after every authenticated reconnect', async () => {
let runtimeId = 'host-before'
const server = await createSharedControlTestServer({
resultForRequest: () => ({
runtimeId,
capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY]
})
})
const profile = mkdtempSync(join(tmpdir(), 'orca-status-socket-'))
profiles.push(profile)
const environment = addEnvironmentFromPairingCode(profile, {
name: 'host',
pairingCode: encodePairingOffer(server.pairing)
})
await getRuntimeEnvironmentStatus(profile, environment.id)
const owner = getRuntimeEnvironmentStatusOwner(profile, environment.id)
await vi.waitFor(
() => {
expect(owner.read()).toMatchObject({ transport: 'ready', verification: 'verified' })
expect(server.requests).toHaveLength(2)
},
{ timeout: 3_000 }
)
expect(server.connectionCount()).toBe(2) // Bootstrap plus persistent control.
runtimeId = 'host-after'
server.closeClients()
await vi.waitFor(
() => {
expect(owner.read().status?.runtimeId).toBe('host-after')
expect(owner.read().verification).toBe('verified')
},
{ timeout: 3_000 }
)
expect(server.connectionCount()).toBe(3)
expect(server.requests.map((request) => request.method)).toEqual([
'status.get',
'status.get',
'status.get'
])
})
@@ -0,0 +1,89 @@
import { BrowserWindow } from 'electron'
import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client'
import {
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES,
REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY
} from '../../shared/protocol-version'
import {
getPreferredPairingOffer,
type KnownRuntimeEnvironment
} from '../../shared/runtime-environments'
import { markEnvironmentUsed } from '../../shared/runtime-environment-store'
import { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner'
import {
RUNTIME_HOST_STATUS_CHANNEL,
type RuntimeHostStatusResponse
} from '../../shared/runtime-host-status'
import {
applyRuntimeEnvironmentCapabilityVerdict,
getAcceptedRuntimeEnvironmentCapabilityOutcome,
captureRuntimeEnvironmentCapabilityEvidence
} from './runtime-environment-capability-evidence'
import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect'
export function createRuntimeEnvironmentStatusOwner(
userDataPath: string,
environment: KnownRuntimeEnvironment,
transport: {
isReady: () => boolean
request: (signal: AbortSignal) => Promise<RuntimeHostStatusResponse>
establish: () => void
pause: () => void
}
): RuntimeHostStatusOwner {
const pairing = getPreferredPairingOffer(environment)
let evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing)
return new RuntimeHostStatusOwner({
environmentId: environment.id,
pairingRevision: environment.pairingRevision ?? environment.createdAt,
request: (signal) => {
evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing)
return transport.isReady() &&
getAcceptedRuntimeEnvironmentCapabilityOutcome(environment.id, pairing, null)?.kind ===
'supported'
? transport.request(signal)
: sendRemoteRuntimeRequest(
pairing,
'status.get',
undefined,
15_000,
undefined,
signal,
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES
)
},
verified: (response, active) => {
const capable =
response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) ?? false
const accepted = applyRuntimeEnvironmentCapabilityVerdict({
evidence,
verdict: capable ? 'capable' : 'absent',
runtimeId: response._meta.runtimeId
})
if (accepted && active && !isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
markEnvironmentUsed(userDataPath, environment.id, {
runtimeId: response._meta.runtimeId,
pairedDeviceId: response.result.pairedDeviceId
})
if (capable) {
transport.establish()
} else {
transport.pause()
}
}
return capable && active
},
publish: (snapshot) => {
for (const window of BrowserWindow.getAllWindows()) {
if (window.isDestroyed()) {
continue
}
try {
window.webContents.send(RUNTIME_HOST_STATUS_CHANNEL, snapshot)
} catch {
/* A renderer can close during publication. */
}
}
}
})
}
@@ -0,0 +1,89 @@
import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store'
import { pairingCode } from './runtime-environments-ipc-test-harness'
import {
getRuntimeEnvironmentStatus,
resetSharedControlSupport
} from './runtime-environment-transport-routing'
const { request, publish } = vi.hoisted(() => ({ request: vi.fn(), publish: vi.fn() }))
vi.mock('../../shared/remote-runtime-client', () => ({
sendRemoteRuntimeRequest: request,
subscribeRemoteRuntimeRequest: vi.fn()
}))
vi.mock('electron', () => ({
BrowserWindow: {
getAllWindows: () => [
{
isDestroyed: () => false,
webContents: { send: publish }
}
]
}
}))
let profile: string
beforeEach(() => {
vi.useFakeTimers()
request.mockReset()
publish.mockReset()
profile = mkdtempSync(join(tmpdir(), 'orca-status-recovery-'))
})
afterEach(() => {
resetSharedControlSupport()
vi.useRealTimers()
rmSync(profile, { recursive: true, force: true })
})
it('recovers a saved host after its first status check fails, without another UI request', async () => {
const environment = addEnvironmentFromPairingCode(profile, {
name: 'offline-at-startup',
pairingCode: pairingCode()
})
request
.mockRejectedValueOnce(
Object.assign(new Error('host offline'), { code: 'runtime_unavailable' })
)
.mockResolvedValue({
id: 'status',
ok: true,
result: { runtimeId: 'host-1', graphStatus: 'ready', capabilities: [] },
_meta: { runtimeId: 'host-1' }
})
expect((await getRuntimeEnvironmentStatus(profile, environment.id)).ok).toBe(false)
await vi.advanceTimersByTimeAsync(3_000)
expect(request).toHaveBeenCalledTimes(2)
expect(publish).toHaveBeenCalledWith(
'runtimeEnvironments:statusChanged',
expect.objectContaining({
environmentId: environment.id,
verification: 'verified',
status: expect.objectContaining({ runtimeId: 'host-1' })
})
)
await vi.advanceTimersByTimeAsync(300_000)
expect(request).toHaveBeenCalledTimes(2)
})
it('a passive capability check does not strand later active bootstrap recovery', async () => {
const environment = addEnvironmentFromPairingCode(profile, {
name: 'passive-first',
pairingCode: pairingCode()
})
request
.mockResolvedValueOnce({
id: 'status',
ok: true,
result: { runtimeId: 'host-1', capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] },
_meta: { runtimeId: 'host-1' }
})
.mockRejectedValue(new Error('host offline'))
await getRuntimeEnvironmentStatus(profile, environment.id, undefined, { observeOnly: true })
await getRuntimeEnvironmentStatus(profile, environment.id)
await vi.advanceTimersByTimeAsync(3_000)
expect(request).toHaveBeenCalledTimes(3)
})
@@ -57,7 +57,6 @@ describe('runtime environment support routing', () => {
).resolves.toMatchObject({ ok: true })
expect(supportsMock).toHaveBeenCalledTimes(2)
expect(clearSupportMock).toHaveBeenCalledOnce()
expect(supported).toHaveBeenCalledOnce()
expect(unsupported).not.toHaveBeenCalled()
})
@@ -18,10 +18,7 @@ import {
type RuntimeEnvironmentCapabilityOutcome
} from './runtime-environment-capability-evidence'
import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard'
import {
clearSharedControlSupport,
supportsSharedControl
} from './runtime-environment-shared-control-support'
import { supportsSharedControl } from './runtime-environment-shared-control-support'
import {
sendRemoteRuntimeRequestAbortable,
sendRemoteRuntimeSharedControlRequestAbortable
@@ -205,7 +202,6 @@ export async function routeRuntimeEnvironmentCallBySupport(args: {
}
return response
}
clearSharedControlSupport(environment.id)
environment = resolveEnvironment(args.userDataPath, environment.id)
}
return runtimeEnvironmentChangedFailure(environment, args.method)
@@ -1,20 +1,23 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { generateKeyPair, publicKeyToBase64 } from '../../shared/e2ee-crypto'
import { encodePairingOffer, type PairingOffer } from '../../shared/pairing'
import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store'
import {
callRuntimeEnvironment,
getRuntimeEnvironmentStatus,
subscribeRuntimeEnvironment
subscribeRuntimeEnvironment,
resetSharedControlSupport
} from './runtime-environment-transport-routing'
// Why: prove the wiring, not just the helper — an unreachable endpoint exercises
// the real WebSocket failure → reject → Tailscale-hint join points the settings
// probe (returned ok:false) and in-use calls (thrown) actually use.
vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } }))
let userDataPath: string
function seedEnvironment(name: string, endpoint: string): string {
@@ -39,6 +42,7 @@ beforeEach(() => {
})
afterEach(() => {
resetSharedControlSupport()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -1,8 +1,5 @@
import { getPreferredPairingOffer } from '../../shared/runtime-environments'
import {
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES,
REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY
} from '../../shared/protocol-version'
import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/protocol-version'
import { resolveEnvironment, markEnvironmentUsed } from '../../shared/runtime-environment-store'
import { isOrchestrationMutation } from '../../shared/orchestration-rpc-contract'
import type {
@@ -11,33 +8,22 @@ import type {
} from '../../shared/runtime-rpc-envelope'
import type { RuntimeStatus } from '../../shared/runtime-types'
import {
sendRemoteRuntimeRequest,
subscribeRemoteRuntimeRequest,
type RemoteRuntimeSubscription
} from '../../shared/remote-runtime-client'
import { withRemoteRuntimeTailscaleHint } from '../../shared/remote-runtime-tailscale-hint'
import { enqueueRuntimeCall } from './runtime-environment-call-queue'
import {
ensureRemoteRuntimeSharedControlConnection,
pauseRemoteRuntimeSharedControlRetry,
reconnectRemoteRuntimeSharedControlConnection
} from './runtime-environment-request-connections'
import { getRuntimeEnvironmentStatusOwner } from './runtime-environment-request-connections'
import {
sendRemoteRuntimeConnectionRequestAbortable,
sendRemoteRuntimeRequestAbortable
} from './runtime-environment-abortable-requests'
import { attachRemoteControlDiagnostics } from './runtime-environment-status-diagnostics'
import {
applyRuntimeEnvironmentCapabilityVerdict,
captureRuntimeEnvironmentCapabilityEvidence
} from './runtime-environment-capability-evidence'
import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect'
import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard'
import { withTailscaleHintForResponse } from './runtime-environment-tailscale-response'
import {
clearSharedControlSupport,
resetSharedControlSupport
} from './runtime-environment-shared-control-support'
import { resetSharedControlSupport } from './runtime-environment-shared-control-support'
import {
executeSupportRoutedCall,
shouldRouteCallBySupport,
@@ -47,72 +33,31 @@ import {
const DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS = 15_000
export { clearSharedControlSupport, resetSharedControlSupport }
export { resetSharedControlSupport }
export async function getRuntimeEnvironmentStatus(
userDataPath: string,
selector: string,
timeoutMs?: number,
options?: { observeOnly?: true }
options?: { observeOnly?: true; signal?: AbortSignal; reconnect?: true }
): Promise<RuntimeRpcResponse<RuntimeStatus>> {
const environment = resolveEnvironment(userDataPath, selector)
const pairing = getPreferredPairingOffer(environment)
const evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing)
let response: RuntimeRpcResponse<RuntimeStatus>
try {
response = await sendRemoteRuntimeRequest<RuntimeStatus>(
pairing,
'status.get',
undefined,
timeoutMs ?? DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS,
undefined,
undefined,
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES
)
} catch (error) {
// Why: the status UI needs shared-control diagnostics most when the
// fresh status probe failed and the host is reconnecting/offline.
return attachRemoteControlDiagnostics(
withTailscaleHintForResponse(
{
id: 'status.get',
ok: false,
error: {
code: 'runtime_unavailable',
message: error instanceof Error ? error.message : String(error)
},
_meta: { runtimeId: environment.runtimeId }
},
pairing.endpoint
),
environment.id
)
}
if (response.ok === true) {
const verdict = response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY)
? 'capable'
: 'absent'
const accepted = applyRuntimeEnvironmentCapabilityVerdict({
evidence,
verdict,
runtimeId: response._meta.runtimeId,
onCapable: () => {
if (!options?.observeOnly && !isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
ensureRemoteRuntimeSharedControlConnection(environment.id, pairing)
reconnectRemoteRuntimeSharedControlConnection(environment.id)
}
},
onAbsent: () => pauseRemoteRuntimeSharedControlRetry(environment.id)
})
if (accepted && !options?.observeOnly) {
markEnvironmentUsed(userDataPath, environment.id, {
runtimeId: response._meta.runtimeId,
pairedDeviceId: response.result.pairedDeviceId
})
if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
return {
id: 'status.get',
ok: false,
error: {
code: 'runtime_manually_disconnected',
message: 'Runtime environment is manually disconnected.'
}
}
}
const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({
timeoutMs,
...options
})
return attachRemoteControlDiagnostics(
withTailscaleHintForResponse(response, pairing.endpoint),
withTailscaleHintForResponse(response, getPreferredPairingOffer(environment).endpoint),
environment.id
)
}
@@ -127,6 +72,15 @@ export async function callRuntimeEnvironment(
envelope?: RuntimeOrchestrationEnvelope,
options?: { signal?: AbortSignal }
): Promise<RuntimeRpcResponse<unknown>> {
if (method === 'status.get') {
const environment = resolveEnvironment(userDataPath, selector)
const failure = runtimeEnvironmentRevisionFailure(
environment,
expectedEnvironmentPairingRevision,
method
)
return failure ?? getRuntimeEnvironmentStatus(userDataPath, selector, timeoutMs, options)
}
const environment = resolveEnvironment(userDataPath, selector)
// Why: connection failures reject (they don't resolve as ok:false), so the
// Tailscale hint is applied to the thrown error here — wrapping the resolved
@@ -1,3 +1,4 @@
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -44,6 +45,7 @@ const {
}))
vi.mock('electron', () => ({
BrowserWindow: { getAllWindows: () => [] },
app: { getPath: getPathMock },
ipcMain: {
handle: handleMock,
@@ -58,18 +60,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({
subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection:
reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow:
retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
})
})
import { registerRuntimeEnvironmentHandlers } from './runtime-environments'
import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness'
@@ -112,6 +119,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -339,7 +347,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
undefined,
15_000,
undefined,
undefined,
expect.any(AbortSignal),
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES
)
expect(sendRemoteRuntimeSharedControlRequestMock).toHaveBeenCalledWith(
@@ -451,7 +459,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
}
)
it('keeps uncoded call failures on the rejected IPC fallback path', async () => {
it('returns uncoded status failures through the owner response', async () => {
registerRuntimeEnvironmentHandlers(store as never)
sendRemoteRuntimeRequestMock.mockRejectedValue(new Error('shared down'))
@@ -464,9 +472,10 @@ describe('registerRuntimeEnvironmentHandlers', () => {
'runtimeEnvironments:call'
)
await expect(call(null, { selector: 'desk', method: 'status.get' })).rejects.toThrow(
'shared down'
)
await expect(call(null, { selector: 'desk', method: 'status.get' })).resolves.toMatchObject({
ok: false,
error: { code: 'runtime_unavailable', message: 'shared down' }
})
})
it('does not fall back after a shared-control request fails on a supported runtime', async () => {
@@ -1,3 +1,4 @@
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -37,6 +38,7 @@ const {
}))
vi.mock('electron', () => ({
BrowserWindow: { getAllWindows: () => [] },
app: { getPath: getPathMock },
ipcMain: {
handle: handleMock,
@@ -51,18 +53,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({
subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection:
reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow:
retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
})
})
import { registerRuntimeEnvironmentHandlers } from './runtime-environments'
import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness'
@@ -105,6 +112,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -182,9 +190,10 @@ describe('registerRuntimeEnvironmentHandlers', () => {
{ selector: string; method: string; params?: unknown; timeoutMs?: number },
{ ok: true; result: unknown }
>('runtimeEnvironments:call')
await expect(call(null, { selector: 'desk', method: 'repo.list' })).rejects.toThrow(
'probe failed'
)
await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({
ok: false,
error: { code: 'runtime_unavailable', message: 'probe failed' }
})
await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({
ok: true,
result: { repos: [] }
@@ -1,6 +1,62 @@
import { expect } from 'vitest'
import type { Mock } from 'vitest'
import { getPreferredPairingOffer } from '../../shared/runtime-environments'
import { encodePairingOffer } from '../../shared/pairing'
import { resolveEnvironment } from '../../shared/runtime-environment-store'
import { createRuntimeEnvironmentStatusOwner } from './runtime-environment-status-owner'
import type { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner'
import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect'
/** Keep IPC tests on the production owner while replacing only its transport. */
export function withRuntimeStatusOwners<T extends Record<string, Mock>>(transport: T) {
const owners = new Map<string, RuntimeHostStatusOwner>()
return {
...transport,
getRuntimeEnvironmentStatusOwner: (profile: string, selector: string) => {
const environment = resolveEnvironment(profile, selector)
let owner = owners.get(environment.id)
if (!owner || owner.read().retired) {
owner = createRuntimeEnvironmentStatusOwner(profile, environment, {
isReady: () =>
transport.getRemoteRuntimeSharedControlDiagnostics?.(environment.id)?.state === 'ready',
request: (signal) =>
transport.sendRemoteRuntimeSharedControlRequest(
environment.id,
undefined,
'status.get',
undefined,
15_000,
undefined,
signal
),
establish: () => {
transport.ensureRemoteRuntimeSharedControlConnection?.(
environment.id,
getPreferredPairingOffer(environment)
)
transport.reconnectRemoteRuntimeSharedControlConnection?.(environment.id)
},
pause: () => transport.pauseRemoteRuntimeSharedControlRetry?.(environment.id)
})
owners.set(environment.id, owner)
if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
owner.dispose()
}
}
return owner
},
getRuntimeEnvironmentStatusSnapshots: () => [...owners.values()].map((owner) => owner.read()),
resetRuntimeEnvironmentStatusOwners: () => {
owners.forEach((owner) => owner.dispose())
owners.clear()
},
closeRemoteRuntimeRequestConnection: (...args: unknown[]) => {
owners.get(args[0] as string)?.dispose()
owners.delete(args[0] as string)
transport.closeRemoteRuntimeRequestConnection(...args)
}
}
}
export function pairingCode(endpoint = 'ws://127.0.0.1:6768'): string {
return encodePairingOffer({
@@ -1,3 +1,5 @@
import type { RuntimeHostStatusSnapshot } from '../../shared/runtime-host-status'
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -44,6 +46,7 @@ const {
}))
vi.mock('electron', () => ({
BrowserWindow: { getAllWindows: () => [] },
app: { getPath: getPathMock },
ipcMain: {
handle: handleMock,
@@ -58,18 +61,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({
subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: retryRemoteRuntimeSharedControlConnectionNowMock,
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection:
reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow:
retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: retryRemoteRuntimeSharedControlConnectionNowMock,
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
})
})
import { registerRuntimeEnvironmentHandlers } from './runtime-environments'
import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness'
@@ -125,6 +133,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -132,6 +141,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
registerRuntimeEnvironmentHandlers(store as never)
expect(handleMock.mock.calls.map((call) => call[0])).toEqual([
'runtimeEnvironments:getStatusSnapshots',
'runtimeEnvironments:list',
'runtimeEnvironments:addFromPairingCode',
'runtimeEnvironments:verifyAndAddFromPairingCode',
@@ -166,6 +176,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
'runtimeEnvironments:retryControlConnection',
'runtimeEnvironments:prepareBrowserClientHostPlacement',
'runtimeEnvironments:getStatus',
'runtimeEnvironments:getStatusSnapshots',
'runtimeEnvironments:call',
'runtimeEnvironments:subscribe',
'runtimeEnvironments:unsubscribe',
@@ -467,6 +478,13 @@ describe('registerRuntimeEnvironmentHandlers', () => {
ok: false,
error: { code: 'runtime_manually_disconnected' }
})
const getSnapshots = handler<undefined, RuntimeHostStatusSnapshot[]>(
'runtimeEnvironments:getStatusSnapshots'
)
// A new renderer only has the snapshot read, not the earlier disconnect event.
expect(await getSnapshots(null, undefined)).toMatchObject([
{ environmentId: added.environment.id, retired: true, transport: 'disconnected' }
])
const call = handler<
{ selector: string; method: string },
{ ok: boolean; error?: { code: string } }
@@ -492,6 +510,10 @@ describe('registerRuntimeEnvironmentHandlers', () => {
result: { runtimeId: 'runtime-remote' }
})
expect(sendRemoteRuntimeRequestMock).toHaveBeenCalledOnce()
expect(await getSnapshots(null, undefined)).toMatchObject([
{ environmentId: added.environment.id, verification: 'verified' }
])
expect((await getSnapshots(null, undefined))[0].retired).not.toBe(true)
})
it('marks environments owned by ephemeral VM runtimes in the public list', async () => {
@@ -1,3 +1,4 @@
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -44,6 +45,7 @@ const {
}))
vi.mock('electron', () => ({
BrowserWindow: { getAllWindows: () => [] },
app: { getPath: getPathMock },
ipcMain: {
handle: handleMock,
@@ -58,18 +60,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({
subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: ensureRemoteRuntimeSharedControlConnectionMock,
pauseRemoteRuntimeSharedControlRetry: pauseRemoteRuntimeSharedControlRetryMock,
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection:
reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow:
retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: ensureRemoteRuntimeSharedControlConnectionMock,
pauseRemoteRuntimeSharedControlRetry: pauseRemoteRuntimeSharedControlRetryMock,
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
})
})
import { registerRuntimeEnvironmentHandlers } from './runtime-environments'
import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness'
@@ -114,6 +121,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -148,9 +156,9 @@ describe('registerRuntimeEnvironmentHandlers', () => {
expect.objectContaining({ endpoint: 'ws://127.0.0.1:6768', deviceToken: 'device-token' }),
'status.get',
undefined,
50,
undefined,
15_000,
undefined,
expect.any(AbortSignal),
ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES
)
expect(reconnectRemoteRuntimeSharedControlConnectionMock).toHaveBeenCalledWith(
@@ -319,36 +327,41 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
})
it('returns shared-control diagnostics when saved remote runtime status throws', async () => {
registerRuntimeEnvironmentHandlers(store as never)
getRemoteRuntimeSharedControlDiagnosticsMock.mockReturnValue({
state: 'reconnecting',
pendingRequestCount: 0,
subscriptionCount: 1,
reconnectAttempt: 2,
lastConnectedAt: 123,
lastClose: { code: 1006, reason: '' },
lastError: 'closed'
})
sendRemoteRuntimeRequestMock.mockRejectedValue(new Error('socket closed'))
it.each(['runtimeEnvironments:getStatus', 'runtimeEnvironments:connect'])(
'preserves failure diagnostics and guidance on %s',
async (channel) => {
registerRuntimeEnvironmentHandlers(store as never)
getRemoteRuntimeSharedControlDiagnosticsMock.mockReturnValue({
state: 'reconnecting',
pendingRequestCount: 0,
subscriptionCount: 1,
reconnectAttempt: 2,
lastConnectedAt: 123,
lastClose: { code: 1006, reason: '' },
lastError: 'closed'
})
sendRemoteRuntimeRequestMock.mockRejectedValue(
new Error('Could not connect to the remote Orca runtime.')
)
const add = handler<
{ name: string; pairingCode: string },
{ environment: { id: string; name: string } }
>('runtimeEnvironments:addFromPairingCode')
await add(null, { name: 'desk', pairingCode: pairingCode() })
const add = handler<
{ name: string; pairingCode: string },
{ environment: { id: string; name: string } }
>('runtimeEnvironments:addFromPairingCode')
await add(null, { name: 'desk', pairingCode: pairingCode() })
const getStatus = handler<
{ selector: string; timeoutMs?: number },
{ ok: false; error: { message: string; data?: { remoteControl?: { state: string } } } }
>('runtimeEnvironments:getStatus')
const getStatus = handler<
{ selector: string; timeoutMs?: number },
{ ok: false; error: { message: string; data?: { remoteControl?: { state: string } } } }
>(channel)
await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({
ok: false,
error: {
message: 'socket closed',
data: { remoteControl: { state: 'reconnecting' } }
}
})
})
await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({
ok: false,
error: {
message: expect.stringContaining('connect both devices to Tailscale'),
data: { remoteControl: { state: 'reconnecting' } }
}
})
}
)
})
@@ -1,3 +1,4 @@
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -38,6 +39,7 @@ const {
}))
vi.mock('electron', () => ({
BrowserWindow: { getAllWindows: () => [] },
app: { getPath: getPathMock },
ipcMain: {
handle: handleMock,
@@ -52,18 +54,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({
subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection:
reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow:
retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
})
})
import {
invalidateRuntimeEnvironmentTransport,
@@ -109,6 +116,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -1,3 +1,4 @@
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -40,6 +41,7 @@ const {
}))
vi.mock('electron', () => ({
BrowserWindow: { getAllWindows: () => [] },
app: { getPath: getPathMock },
ipcMain: {
handle: handleMock,
@@ -54,18 +56,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({
subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection:
reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow:
retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
})
})
import { registerRuntimeEnvironmentHandlers } from './runtime-environments'
import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness'
@@ -108,6 +115,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
@@ -1,3 +1,4 @@
import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -38,6 +39,7 @@ const {
}))
vi.mock('electron', () => ({
BrowserWindow: { getAllWindows: () => [] },
app: { getPath: getPathMock },
ipcMain: {
handle: handleMock,
@@ -52,18 +54,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({
subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock
}))
vi.mock('./runtime-environment-request-connections', () => ({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
}))
vi.mock('./runtime-environment-request-connections', async () => {
const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness')
return withRuntimeStatusOwners({
sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock,
sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock,
subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock,
getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock,
reconnectRemoteRuntimeSharedControlConnection:
reconnectRemoteRuntimeSharedControlConnectionMock,
retryRemoteRuntimeSharedControlConnectionsNow:
retryRemoteRuntimeSharedControlConnectionsNowMock,
retryRemoteRuntimeSharedControlConnectionNow: vi.fn(),
ensureRemoteRuntimeSharedControlConnection: vi.fn(),
pauseRemoteRuntimeSharedControlRetry: vi.fn(),
closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock
})
})
vi.mock('../browser/paired-runtime-browser-client-host-runtime', () => ({
retirePairedRuntimeBrowserClientHostEnvironment:
retirePairedRuntimeBrowserClientHostEnvironmentMock
@@ -115,6 +122,7 @@ describe('registerRuntimeEnvironmentHandlers', () => {
})
afterEach(() => {
resetRuntimeEnvironmentStatusOwners()
rmSync(userDataPath, { recursive: true, force: true })
})
+10 -4
View File
@@ -1,6 +1,6 @@
import { app, ipcMain } from 'electron'
import { randomUUID } from 'node:crypto'
import { resolveEnvironment } from '../../shared/runtime-environment-store'
import { listEnvironments, resolveEnvironment } from '../../shared/runtime-environment-store'
import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client'
import type { Store } from '../persistence'
import {
@@ -8,14 +8,16 @@ import {
registerRuntimeEnvironmentConnectivityHandlers,
registerRuntimeEnvironmentPassiveHandlers
} from './runtime-environment-connectivity-handlers'
import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections'
import {
closeRemoteRuntimeRequestConnection,
getRuntimeEnvironmentStatusOwner
} from './runtime-environment-request-connections'
import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler'
import {
advanceRuntimeEnvironmentTransportGeneration,
getRuntimeEnvironmentTransportGeneration
} from './runtime-environment-transport-generation'
import {
clearSharedControlSupport,
resetSharedControlSupport,
subscribeRuntimeEnvironment
} from './runtime-environment-transport-routing'
@@ -64,7 +66,6 @@ export function invalidateRuntimeEnvironmentTransport(environmentId: string): Pr
advanceRuntimeEnvironmentCapabilityIncarnation(environmentId)
advanceRuntimeEnvironmentTransportGeneration(environmentId)
closeRemoteRuntimeRequestConnection(environmentId)
clearSharedControlSupport(environmentId)
closeSubscriptionsForEnvironment(environmentId)
return retirePairedRuntimeBrowserClientHostEnvironment(
environmentId,
@@ -97,6 +98,11 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void {
})
registerRuntimeEnvironmentRecoveryHandler()
registerRuntimeEnvironmentPassiveHandlers(getUserDataPath)
for (const environment of listEnvironments(getUserDataPath())) {
if (!isRuntimeEnvironmentManuallyDisconnected(environment.id)) {
getRuntimeEnvironmentStatusOwner(getUserDataPath(), environment.id).activate()
}
}
ipcMain.handle(
'runtimeEnvironments:subscribe',
async (
+3
View File
@@ -1,3 +1,4 @@
import type { RuntimeHostStatusSnapshot } from '../../shared/runtime-host-status'
import type {
RuntimeBrowserDriverState,
RuntimeRendererSyncWindowGraph,
@@ -77,6 +78,8 @@ export type RuntimeApi = {
) => () => void
}
runtimeEnvironments: {
getStatusSnapshots: () => Promise<RuntimeHostStatusSnapshot[]>
onStatusChanged: (callback: (snapshot: RuntimeHostStatusSnapshot) => void) => () => void
list: () => Promise<PublicKnownRuntimeEnvironment[]>
addFromPairingCode: (args: {
name: string
@@ -1,4 +1,8 @@
import { ipcRenderer } from 'electron'
import {
RUNTIME_HOST_STATUS_CHANNEL,
type RuntimeHostStatusSnapshot
} from '../../shared/runtime-host-status'
import type { VerifyAndAddRuntimeEnvironmentResult } from '../../shared/remote-pairing-verification'
import type { RuntimeStatus } from '../../shared/runtime-types'
import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope'
@@ -12,6 +16,16 @@ import {
import type { PreloadApi } from '../api-types'
export const runtimeEnvironmentsApi = {
getStatusSnapshots: (): Promise<RuntimeHostStatusSnapshot[]> =>
ipcRenderer.invoke('runtimeEnvironments:getStatusSnapshots'),
onStatusChanged: (callback: (snapshot: RuntimeHostStatusSnapshot) => void): (() => void) => {
const listener = (
_event: Electron.IpcRendererEvent,
snapshot: RuntimeHostStatusSnapshot
): void => callback(snapshot)
ipcRenderer.on(RUNTIME_HOST_STATUS_CHANNEL, listener)
return () => ipcRenderer.removeListener(RUNTIME_HOST_STATUS_CHANNEL, listener)
},
list: (): Promise<PublicKnownRuntimeEnvironment[]> =>
ipcRenderer.invoke('runtimeEnvironments:list'),
addFromPairingCode: (args: {
@@ -242,17 +242,11 @@ export default function NewWorkspaceComposerCard(
selector: action.environmentId,
timeoutMs: 15_000
})
const runtimeStatus = unwrapRuntimeRpcResult<RuntimeStatus>(response)
useAppStore.getState().setRuntimeEnvironmentStatus(action.environmentId, {
status: runtimeStatus,
checkedAt: Date.now()
})
unwrapRuntimeRpcResult<RuntimeStatus>(response)
await useAppStore.getState().readRuntimeHostStatusSnapshots()
} catch (error) {
if (action.kind === 'runtime') {
useAppStore.getState().setRuntimeEnvironmentStatus(action.environmentId, {
status: null,
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
}
toast.error(
error instanceof Error
@@ -83,8 +83,7 @@ describe('getPaletteHostBadge', () => {
repos: [{ executionHostId: 'runtime:env-1' }],
sshTargetLabels: new Map(),
settings: { activeRuntimeEnvironmentId: 'env-2' },
// A live status makes the runtime 'available'; without it the host reads
// 'disconnected' and the badge is suppressed (covered below).
// Only verified availability enables unfiltered host badges.
runtimeStatusByEnvironmentId: new Map([
[
'env-1',
@@ -145,3 +144,19 @@ describe('getPaletteHostBadge', () => {
expect(getPaletteHostBadge(null, hosts)).toBeNull()
})
})
it.each(['connecting', 'blocked', 'disconnected', 'error'] as const)(
'does not infer reachability from %s health, but preserves explicit filter labels',
(health) => {
const hosts = buildSidebarHostOptions({
repos: [{ executionHostId: 'runtime:env-1' }],
sshTargetLabels: new Map(),
settings: { activeRuntimeEnvironmentId: null }
}).map((host) => (host.kind === 'runtime' ? { ...host, health } : host))
expect(getPaletteHostBadge({ connectionId: null }, hosts)).toBeNull()
expect(getPaletteHostBadge({ executionHostId: 'runtime:env-1' }, hosts, true)).toEqual({
hostId: 'runtime:env-1',
label: 'env-1'
})
}
)
@@ -17,7 +17,7 @@ export type PaletteHostBadge = {
// unlike the sidebar gate, which lists disconnected hosts so users can connect.
function hasActiveRemoteHost(hostOptions: readonly SidebarHostOption[]): boolean {
return hostOptions.some(
(host) => host.id !== LOCAL_EXECUTION_HOST_ID && host.health !== 'disconnected'
(host) => host.id !== LOCAL_EXECUTION_HOST_ID && host.health === 'available'
)
}
@@ -28,7 +28,7 @@ import {
} from '@/store/slices/runtime-environment-ssh'
import {
isConnectedRuntimeHostState,
runtimeHostConnectionState
runtimeHostConnectionStateForEntry
} from '@/runtime/runtime-host-connection-state'
type RepositoryHostSetupsSectionProps = {
@@ -227,14 +227,7 @@ export function RepositoryHostSetupsSection({
? runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId)
: undefined
const runtimeOwnerState = runtimeOwnerEnvironmentId
? runtimeHostConnectionState({
hasStatusEntry: Boolean(runtimeOwnerStatusEntry),
status: runtimeOwnerStatusEntry?.status,
remoteControl:
runtimeOwnerStatusEntry?.remoteControl ??
runtimeOwnerStatusEntry?.status?.remoteControl ??
null
})
? runtimeHostConnectionStateForEntry(runtimeOwnerStatusEntry)
: null
const runtimeOwnerReachable =
runtimeOwnerState === null || isConnectedRuntimeHostState(runtimeOwnerState)
@@ -51,10 +51,7 @@ export function useRuntimeEnvironmentCatalog(): RuntimeEnvironmentCatalog {
// linger in the sidebar registry.
useAppStore.getState().setRuntimeEnvironments(nextEnvironments)
if (verified) {
useAppStore.getState().setRuntimeEnvironmentStatus(verified.environmentId, {
status: verified.runtimeStatus,
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
}
if (mountedRef.current) {
setEnvironments(visibleEnvironments)
@@ -93,10 +90,7 @@ export function useRuntimeEnvironmentCatalog(): RuntimeEnvironmentCatalog {
const runtimeStatus = unwrapRuntimeRpcResult<RuntimeStatus>(response)
// Why: feed the live status into the store so sidebar host pickers
// reflect manual refreshes, not just the settings pane.
useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, {
status: runtimeStatus,
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
if (!mountedRef.current) {
return
}
@@ -114,11 +108,7 @@ export function useRuntimeEnvironmentCatalog(): RuntimeEnvironmentCatalog {
// Why: record the failed probe (null status) so the sidebar can
// distinguish unreachable from never-checked.
const remoteControl = extractRuntimeTransportDiagnostics(error)
useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, {
status: null,
...(remoteControl ? { remoteControl } : {}),
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
if (!mountedRef.current) {
return
}
@@ -40,14 +40,7 @@ export function useRuntimeEnvironmentConnectionActions({
await window.api.runtimeEnvironments.disconnect({ selector: environment.id })
// Why: disconnect is non-destructive; keep the saved server but show the
// user that this live client is no longer attached to it.
useAppStore.getState().setRuntimeEnvironmentStatus(
environment.id,
{
status: null,
checkedAt: Date.now()
},
{ suppressDisconnectToast: true }
)
await useAppStore.getState().readRuntimeHostStatusSnapshots()
if (mountedRef.current) {
setDetailsByEnvironmentId((current) => ({
...current,
@@ -96,10 +89,7 @@ export function useRuntimeEnvironmentConnectionActions({
const compatibility = evaluateHostDetails(runtimeStatus)
// Why: row Connect is reachability only. The Advanced selector is the
// explicit default-host control and should be the only active-server path.
useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, {
status: runtimeStatus,
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
if (mountedRef.current) {
setDetailsByEnvironmentId((current) => ({
...current,
@@ -143,11 +133,7 @@ export function useRuntimeEnvironmentConnectionActions({
} catch (error) {
const message = error instanceof Error ? error.message : 'Failed to connect server.'
const remoteControl = extractRuntimeTransportDiagnostics(error)
useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, {
status: null,
...(remoteControl ? { remoteControl } : {}),
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
if (mountedRef.current) {
setDetailsByEnvironmentId((current) => ({
...current,
@@ -68,7 +68,7 @@ export function AddRemoteHostDialog({
const setSshTargetsMetadata = useAppStore((s) => s.setSshTargetsMetadata)
const recordSshRepoReadoptions = useAppStore((s) => s.recordSshRepoReadoptions)
const setRuntimeEnvironments = useAppStore((s) => s.setRuntimeEnvironments)
const setRuntimeEnvironmentStatus = useAppStore((s) => s.setRuntimeEnvironmentStatus)
const readRuntimeHostStatusSnapshots = useAppStore((s) => s.readRuntimeHostStatusSnapshots)
const recordFeatureInteraction = useAppStore((s) => s.recordFeatureInteraction)
const busy = isSaving || isBulkImporting || resolvingConfigAlias !== null
@@ -283,10 +283,7 @@ export function AddRemoteHostDialog({
}
const environments = await window.api.runtimeEnvironments.list()
setRuntimeEnvironments(environments)
setRuntimeEnvironmentStatus(result.environment.id, {
status: result.runtimeStatus,
checkedAt: Date.now()
})
await readRuntimeHostStatusSnapshots()
toast.success(
translate('auto.components.sidebar.AddRemoteHostDialog.serverSaved', 'Remote server added.')
)
@@ -141,13 +141,10 @@ export function HostSectionHeaderMenu({ row }: { row: HostHeaderRow }): React.JS
selector: parsed.environmentId,
timeoutMs: 10_000
})
const runtimeStatus = unwrapRuntimeRpcResult<RuntimeStatus>(response)
unwrapRuntimeRpcResult<RuntimeStatus>(response)
// Why: feed the probe result into the shared store so the host header and
// other host pickers reflect this check without a separate fetch.
useAppStore.getState().setRuntimeEnvironmentStatus(parsed.environmentId, {
status: runtimeStatus,
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
toast.success(
translate(
'auto.components.sidebar.HostSectionHeaderMenu.7f1a2b3c4d',
@@ -160,10 +157,7 @@ export function HostSectionHeaderMenu({ row }: { row: HostHeaderRow }): React.JS
} catch (err) {
// Why: record the failed probe so the host registry can drop a previously
// healthy verdict instead of showing stale "compatible" state.
useAppStore.getState().setRuntimeEnvironmentStatus(parsed.environmentId, {
status: null,
checkedAt: Date.now()
})
await useAppStore.getState().readRuntimeHostStatusSnapshots()
toast.error(
err instanceof Error
? err.message
@@ -83,7 +83,8 @@ describe('NoticeHostGlyph', () => {
)
})
it('marks a paired runtime with no live status as disconnected', async () => {
it('marks a paired runtime a probe found unreachable as disconnected', async () => {
runtimeStatusByEnvironmentId.set('openclaw-env', { status: null })
const container = await render('runtime:openclaw-env')
expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe(
@@ -91,6 +92,17 @@ describe('NoticeHostGlyph', () => {
)
})
it('does not call a host disconnected before its first probe answers', async () => {
// No entry means "not asked yet", not "asked and unreachable" — collapsing the two
// painted every remote row destructive between launch and the first probe.
const container = await render('runtime:openclaw-env')
expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe(
'Project on openclaw'
)
expect(container.querySelector('svg')?.getAttribute('class')).not.toContain('text-destructive')
})
it('gives the local host the monitor glyph the run-target rows use', async () => {
const container = await render('local', 'Local Mac')
@@ -5,6 +5,10 @@ import { HostRowIcon } from '../host-row-icon'
import { useAppStore } from '@/store'
import { parseExecutionHostId, type ExecutionHostId } from '../../../../shared/execution-host'
import { translate } from '@/i18n/i18n'
import {
isDisconnectedRuntimeHostState,
runtimeHostConnectionStateForEntry
} from '@/runtime/runtime-host-connection-state'
type NoticeHostGlyphProps = {
hostId: ExecutionHostId
@@ -26,11 +30,15 @@ export default function NoticeHostGlyph({
keyboardFocusable
}: NoticeHostGlyphProps): React.JSX.Element | null {
const host = parseExecutionHostId(hostId)
// Why the shared derivation, not raw truthiness: an absent entry means "not probed yet",
// which is not the same verdict as a probe that came back unreachable.
const isDisconnected = useAppStore((s) => {
if (host?.kind !== 'runtime') {
return false
}
return !s.runtimeStatusByEnvironmentId.get(host.environmentId)?.status
return isDisconnectedRuntimeHostState(
runtimeHostConnectionStateForEntry(s.runtimeStatusByEnvironmentId.get(host.environmentId))
)
})
if (!host) {
@@ -218,18 +218,35 @@ describe('WorktreeCard SSH reconnect prompt', () => {
expect(markup).not.toContain('Retry SSH connection')
})
it('marks a runtime-host worktree disconnected when its environment has no status', () => {
it('marks a runtime-host worktree disconnected once a probe finds it unreachable', () => {
runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }]
runtimeStatusByEnvironmentId.set('env-1', { status: null })
const runtimeRepo: Repo = {
...makeRepo(),
connectionId: undefined,
executionHostId: 'runtime:env-1'
}
const markup = renderToStaticMarkup(
<WorktreeCard worktree={makeWorktree()} repo={runtimeRepo} isActive={false} />
)
expect(markup).toContain('Remote Mac disconnected')
})
// Why: "not probed yet" is not "probed and unreachable" — collapsing them painted every
// remote card destructive and dimmed between launch and the first probe answering.
it('leaves a runtime-host worktree undimmed before its first probe answers', () => {
runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }]
const runtimeRepo: Repo = {
...makeRepo(),
connectionId: undefined,
executionHostId: 'runtime:env-1'
}
// No status entry for env-1 → host is disconnected.
const markup = renderToStaticMarkup(
<WorktreeCard worktree={makeWorktree()} repo={runtimeRepo} isActive={false} />
)
expect(markup).toContain('Remote Mac disconnected')
expect(markup).not.toContain('Remote Mac disconnected')
expect(markup).toContain('Project on Remote Mac')
expect(markup).not.toContain('opacity-60')
})
it('distinguishes connected worktrees on different Orca servers', () => {
@@ -77,11 +77,10 @@ describe('sidebar host options', () => {
})
expect(hosts.map((host) => host.id)).toEqual(['local', 'runtime:runtime-1'])
// Without live status the focused runtime has no proof of reachability, so it
// reads 'disconnected' rather than defaulting to 'available'/"Connected".
// A first probe still in progress is not evidence of disconnection.
expect(hosts.find((host) => host.id === 'runtime:runtime-1')).toMatchObject({
detail: 'Orca server',
health: 'disconnected'
health: 'connecting'
})
})
@@ -10,6 +10,10 @@ import { getHostDisplayLabelOverrides } from '../../../../shared/host-setting-ov
import { getWorkspacePortsByWorktreeId } from '@/lib/workspace-port-groups'
import { getExplicitRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner'
import { hydrateRuntimeEnvironmentSshState } from '@/runtime/runtime-environment-ssh-state'
import {
isDisconnectedRuntimeHostState,
runtimeHostConnectionStateForEntry
} from '@/runtime/runtime-host-connection-state'
import { useAppStore } from '@/store'
import {
selectRuntimeAwareSshStatus,
@@ -177,12 +181,17 @@ export function useWorktreeCardFoundation({
const runtimeHostLabel = runtimeHostId
? (getHostDisplayLabelOverrides(settings).get(runtimeHostId) ?? runtimeEnvironmentName)
: null
// Why: runtime ("Orca server") hosts get the same disconnected dimming as SSH when their environment has no live status.
// Why the shared derivation, not raw truthiness: an absent entry means "not probed yet",
// which is not the same verdict as a probe that came back unreachable.
const isRuntimeDisconnected = useAppStore((s) => {
if (!runtimeOwnerEnvironmentId) {
return false
}
return !s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId)?.status
return isDisconnectedRuntimeHostState(
runtimeHostConnectionStateForEntry(
s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId)
)
)
})
const [titleRenaming, setTitleRenaming] = useState(false)
const [showRenameErrorDialog, setShowRenameErrorDialog] = useState(false)
@@ -33,7 +33,7 @@ import {
} from './remote-host-connection-status'
import {
isConnectedRuntimeHostState,
runtimeHostConnectionState,
runtimeHostConnectionStateForEntry,
runtimeStatusForOverall
} from '@/runtime/runtime-host-connection-state'
import { refreshRuntimeProjectWorktreesAndLineage } from '@/hooks/runtime-project-refresh-scheduler'
@@ -74,7 +74,7 @@ export function SshStatusSegment({
const settings = useAppStore((s) => s.settings)
const runtimeEnvironments = useAppStore((s) => s.runtimeEnvironments)
const runtimeStatusByEnvironmentId = useAppStore((s) => s.runtimeStatusByEnvironmentId)
const setRuntimeEnvironmentStatus = useAppStore((s) => s.setRuntimeEnvironmentStatus)
const readRuntimeHostStatusSnapshots = useAppStore((s) => s.readRuntimeHostStatusSnapshots)
const hydrateRuntimeEnvironmentStatuses = useAppStore((s) => s.hydrateRuntimeEnvironmentStatuses)
const remoteWorkspaceSyncStatusByTargetId = useAppStore(
(s) => s.remoteWorkspaceSyncStatusByTargetId
@@ -105,7 +105,7 @@ export function SshStatusSegment({
return {
id: environment.id,
label: override || environment.name || environment.id,
hasStatusEntry: Boolean(statusEntry),
snapshot: statusEntry?.snapshot,
status: statusEntry?.status ?? null,
active: settings?.activeRuntimeEnvironmentId === environment.id,
remoteControl: statusEntry?.remoteControl ?? statusEntry?.status?.remoteControl ?? null
@@ -113,7 +113,7 @@ export function SshStatusSegment({
})
const runtimeHostRows = runtimeHosts.map((host) => ({
...host,
state: runtimeHostConnectionState(host)
state: runtimeHostConnectionStateForEntry(runtimeStatusByEnvironmentId.get(host.id))
}))
// Available remote servers are online even when they are not the active runtime.
// Keep host health separate from the advanced active-server selection.
@@ -152,11 +152,7 @@ export function SshStatusSegment({
async (environmentId: string): Promise<void> => {
try {
await window.api.runtimeEnvironments.disconnect({ selector: environmentId })
setRuntimeEnvironmentStatus(
environmentId,
{ status: null, checkedAt: Date.now() },
{ suppressDisconnectToast: true }
)
await readRuntimeHostStatusSnapshots()
recordFeatureInteraction('ssh')
} catch (err) {
toast.error(
@@ -169,7 +165,7 @@ export function SshStatusSegment({
)
}
},
[recordFeatureInteraction, setRuntimeEnvironmentStatus]
[recordFeatureInteraction, readRuntimeHostStatusSnapshots]
)
if (targets.length === 0 && runtimeHosts.length === 0) {
@@ -140,6 +140,9 @@ export async function loadIpcEventsHarness(
dispatchEvent: vi.fn(),
api: new Proxy(
{
runtimeEnvironments: createApiNamespaceStub({
getStatusSnapshots: () => Promise.resolve([])
}),
ui: createApiNamespaceStub({
getZoomLevel: () => 0,
consumePendingOpenSettings: () => Promise.resolve(false),
@@ -1,3 +1,4 @@
import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status'
import { getTabIdsAwaitingHostHydrationRemount } from '@/lib/parked-terminal-host-hydration'
import { emitAutomationsChangedWindowEvent } from '@/lib/automations-changed-window-event'
import { createBackgroundSleepingAgentWakeDispatcher } from '@/lib/wake-sleeping-agents-in-background'
@@ -63,13 +64,23 @@ export function installAppLifetimeIpcEvents(
)
const worktreeRuntime = createWorktreeEventRuntime(unsubs, isRuntimeEnvironmentActive)
const onSharedControlDiagnostics = window.api.runtimeEnvironments?.onSharedControlDiagnostics
if (onSharedControlDiagnostics) {
unsubs.push(
onSharedControlDiagnostics((event) => {
useAppStore.getState().publishRuntimeEnvironmentDiagnostics(event)
const statusApi = window.api.runtimeEnvironments
if (statusApi?.onStatusChanged) {
const apply = (snapshot: RuntimeHostStatusSnapshot): void => {
useAppStore.getState().applyRuntimeHostStatusSnapshot(snapshot)
}
let stopped = false
unsubs.push(statusApi.onStatusChanged(apply), () => {
stopped = true
})
void statusApi
.getStatusSnapshots()
.then((snapshots) => {
if (!stopped) {
snapshots.forEach(apply)
}
})
)
.catch((error) => console.error('Failed to read runtime status snapshots:', error))
}
const unsubscribeRuntimeEnvironmentStore = registerRuntimeClientIpcBridge(unsubs, worktreeRuntime)
registerProjectCatalogIpcBridge(
@@ -29,20 +29,6 @@ import {
} from './runtime-environment-subscription-selection'
import type { WorktreeEventRuntime } from './worktree-event-runtime'
/** Backoff for re-asking status.get after a probe that failed on its own socket. */
const RUNTIME_STATUS_PROBE_RETRY_DELAYS_MS = [2_000, 10_000]
/**
* Why a request carries its reason: `reconnected` must re-ask even when the cache reads
* reachable (a restart changes the runtimeId under an unchanged-looking status), while
* `recordedUnreachable` is satisfied by any answer that clears the offline verdict.
*/
type RuntimeStatusProbeTrigger = 'reconnected' | 'recordedUnreachable'
function isRuntimeStatusRecordedUnreachable(environmentId: string): boolean {
return useAppStore.getState().runtimeStatusByEnvironmentId?.get(environmentId)?.status === null
}
export function registerRuntimeClientIpcBridge(
unsubs: (() => void)[],
worktreeRuntime: WorktreeEventRuntime
@@ -148,107 +134,6 @@ export function registerRuntimeClientIpcBridge(
})
}
const inFlightRuntimeStatusProbes = new Set<string>()
const trailingRuntimeStatusProbes = new Map<string, RuntimeStatusProbeTrigger>()
const runtimeStatusProbeRetryTimers = new Map<string, ReturnType<typeof setTimeout>>()
let runtimeStatusProbesStopped = false
// Why not the desired-subscription set alone: a host that is not the active environment
// drops out of it the moment anything records its status unreachable, so gating the
// retry on it cancels the retry exactly where recovery matters. Only removal
// (or a still-unhydrated catalog behind an active id) decides whether to keep asking,
// and the tombstone covers the window where settings still names a deleted host active.
const shouldProbeRuntimeStatus = (environmentId: string): boolean => {
const state = useAppStore.getState()
if (state.removedRuntimeEnvironmentIds?.has(environmentId)) {
return false
}
return (
(state.runtimeEnvironments ?? []).some((environment) => environment.id === environmentId) ||
getRuntimeClientEventEnvironmentIds(state).includes(environmentId)
)
}
// Why: concurrent probes resolve in arbitrary order, so a slow one can publish its
// stale answer over a newer one and leave the sidebar naming a superseded runtime.
const probeRuntimeStatus = (
environmentId: string,
attempt = 0,
trigger: RuntimeStatusProbeTrigger = 'reconnected'
): void => {
if (runtimeStatusProbesStopped || !shouldProbeRuntimeStatus(environmentId)) {
return
}
if (inFlightRuntimeStatusProbes.has(environmentId)) {
// Serialize, don't drop: the in-flight answer predates this request, so a reconnect
// that lands mid-probe would otherwise go unasked — and a probe that succeeds
// schedules no retry to pick it up later. A reconnect outranks a queued
// recorded-unreachable request, which the in-flight answer may already settle.
if (trigger === 'reconnected' || !trailingRuntimeStatusProbes.has(environmentId)) {
trailingRuntimeStatusProbes.set(environmentId, trigger)
}
return
}
const pendingRetry = runtimeStatusProbeRetryTimers.get(environmentId)
if (pendingRetry !== undefined) {
clearTimeout(pendingRetry)
runtimeStatusProbeRetryTimers.delete(environmentId)
}
inFlightRuntimeStatusProbes.add(environmentId)
void useAppStore
.getState()
// publishUnreachable: false — the transport that just proved this host alive is not the
// socket status.get dials, so a failed probe here is unverifiable and must publish nothing.
.refreshRuntimeEnvironmentStatus(environmentId, undefined, { publishUnreachable: false })
.catch(() => false)
.then((reachable) => {
inFlightRuntimeStatusProbes.delete(environmentId)
const trailingTrigger = trailingRuntimeStatusProbes.get(environmentId)
if (trailingTrigger !== undefined) {
trailingRuntimeStatusProbes.delete(environmentId)
// A newer reconnect asked while this one was dialing: restart the attempt chain.
// A resubscribe only asked because the cache read unreachable, so skip the extra
// socket + E2EE handshake when this answer already cleared that.
if (
trailingTrigger === 'reconnected' ||
isRuntimeStatusRecordedUnreachable(environmentId)
) {
probeRuntimeStatus(environmentId, 0, trailingTrigger)
return
}
}
// Why: status.get dials its own short-lived socket, so it can fail while the
// control transport that just proved the host is up stays healthy. That failure
// is unverifiable and publishes nothing, so no store transition, resubscribe or
// further trigger follows — without this bounded retry one unlucky probe leaves
// a host already recorded offline stranded until the next transport gap.
const retryDelayMs = RUNTIME_STATUS_PROBE_RETRY_DELAYS_MS[attempt]
if (
reachable ||
retryDelayMs === undefined ||
runtimeStatusProbesStopped ||
!shouldProbeRuntimeStatus(environmentId)
) {
return
}
runtimeStatusProbeRetryTimers.set(
environmentId,
setTimeout(() => {
runtimeStatusProbeRetryTimers.delete(environmentId)
probeRuntimeStatus(environmentId, attempt + 1)
}, retryDelayMs)
)
})
}
unsubs.push(() => {
// The flag, not just the timers: a probe still in flight at teardown would
// otherwise schedule a fresh retry chain after the bridge is gone.
runtimeStatusProbesStopped = true
trailingRuntimeStatusProbes.clear()
for (const retryTimer of runtimeStatusProbeRetryTimers.values()) {
clearTimeout(retryTimer)
}
runtimeStatusProbeRetryTimers.clear()
})
const runtimeClientEventsSync = createRuntimeClientEventsSync({
getDesiredEnvironmentIds: () => getRuntimeClientEventEnvironmentIds(useAppStore.getState()),
getSubscriptionKey: (environmentId) => buildRuntimeClientEventEnvironmentKey([environmentId]),
@@ -271,7 +156,13 @@ export function registerRuntimeClientIpcBridge(
() => {
invalidateRuntimeClientEventReplay({
getSshStateReference: () => useAppStore.getState().sshStateByEnvironment,
refreshRuntimeStatus: () => probeRuntimeStatus(environmentId),
refreshRuntimeStatus: () => {
const state = useAppStore.getState()
const snapshot = state.runtimeStatusByEnvironmentId.get(environmentId)?.snapshot
if (!snapshot || snapshot.transport === 'unknown') {
void state.refreshRuntimeEnvironmentStatus(environmentId)
}
},
requestProjectRefresh: () => runtimeProjectRefreshScheduler.request(environmentId),
markEnvironmentSshStateStale: () =>
useAppStore.getState().markEnvironmentSshStateStale(environmentId),
@@ -281,19 +172,6 @@ export function registerRuntimeClientIpcBridge(
})
}
)
// Why: only a reconnect of an already-ready transport replays with the tag above.
// A connection whose first ready lands after the host recovered (app started, or
// the env was added, while it was down) never replays, so the recorded-unreachable
// verdict this subscribe just disproved has to be re-asked here. Kept off the
// returned promise so subscription registration/teardown ordering is unchanged.
void subscription.then(
() => {
if (isRuntimeStatusRecordedUnreachable(environmentId)) {
probeRuntimeStatus(environmentId, 0, 'recordedUnreachable')
}
},
() => {}
)
return subscription
},
onEvent: handleRuntimeClientEvent
@@ -67,16 +67,12 @@ describe('remote Orca server reconnect', () => {
}[] = []
let liveRuntimeId = 'remote-runtime'
let failingStatusProbes = 0
let blockNextStatusProbe = false
let releaseBlockedStatusProbe: (() => void) | null = null
beforeEach(() => {
subscriptionResponders = []
unsubs = []
liveRuntimeId = 'remote-runtime'
failingStatusProbes = 0
blockNextStatusProbe = false
releaseBlockedStatusProbe = null
// Module-level sonner double: without this a toast from an earlier test leaks into
// the assertions below.
vi.mocked(toast.warning).mockClear()
@@ -88,12 +84,6 @@ describe('remote Orca server reconnect', () => {
// Captured before the block so a probe that is still dialing answers with the
// runtime it was dispatched against, not with whatever restarted meanwhile.
const dispatchedRuntimeId = liveRuntimeId
if (blockNextStatusProbe) {
blockNextStatusProbe = false
await new Promise<void>((resolve) => {
releaseBlockedStatusProbe = resolve
})
}
if (failingStatusProbes > 0) {
failingStatusProbes -= 1
// status.get dials its own socket; it can fail while the control transport is up.
@@ -170,261 +160,42 @@ describe('remote Orca server reconnect', () => {
} as unknown as WorktreeEventRuntime)
}
it('re-probes a replayed subscription while the cached status still looks reachable', async () => {
it('keeps legacy event recovery as a single request, with retries owned outside the renderer', async () => {
vi.useFakeTimers()
startBridge()
await settle()
expect(sidebarHostHealth()).toBe('available')
// The gap was short enough that nothing probed during it, so the cached status still
// names the pre-restart runtime and the replay tag is the only evidence it is stale.
liveRuntimeId = 'remote-runtime-restarted'
failingStatusProbes = 1
replaySubscription()
await settle()
expect(
useAppStore.getState().runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status?.runtimeId
).toBe('remote-runtime-restarted')
expect(sidebarHostHealth()).toBe('available')
})
it('returns the sidebar host to online when the first subscription lands untagged', async () => {
// A connection that was never ready does not replay: nothing tags its first
// response, so a client that booted while the host was down has only the
// successful subscribe as evidence that the recorded verdict is stale.
useAppStore.setState({
runtimeStatusByEnvironmentId: new Map([
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
]) as never
})
expect(sidebarHostHealth()).toBe('disconnected')
startBridge()
await settle()
expect(sidebarHostHealth()).toBe('available')
expect(subscriptionResponders.length).toBeGreaterThan(0)
})
it('re-asks after a probe that failed while the transport stayed up', async () => {
vi.useFakeTimers()
// Already recorded unreachable, so the failing probe's `null` is an unchanged
// re-publication: it writes nothing and leaves no store transition for the
// resubscribe path to key off. Only a retry can still recover this host.
useAppStore.setState({
runtimeStatusByEnvironmentId: new Map([
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
]) as never
})
failingStatusProbes = 1
startBridge()
await settle()
expect(sidebarHostHealth()).toBe('disconnected')
await vi.advanceTimersByTimeAsync(2_000)
await settle()
expect(sidebarHostHealth()).toBe('available')
})
it('stops re-asking a host that keeps refusing, instead of polling it forever', async () => {
vi.useFakeTimers()
useAppStore.setState({
runtimeStatusByEnvironmentId: new Map([
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
]) as never
})
failingStatusProbes = Number.POSITIVE_INFINITY
startBridge()
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(120_000)
await settle()
// One probe on the successful subscribe plus the two bounded retries.
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(3)
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
})
it('stops probing once the bridge is torn down mid-probe', async () => {
vi.useFakeTimers()
it('does not add a status request when a connection-owned subscription replays', async () => {
useAppStore.getState().applyRuntimeHostStatusSnapshot({
environmentId: ENVIRONMENT_ID,
pairingRevision: 1,
sequence: 100,
checkedAt: 1,
transport: 'ready',
verification: 'verified',
status: liveRuntimeStatus()
})
startBridge()
await settle()
replaySubscription()
await settle()
expect(window.api.runtimeEnvironments.getStatus).not.toHaveBeenCalled()
expect(sidebarHostHealth()).toBe('available')
})
it('does not start UI recovery machinery when an initial subscription attaches', async () => {
useAppStore.setState({
runtimeStatusByEnvironmentId: new Map([
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
]) as never
})
failingStatusProbes = Number.POSITIVE_INFINITY
blockNextStatusProbe = true
startBridge()
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
// Teardown clears scheduled retries, but this probe has not answered yet.
stopBridge?.()
stopBridge = null
for (const unsub of unsubs.splice(0)) {
unsub()
}
releaseBlockedStatusProbe?.()
await settle()
await vi.advanceTimersByTimeAsync(120_000)
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
})
it('re-asks for a saved host that is not the active environment', async () => {
vi.useFakeTimers()
// A non-active host is only in the desired-subscription set while its status is non-null,
// so gating the retry on that set would strand it offline with its subscription already
// torn down the moment anything else (an explicit disconnect, the toast's own retry)
// records the same outage.
useAppStore.setState({
settings: { activeRuntimeEnvironmentId: 'env-laptop' } as never
runtimeStatusByEnvironmentId: new Map([[ENVIRONMENT_ID, { status: null, checkedAt: 1 }]])
})
startBridge()
await settle()
expect(sidebarHostHealth()).toBe('available')
failingStatusProbes = 1
replaySubscription()
await settle()
useAppStore.getState().setRuntimeEnvironmentStatus(ENVIRONMENT_ID, {
status: null,
checkedAt: Date.now()
})
expect(sidebarHostHealth()).toBe('disconnected')
await vi.advanceTimersByTimeAsync(2_000)
await settle()
expect(sidebarHostHealth()).toBe('available')
})
it('re-asks after a reconnect that lands while an earlier probe is still dialing', async () => {
blockNextStatusProbe = true
startBridge()
await settle()
replaySubscription()
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
// The host restarts while that probe is still on its own socket: the transport drops,
// reconnects and replays again. Serializing is right, dropping the request is not —
// the in-flight answer predates the restart this replay is reporting.
liveRuntimeId = 'remote-runtime-restarted'
replaySubscription()
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
releaseBlockedStatusProbe?.()
await settle()
expect(
useAppStore.getState().runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status?.runtimeId
).toBe('remote-runtime-restarted')
})
it('does not resurrect a host removed while a retry was pending', async () => {
vi.useFakeTimers()
useAppStore.setState({
runtimeStatusByEnvironmentId: new Map([
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
]) as never
})
failingStatusProbes = Number.POSITIVE_INFINITY
startBridge()
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
// The user deletes the remote server before the first retry fires. Settings can still
// name it as active until the next settings read, so removal is what has to stop this.
useAppStore.getState().setRuntimeEnvironments([])
await settle()
expect(useAppStore.getState().runtimeStatusByEnvironmentId.has(ENVIRONMENT_ID)).toBe(false)
await vi.advanceTimersByTimeAsync(120_000)
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
// buildExecutionHostRegistry enumerates the status map, so a re-published entry for a
// deleted id puts that host back in the sidebar under its raw id.
expect(useAppStore.getState().runtimeStatusByEnvironmentId.has(ENVIRONMENT_ID)).toBe(false)
})
it('keeps a live cached status when the replay-triggered probe fails on its own socket', async () => {
startBridge()
await settle()
expect(sidebarHostHealth()).toBe('available')
// Asserted over every write, not just the end state: a demotion that a later probe
// undoes still flashed the sidebar offline and still fired the toast.
let recordedUnreachable = false
unsubs.push(
useAppStore.subscribe((state) => {
recordedUnreachable ||=
state.runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status === null
})
)
// status.get dials its own short-lived socket, so its failure is unverifiable — and the
// transport that just replayed is proof the host is up. Recording it as offline would
// manufacture the stuck-offline sidebar this re-probe exists to cure.
failingStatusProbes = 1
replaySubscription()
await settle()
expect(recordedUnreachable).toBe(false)
expect(sidebarHostHealth()).toBe('available')
expect(toast.warning).not.toHaveBeenCalled()
})
it('returns a stuck-offline host to online when the replayed probe answers', async () => {
// The reported bug: the sidebar stayed offline after the connection recovered. The first
// probe failing keeps the recorded verdict unreachable, so only the replay recovers it
// (its retry chain is still parked behind a 2s timer this test never advances).
useAppStore.setState({
runtimeStatusByEnvironmentId: new Map([
[ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }]
]) as never
})
failingStatusProbes = 1
startBridge()
await settle()
expect(sidebarHostHealth()).toBe('disconnected')
replaySubscription()
await settle()
expect(sidebarHostHealth()).toBe('available')
})
it('does not dial a second status socket when the in-flight probe already answered', async () => {
startBridge()
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(0)
// A reconnect re-probes unconditionally, and that probe is still on its own socket.
blockNextStatusProbe = true
replaySubscription()
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
// Meanwhile the outage's own failed probe is recorded, which resubscribes; that
// resubscribe resolves against a cache that still reads unreachable.
useAppStore.getState().setRuntimeEnvironmentStatus(ENVIRONMENT_ID, {
status: null,
checkedAt: Date.now()
})
await settle()
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
releaseBlockedStatusProbe?.()
await settle()
// The resubscribe only wanted an answer for a host the cache called unreachable, and
// the probe it waited on gave one: a second status.get is a whole extra socket dial.
expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1)
expect(sidebarHostHealth()).toBe('available')
expect(window.api.runtimeEnvironments.getStatus).not.toHaveBeenCalled()
})
})
@@ -30,7 +30,7 @@ const EXPECTED_DIRECT_CALLBACK_METHODS = [
'runtime.onNativeChatLaunchDraftResolved',
'runtime.onTerminalDriverChanged',
'runtime.onTerminalFitOverrideChanged',
'runtimeEnvironments.onSharedControlDiagnostics',
'runtimeEnvironments.onStatusChanged',
'settings.onChanged',
'ssh.onCredentialRequest',
'ssh.onCredentialResolved',
@@ -106,7 +106,7 @@ const EXPECTED_DIRECT_CALLBACK_METHODS = [
const EXPECTED_CALLBACK_REGISTRATION_SEQUENCE = [
'ui.onMobileMarkdownRequest',
'automations.onChanged',
'runtimeEnvironments.onSharedControlDiagnostics',
'runtimeEnvironments.onStatusChanged',
'repos.onChanged',
'worktrees.onChanged',
'worktrees.onHeadIdentitiesChanged',
@@ -382,7 +382,7 @@ describe('useIpcEvents App-lifetime lifecycle', () => {
).toEqual([
'ui.onMobileMarkdownRequest',
'automations.onChanged',
'runtimeEnvironments.onSharedControlDiagnostics',
'runtimeEnvironments.onStatusChanged',
'runtimeEnvironments.subscribe',
...EXPECTED_CALLBACK_REGISTRATION_SEQUENCE.slice(3)
])
@@ -2,7 +2,9 @@ import { describe, expect, it } from 'vitest'
import type { RuntimeStatus } from '../../../shared/runtime-types'
import {
isConnectedRuntimeHostState,
isDisconnectedRuntimeHostState,
runtimeHostConnectionState,
runtimeHostConnectionStateForEntry,
runtimeStatusForOverall
} from './runtime-host-connection-state'
@@ -177,3 +179,72 @@ describe('runtime host connection state', () => {
).toBe('disconnected')
})
})
describe('runtime host connection state for a recorded status entry', () => {
it('separates a host that was never probed from one a probe found unreachable', () => {
// The sidebar read raw truthiness, which collapsed these two into the same red glyph.
expect(runtimeHostConnectionStateForEntry(undefined)).toBe('checking')
expect(runtimeHostConnectionStateForEntry({ status: null })).toBe('disconnected')
})
it('reads the remote-control diagnostics recorded beside a failed probe', () => {
expect(
runtimeHostConnectionStateForEntry({
status: null,
remoteControl: remoteControl('reconnecting')
})
).toBe('reconnecting')
})
it('agrees with the status bar that a closed control channel is disconnected', () => {
expect(
runtimeHostConnectionStateForEntry({
status: makeStatus({ remoteControl: remoteControl('closed') })
})
).toBe('disconnected')
})
it('names only the disconnected verdict as disconnected', () => {
expect(isDisconnectedRuntimeHostState('disconnected')).toBe(true)
for (const state of [
'connected',
'checking',
'reconnecting',
'runtime-unavailable',
'workspace-window-closed'
] as const) {
expect(isDisconnectedRuntimeHostState(state)).toBe(false)
}
})
})
function remoteControl(
state: NonNullable<RuntimeStatus['remoteControl']>['state']
): NonNullable<RuntimeStatus['remoteControl']> {
return {
state,
pendingRequestCount: 0,
subscriptionCount: 0,
reconnectAttempt: 1,
lastConnectedAt: null,
lastClose: null,
lastError: null
}
}
it('does not report reconnecting after verification is terminally blocked', () => {
expect(
runtimeHostConnectionStateForEntry({
status: null,
snapshot: {
environmentId: 'browser',
pairingRevision: 1,
sequence: 1,
checkedAt: 1,
status: null,
verification: 'blocked',
transport: 'disconnected'
}
})
).toBe('disconnected')
})
@@ -1,3 +1,4 @@
import type { RuntimeHostStatusSnapshot } from '../../../shared/runtime-host-status'
import type { RuntimeStatus } from '../../../shared/runtime-types'
import { isRuntimeWorkspaceWindowClosed } from '../../../shared/runtime-workspace-window-availability'
@@ -97,3 +98,44 @@ export function isConnectedRuntimeHostState(state: RuntimeHostConnectionState):
state === 'connected' || state === 'runtime-unavailable' || state === 'workspace-window-closed'
)
}
/**
* Only this verdict earns the destructive glyph. 'checking' and 'reconnecting' are
* unverifiable, not down, per docs/reference/ssh-execution-boundary.md.
*/
export function isDisconnectedRuntimeHostState(state: RuntimeHostConnectionState): boolean {
return state === 'disconnected'
}
/** The same derivation, read straight off a recorded status entry. */
export function runtimeHostConnectionStateForEntry(
entry:
| {
status: RuntimeStatus | null
remoteControl?: RuntimeStatus['remoteControl'] | null
snapshot?: RuntimeHostStatusSnapshot
}
| null
| undefined
): RuntimeHostConnectionState {
if (entry?.snapshot) {
const snapshot = entry.snapshot
if (snapshot.retired || snapshot.verification === 'blocked') {
return 'disconnected'
}
if (snapshot.transport === 'disconnected') {
return 'reconnecting'
}
if (snapshot.verification === 'checking' && !entry.status) {
return 'checking'
}
if (snapshot.transport === 'ready' && snapshot.verification !== 'verified') {
return 'runtime-unavailable'
}
}
return runtimeHostConnectionState({
hasStatusEntry: Boolean(entry),
status: entry?.status ?? null,
remoteControl: entry?.remoteControl ?? entry?.status?.remoteControl ?? null
})
}
@@ -1,52 +0,0 @@
import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version'
import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types'
import type { RuntimeEnvironmentStatus } from './runtime-status'
const diagnosticsGenerationByEnvironment = new Map<string, number>()
export function updateRuntimeEnvironmentStatusOverlay(
state: Map<string, RuntimeEnvironmentStatus>,
environmentId: string,
status: RuntimeEnvironmentStatus
): Map<string, RuntimeEnvironmentStatus> {
const current = state.get(environmentId)
if (!current || current.status?.runtimeId !== status.status?.runtimeId) {
return state
}
return new Map(state).set(environmentId, status)
}
export function acceptRuntimeEnvironmentDiagnosticsGeneration(
environmentId: string,
transportGeneration: number
): boolean {
const previous = diagnosticsGenerationByEnvironment.get(environmentId)
if (previous !== undefined && transportGeneration < previous) {
return false
}
diagnosticsGenerationByEnvironment.set(environmentId, transportGeneration)
return true
}
export function clearRuntimeEnvironmentDiagnosticsGenerationsForTests(): void {
diagnosticsGenerationByEnvironment.clear()
}
export function mergePushedRuntimeEnvironmentDiagnostics(args: {
environmentId: string
transportGeneration: number
diagnostics: RemoteRuntimeSharedConnectionDiagnostics
current: RuntimeEnvironmentStatus | undefined
publish: (status: RuntimeEnvironmentStatus) => void
}): void {
if (
!args.current?.status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) ||
!acceptRuntimeEnvironmentDiagnosticsGeneration(args.environmentId, args.transportGeneration)
) {
return
}
args.publish({
...args.current,
status: { ...args.current.status, remoteControl: args.diagnostics }
})
}
@@ -1,106 +0,0 @@
import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types'
import type { AppState } from '../types'
import type { RuntimeEnvironmentStatus } from './runtime-status'
import * as diagnosticsGeneration from './runtime-status-diagnostics-generation'
import * as runtimeStatusRecheck from './runtime-status-recheck'
export function updateRuntimeStatusStore(
state: AppState,
updater: (state: Map<string, RuntimeEnvironmentStatus>) => Map<string, RuntimeEnvironmentStatus>
): AppState | Pick<AppState, 'runtimeStatusByEnvironmentId'> {
const next = updater(state.runtimeStatusByEnvironmentId)
return next === state.runtimeStatusByEnvironmentId
? state
: { runtimeStatusByEnvironmentId: next }
}
export function publishRuntimeEnvironmentDiagnostics(args: {
environmentId: string
transportGeneration: number
diagnostics: RemoteRuntimeSharedConnectionDiagnostics
getCurrent: () => RuntimeEnvironmentStatus | undefined
updateState: (status: RuntimeEnvironmentStatus) => boolean
afterPublish?: (status: RuntimeEnvironmentStatus) => void
}): void {
diagnosticsGeneration.mergePushedRuntimeEnvironmentDiagnostics({
environmentId: args.environmentId,
transportGeneration: args.transportGeneration,
diagnostics: args.diagnostics,
current: args.getCurrent(),
publish: (status) => {
if (args.updateState(status)) {
args.afterPublish?.(status)
}
}
})
}
export function applyRuntimeEnvironmentStatusOverlay(args: {
environmentId: string
status: RuntimeEnvironmentStatus
setState: (
updater: (state: Map<string, RuntimeEnvironmentStatus>) => Map<string, RuntimeEnvironmentStatus>
) => void
}): boolean {
let updated = false
args.setState((state) => {
const next = diagnosticsGeneration.updateRuntimeEnvironmentStatusOverlay(
state,
args.environmentId,
args.status
)
updated = next !== state
return next
})
return updated
}
export function createRuntimeEnvironmentDiagnosticsPublisher(args: {
getCurrent: (environmentId: string) => RuntimeEnvironmentStatus | undefined
setState: (
updater: (state: Map<string, RuntimeEnvironmentStatus>) => Map<string, RuntimeEnvironmentStatus>
) => void
afterPublish: (environmentId: string, status: RuntimeEnvironmentStatus) => void
}): (event: {
environmentId: string
transportGeneration: number
diagnostics: RemoteRuntimeSharedConnectionDiagnostics
}) => void {
return (event) =>
publishRuntimeEnvironmentDiagnostics({
...event,
getCurrent: () => args.getCurrent(event.environmentId),
updateState: (status) =>
applyRuntimeEnvironmentStatusOverlay({
environmentId: event.environmentId,
status,
setState: args.setState
}),
afterPublish: (status) => args.afterPublish(event.environmentId, status)
})
}
export function createRuntimeEnvironmentDiagnosticsSlicePublisher(args: {
getCurrent: (environmentId: string) => RuntimeEnvironmentStatus | undefined
setState: (
updater: (state: Map<string, RuntimeEnvironmentStatus>) => Map<string, RuntimeEnvironmentStatus>
) => void
getStore: () => AppState
getConnectionGeneration: (environmentId: string) => number
}): (event: {
environmentId: string
transportGeneration: number
diagnostics: RemoteRuntimeSharedConnectionDiagnostics
}) => void {
return createRuntimeEnvironmentDiagnosticsPublisher({
getCurrent: args.getCurrent,
setState: args.setState,
afterPublish: (environmentId, status) =>
runtimeStatusRecheck.reconcileRuntimeStatusForSlice(
environmentId,
status.status,
args.getStore,
() => args.getConnectionGeneration(environmentId)
)
})
}
@@ -1,89 +0,0 @@
import { describe, expect, it } from 'vitest'
import { create } from 'zustand'
import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version'
import type { RuntimeStatus } from '../../../../shared/runtime-types'
import { createRuntimeStatusSlice, type RuntimeStatusSlice } from './runtime-status'
function makeStatus(overrides: Partial<RuntimeStatus> = {}): RuntimeStatus {
return {
runtimeId: 'runtime-a',
rendererGraphEpoch: 0,
graphStatus: 'ready',
authoritativeWindowId: null,
liveTabCount: 3,
liveLeafCount: 0,
runtimeProtocolVersion: 3,
minCompatibleRuntimeClientVersion: 3,
capabilities: ['browser.screencast.v1'],
...overrides
} as RuntimeStatus
}
function createSliceStore() {
return create<RuntimeStatusSlice>()((...a) => ({
...createRuntimeStatusSlice(...(a as unknown as Parameters<typeof createRuntimeStatusSlice>))
}))
}
describe('runtime-status diagnostics', () => {
it('merges transport diagnostics into the complete status and fences stale pushes', () => {
const store = createSliceStore()
const status = makeStatus({
capabilities: ['browser.screencast.v1', REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY]
})
store.getState().setRuntimeEnvironmentStatus('env-a', { status, checkedAt: 1 })
const closed = {
state: 'closed' as const,
pendingRequestCount: 0,
subscriptionCount: 1,
reconnectAttempt: 2,
lastConnectedAt: 1,
lastClose: { code: 1006, reason: 'network' },
lastError: 'connection lost'
}
store.getState().publishRuntimeEnvironmentDiagnostics({
environmentId: 'env-a',
transportGeneration: 3,
diagnostics: closed
})
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toMatchObject({
runtimeId: 'runtime-a',
capabilities: expect.arrayContaining([
'browser.screencast.v1',
REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY
]),
liveTabCount: 3,
remoteControl: closed
})
store.getState().publishRuntimeEnvironmentDiagnostics({
environmentId: 'env-a',
transportGeneration: 2,
diagnostics: { ...closed, state: 'ready' }
})
expect(
store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl?.state
).toBe('closed')
})
it('ignores diagnostics after the latest status drops shared-control support', () => {
const store = createSliceStore()
const status = makeStatus({ capabilities: [] })
store.getState().setRuntimeEnvironmentStatus('env-a', { status, checkedAt: 1 })
store.getState().publishRuntimeEnvironmentDiagnostics({
environmentId: 'env-a',
transportGeneration: 3,
diagnostics: {
state: 'reconnecting',
pendingRequestCount: 0,
subscriptionCount: 1,
reconnectAttempt: 2,
lastConnectedAt: 1,
lastClose: { code: 1006, reason: 'network' },
lastError: 'connection lost'
}
})
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(status)
})
})
@@ -1,238 +0,0 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { create } from 'zustand'
import { toast } from 'sonner'
import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version'
import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments'
import type { RuntimeStatus } from '../../../../shared/runtime-types'
import {
clearRuntimeEnvironmentConnectionGenerationsForTests,
createRuntimeStatusSlice,
setRuntimeEnvironmentConnectionGenerationForTests,
type RuntimeStatusSlice
} from './runtime-status'
import { clearRuntimeStatusRechecksForTests } from './runtime-status-recheck'
vi.mock('sonner', () => ({ toast: { warning: vi.fn(), dismiss: vi.fn() } }))
beforeEach(() => {
vi.useFakeTimers()
clearRuntimeStatusRechecksForTests()
clearRuntimeEnvironmentConnectionGenerationsForTests()
vi.mocked(toast.warning).mockReset()
})
afterEach(() => {
clearRuntimeStatusRechecksForTests()
vi.useRealTimers()
vi.unstubAllGlobals()
})
describe('runtime status recheck', () => {
it('publishes an observe-only ready result through the setter', async () => {
const getStatus = vi.fn().mockResolvedValue(response(status('ready')))
const store = createStore(getStatus)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_ready'),
checkedAt: 1
})
await vi.advanceTimersByTimeAsync(3_000)
expect(getStatus).toHaveBeenCalledWith({
selector: 'env-a',
timeoutMs: 10_000,
observeOnly: true
})
expect(
store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl
).toMatchObject({
state: 'ready'
})
await vi.advanceTimersByTimeAsync(120_000)
expect(getStatus).toHaveBeenCalledOnce()
})
it('continues indefinitely on the capped ladder, including unchanged publishes', async () => {
const getStatus = vi.fn().mockResolvedValue(response(status('reconnecting')))
const store = createStore(getStatus)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('reconnecting'),
checkedAt: 1
})
await vi.advanceTimersByTimeAsync(3_000 + 6_000 + 12_000 + 30_000 + 60_000 + 60_000)
expect(getStatus).toHaveBeenCalledTimes(6)
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.checkedAt).toBe(1)
})
it('cancels on removal, capability loss, and null without probing again', async () => {
const getStatus = vi.fn()
const store = createStore(getStatus)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_authenticated'),
checkedAt: 1
})
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: { ...status('awaiting_authenticated'), capabilities: [] },
checkedAt: 2
})
await vi.advanceTimersByTimeAsync(60_000)
expect(getStatus).not.toHaveBeenCalled()
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_ready'),
checkedAt: 3
})
store.getState().setRuntimeEnvironments([])
await vi.advanceTimersByTimeAsync(60_000)
expect(getStatus).not.toHaveBeenCalled()
})
it('cancels an armed probe when the connection generation changes', async () => {
const getStatus = vi.fn()
const store = createStore(getStatus)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_ready'),
checkedAt: 1
})
setRuntimeEnvironmentConnectionGenerationForTests('env-a', 2)
await vi.advanceTimersByTimeAsync(3_000)
expect(getStatus).not.toHaveBeenCalled()
})
it('restarts the ladder for a newly published connection generation', async () => {
const getStatus = vi.fn().mockResolvedValue(response(status('ready', 'rt-next')))
const store = createStore(getStatus)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_ready'),
checkedAt: 1
})
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_ready', 'rt-next'),
checkedAt: 2
})
await vi.advanceTimersByTimeAsync(3_000)
expect(getStatus).toHaveBeenCalledOnce()
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.runtimeId).toBe(
'rt-next'
)
})
it('discards an in-flight result after a ready publish bumps the epoch', async () => {
const pending = deferred<ReturnType<typeof response>>()
const getStatus = vi.fn().mockReturnValue(pending.promise)
const store = createStore(getStatus)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_ready'),
checkedAt: 1
})
await vi.advanceTimersByTimeAsync(3_000)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('ready'),
checkedAt: 2
})
pending.resolve(response(status('reconnecting')))
await Promise.resolve()
await Promise.resolve()
expect(
store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl
).toMatchObject({
state: 'ready'
})
})
it('keeps setter side effects when a recheck discovers disconnection', async () => {
const getStatus = vi.fn().mockResolvedValue({
id: 'status.get',
ok: false,
error: {
code: 'runtime_unavailable',
message: 'offline',
data: { remoteControl: status('reconnecting').remoteControl }
},
_meta: { runtimeId: 'rt' }
})
const store = createStore(getStatus)
store.getState().setRuntimeEnvironmentStatus('env-a', {
status: status('awaiting_ready'),
checkedAt: 1
})
await vi.advanceTimersByTimeAsync(3_000)
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBeNull()
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.remoteControl).toMatchObject(
{
state: 'reconnecting'
}
)
expect(toast.warning).toHaveBeenCalledOnce()
})
})
function createStore(getStatus: ReturnType<typeof vi.fn>) {
vi.stubGlobal('window', {
api: { runtimeEnvironments: { getStatus, list: vi.fn() } }
})
const store = create<RuntimeStatusSlice>()((...args) => ({
...createRuntimeStatusSlice(...(args as unknown as Parameters<typeof createRuntimeStatusSlice>))
}))
store.getState().setRuntimeEnvironments([environment()])
return store
}
function status(
controlState: NonNullable<RuntimeStatus['remoteControl']>['state'],
runtimeId = 'rt'
): RuntimeStatus {
return {
runtimeId,
rendererGraphEpoch: 1,
graphStatus: 'ready',
authoritativeWindowId: null,
liveTabCount: 0,
liveLeafCount: 0,
capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY],
remoteControl: {
state: controlState,
pendingRequestCount: 0,
subscriptionCount: 0,
reconnectAttempt: 1,
lastConnectedAt: null,
lastClose: null,
lastError: null
}
} as RuntimeStatus
}
function response(result: RuntimeStatus) {
return { id: 'status.get', ok: true as const, result, _meta: { runtimeId: result.runtimeId } }
}
function environment(): PublicKnownRuntimeEnvironment {
return {
id: 'env-a',
name: 'Dev Box',
createdAt: 1,
updatedAt: 1,
lastUsedAt: null,
runtimeId: 'rt',
endpoints: [{ id: 'ws', kind: 'websocket', label: 'WebSocket', endpoint: 'ws://x' }],
preferredEndpointId: 'ws'
}
}
function deferred<T>() {
let resolve: (value: T) => void = () => {}
const promise = new Promise<T>((done) => {
resolve = done
})
return { promise, resolve }
}
@@ -1,167 +0,0 @@
import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version'
import type { RuntimeStatus } from '../../../../shared/runtime-types'
import { unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client'
import { extractRuntimeTransportDiagnostics } from '@/runtime/runtime-status-probe-diagnostics'
import type { RuntimeEnvironmentStatus } from './runtime-status'
const RECHECK_DELAYS_MS = [3_000, 6_000, 12_000, 30_000, 60_000]
type RecheckState = {
epoch: number
attempt: number
timer: ReturnType<typeof setTimeout> | null
inFlight: boolean
connectionGeneration: number
environmentExists: () => boolean
getConnectionGeneration: () => number
publish: (status: RuntimeEnvironmentStatus) => void
}
type RuntimeStatusStore = {
runtimeEnvironments: readonly { id: string }[]
setRuntimeEnvironmentStatus: (environmentId: string, status: RuntimeEnvironmentStatus) => void
}
const rechecks = new Map<string, RecheckState>()
export function reconcileRuntimeStatusRecheck(args: {
environmentId: string
status: RuntimeStatus | null
connectionGeneration: number
environmentExists: () => boolean
getConnectionGeneration: () => number
publish: (status: RuntimeEnvironmentStatus) => void
}): void {
if (!shouldRecheck(args.status)) {
cancelRuntimeStatusRecheck(args.environmentId)
return
}
let state = rechecks.get(args.environmentId)
if (state && state.connectionGeneration !== args.connectionGeneration) {
cancelRuntimeStatusRecheck(args.environmentId)
state = undefined
}
if (!state) {
state = {
epoch: 0,
attempt: 0,
timer: null,
inFlight: false,
connectionGeneration: args.connectionGeneration,
environmentExists: args.environmentExists,
getConnectionGeneration: args.getConnectionGeneration,
publish: args.publish
}
rechecks.set(args.environmentId, state)
} else {
state.connectionGeneration = args.connectionGeneration
state.environmentExists = args.environmentExists
state.getConnectionGeneration = args.getConnectionGeneration
state.publish = args.publish
}
armRuntimeStatusRecheck(args.environmentId, state)
}
export function reconcileRuntimeStatusForSlice(
environmentId: string,
status: RuntimeStatus | null,
get: () => RuntimeStatusStore,
getConnectionGeneration: () => number
): void {
reconcileRuntimeStatusRecheck({
environmentId,
status,
connectionGeneration: getConnectionGeneration(),
environmentExists: () =>
get().runtimeEnvironments.some((environment) => environment.id === environmentId),
getConnectionGeneration,
publish: (nextStatus) => get().setRuntimeEnvironmentStatus(environmentId, nextStatus)
})
}
export function cancelRuntimeStatusRecheck(environmentId: string): void {
const state = rechecks.get(environmentId)
if (!state) {
return
}
state.epoch += 1
if (state.timer) {
clearTimeout(state.timer)
}
rechecks.delete(environmentId)
}
export function cancelRuntimeStatusRechecks(environmentIds: Iterable<string>): void {
for (const environmentId of environmentIds) {
cancelRuntimeStatusRecheck(environmentId)
}
}
export function clearRuntimeStatusRechecksForTests(): void {
cancelRuntimeStatusRechecks([...rechecks.keys()])
}
function shouldRecheck(status: RuntimeStatus | null): boolean {
return Boolean(
status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) &&
status.remoteControl &&
status.remoteControl.state !== 'ready'
)
}
function armRuntimeStatusRecheck(environmentId: string, state: RecheckState): void {
if (state.timer || state.inFlight) {
return
}
const delay = RECHECK_DELAYS_MS[Math.min(state.attempt, RECHECK_DELAYS_MS.length - 1)]
const generation = state.connectionGeneration
state.attempt += 1
state.timer = setTimeout(
() => void fireRuntimeStatusRecheck(environmentId, state, generation),
delay
)
}
async function fireRuntimeStatusRecheck(
environmentId: string,
state: RecheckState,
generation: number
): Promise<void> {
state.timer = null
const epoch = state.epoch
if (
rechecks.get(environmentId) !== state ||
!state.environmentExists() ||
state.getConnectionGeneration() !== generation
) {
cancelRuntimeStatusRecheck(environmentId)
return
}
state.inFlight = true
let nextEntry: RuntimeEnvironmentStatus
try {
const response = await window.api.runtimeEnvironments.getStatus({
selector: environmentId,
timeoutMs: 10_000,
observeOnly: true
})
nextEntry = { status: unwrapRuntimeRpcResult<RuntimeStatus>(response), checkedAt: Date.now() }
} catch (error: unknown) {
const remoteControl = extractRuntimeTransportDiagnostics(error)
nextEntry = {
status: null,
...(remoteControl ? { remoteControl } : {}),
checkedAt: Date.now()
}
}
state.inFlight = false
if (
rechecks.get(environmentId) !== state ||
state.epoch !== epoch ||
!state.environmentExists() ||
state.getConnectionGeneration() !== generation
) {
return
}
state.publish(nextEntry)
}
@@ -15,6 +15,22 @@ export async function refreshRuntimeEnvironmentStatus(
selector: environmentId,
timeoutMs
})
if (window.api.runtimeEnvironments.getStatusSnapshots) {
try {
const snapshots = await window.api.runtimeEnvironments.getStatusSnapshots()
const snapshot = snapshots.find((entry) => entry.environmentId === environmentId)
if (snapshot) {
publish({
snapshot,
status: snapshot.verification === 'verified' ? snapshot.status : null,
checkedAt: snapshot.checkedAt
})
}
} catch (error) {
console.error('Failed to read runtime host status snapshot:', error)
}
return response.ok
}
const status = unwrapRuntimeRpcResult<RuntimeStatus>(response)
if (getRuntimeEnvironmentRevision(environmentId) !== expectedEnvironmentRevision) {
return false
@@ -73,14 +73,10 @@ describe('restored client-hosted browser host attach on reachability', () => {
})
})
// The reconnect policy suppresses the *failure* publish only. A probe that answered still owes
// both recovery follow-ups, or a restored client-hosted page never comes back after the gap.
it('runs both recovery follow-ups on a success when the caller opted out of publishing failures', async () => {
it('runs both recovery follow-ups after a successful refresh', async () => {
stubApi(vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a')))
await storeWithRestoredHandles(true)
.getState()
.refreshRuntimeEnvironmentStatus('env-a', undefined, { publishUnreachable: false })
await storeWithRestoredHandles(true).getState().refreshRuntimeEnvironmentStatus('env-a')
expect(prepareBrowserClientHostPlacement).toHaveBeenCalledWith({
selector: 'env-a',
@@ -89,24 +85,12 @@ describe('restored client-hosted browser host attach on reachability', () => {
expect(replayClientHostedBrowserCloseIntents).toHaveBeenCalledWith('env-a', expect.anything())
})
// Under either policy a failed probe owes *no* follow-ups: it verified nothing, so there is no
// recovered host to reattach restored pages to and no one to replay closes at.
it.each([
{ name: 'the default policy', options: undefined },
{ name: 'a caller that opted out of publishing', options: { publishUnreachable: false } }
])(
'starts no browser client host when the environment is unreachable: $name',
async (scenario) => {
stubApi(vi.fn().mockRejectedValue(new Error('unreachable')))
await storeWithRestoredHandles(true)
.getState()
.refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options)
expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled()
expect(replayClientHostedBrowserCloseIntents).not.toHaveBeenCalled()
}
)
it('runs no recovery follow-ups when the environment is unreachable', async () => {
stubApi(vi.fn().mockRejectedValue(new Error('unreachable')))
await storeWithRestoredHandles(true).getState().refreshRuntimeEnvironmentStatus('env-a')
expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled()
expect(replayClientHostedBrowserCloseIntents).not.toHaveBeenCalled()
})
it('starts no browser client host for restored pages the server hosts', async () => {
stubApi(vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a')))
@@ -0,0 +1,111 @@
import { beforeEach, expect, it, vi } from 'vitest'
import { create } from 'zustand'
import { toast } from 'sonner'
import {
createRuntimeStatusSlice,
clearRuntimeEnvironmentConnectionGenerationsForTests,
type RuntimeStatusSlice
} from './runtime-status'
import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status'
import type { RuntimeStatus } from '../../../../shared/runtime-types'
import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments'
import { runtimeHostConnectionStateForEntry } from '@/runtime/runtime-host-connection-state'
vi.mock('sonner', () => ({ toast: { warning: vi.fn(), dismiss: vi.fn() } }))
vi.mock('@/runtime/restored-client-hosted-browser-host-attach', () => ({
ensureBrowserClientHostsForRestoredPages: vi.fn(),
ensureBrowserClientHostForRestartedRuntime: vi.fn()
}))
vi.mock('@/runtime/client-hosted-browser-close-intent-replay', () => ({
replayClientHostedBrowserCloseIntents: vi.fn()
}))
beforeEach(() => {
clearRuntimeEnvironmentConnectionGenerationsForTests()
vi.clearAllMocks()
})
const environment = {
id: 'env-a',
name: 'Host',
createdAt: 1,
pairingRevision: 1,
endpoints: [],
preferredEndpointId: ''
} as unknown as PublicKnownRuntimeEnvironment
function store() {
const value = create<RuntimeStatusSlice>()((...args) =>
createRuntimeStatusSlice(...(args as unknown as Parameters<typeof createRuntimeStatusSlice>))
)
value.getState().setRuntimeEnvironments([environment])
return value
}
function snapshot(
sequence: number,
patch: Partial<RuntimeHostStatusSnapshot> = {}
): RuntimeHostStatusSnapshot {
return {
environmentId: 'env-a',
pairingRevision: 1,
sequence,
checkedAt: sequence,
transport: 'ready',
verification: 'verified',
status: { runtimeId: 'rt-1' } as RuntimeStatus,
...patch
}
}
it('hydrates both viewers and rejects an older read after a newer publication', () => {
for (const viewer of [store(), store()]) {
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2))
viewer
.getState()
.applyRuntimeHostStatusSnapshot(snapshot(1, { status: null, verification: 'unavailable' }))
expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.runtimeId).toBe(
'rt-1'
)
}
})
it('represents failed verification honestly without manufacturing a session restart or toast', () => {
const viewer = store()
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(1))
const generation = viewer
.getState()
.runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2, { verification: 'unavailable' }))
expect(
runtimeHostConnectionStateForEntry(viewer.getState().runtimeStatusByEnvironmentId.get('env-a'))
).toBe('runtime-unavailable')
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(3))
expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration).toBe(
generation
)
expect(toast.warning).not.toHaveBeenCalled()
viewer
.getState()
.applyRuntimeHostStatusSnapshot(snapshot(4, { status: { runtimeId: 'rt-2' } as RuntimeStatus }))
expect(
viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration
).toBeGreaterThan(generation ?? 0)
})
it('retains disconnect ordering and rejects publications for removed or replaced pairings', () => {
const viewer = store()
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(1))
viewer
.getState()
.applyRuntimeHostStatusSnapshot(
snapshot(3, { retired: true, verification: 'blocked', transport: 'disconnected' })
)
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2))
expect(
runtimeHostConnectionStateForEntry(viewer.getState().runtimeStatusByEnvironmentId.get('env-a'))
).toBe('disconnected')
viewer.getState().setRuntimeEnvironments([{ ...environment, pairingRevision: 2 }])
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(4))
expect(viewer.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false)
viewer.getState().setRuntimeEnvironments([])
viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(5, { pairingRevision: 2 }))
expect(viewer.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false)
})
@@ -0,0 +1,43 @@
import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status'
import type { AppState } from '../types'
import type { RuntimeEnvironmentStatus } from './runtime-status-types'
import { ensureBrowserClientHostsForRestoredPages } from '@/runtime/restored-client-hosted-browser-host-attach'
import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay'
export function applyRuntimeHostStatusSnapshot(
snapshot: RuntimeHostStatusSnapshot,
state: AppState,
publishEvidence: (entry: RuntimeEnvironmentStatus) => void
): void {
const environment = state.runtimeEnvironments.find((entry) => entry.id === snapshot.environmentId)
if (
!environment ||
(environment.pairingRevision ?? environment.createdAt) !== snapshot.pairingRevision
) {
return
}
const previous = state.runtimeStatusByEnvironmentId.get(snapshot.environmentId)
if (previous?.snapshot && previous.snapshot.sequence >= snapshot.sequence) {
return
}
const entry: RuntimeEnvironmentStatus = {
snapshot,
checkedAt: snapshot.checkedAt,
connectionGeneration: previous?.connectionGeneration,
status: snapshot.verification === 'verified' && !snapshot.retired ? snapshot.status : null,
remoteControl: snapshot.remoteControl
}
if (entry.status) {
if (snapshot.remoteControl) {
entry.status = { ...entry.status, remoteControl: snapshot.remoteControl }
}
state.setRuntimeEnvironmentStatus(snapshot.environmentId, entry)
if (previous?.status == null) {
void ensureBrowserClientHostsForRestoredPages(state)
void replayClientHostedBrowserCloseIntents(snapshot.environmentId, state)
}
} else {
// Lost contact or a failed method observes no runtime session ending.
publishEvidence(entry)
}
}
@@ -1,8 +1,9 @@
import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status'
import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments'
import type { RuntimeStatus } from '../../../../shared/runtime-types'
import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types'
export type RuntimeEnvironmentStatus = {
snapshot?: RuntimeHostStatusSnapshot
status: RuntimeStatus | null
remoteControl?: RuntimeStatus['remoteControl'] | null
appVersion?: string | null
@@ -10,11 +11,9 @@ export type RuntimeEnvironmentStatus = {
connectionGeneration?: number
}
export type RuntimeStatusRefreshOptions = {
publishUnreachable?: boolean
}
export type RuntimeStatusSlice = {
readRuntimeHostStatusSnapshots: () => Promise<void>
applyRuntimeHostStatusSnapshot: (snapshot: RuntimeHostStatusSnapshot) => void
runtimeEnvironments: readonly PublicKnownRuntimeEnvironment[]
runtimeEnvironmentCatalogHydrated: boolean
runtimeEnvironmentCatalogSettled: boolean
@@ -26,17 +25,8 @@ export type RuntimeStatusSlice = {
status: RuntimeEnvironmentStatus,
options?: { suppressDisconnectToast?: boolean }
) => void
publishRuntimeEnvironmentDiagnostics: (args: {
environmentId: string
transportGeneration: number
diagnostics: RemoteRuntimeSharedConnectionDiagnostics
}) => void
clearRuntimeEnvironmentStatus: (environmentId: string) => void
retainRuntimeEnvironmentStatuses: (environmentIds: Iterable<string>) => void
refreshRuntimeEnvironmentStatus: (
environmentId: string,
timeoutMs?: number,
options?: RuntimeStatusRefreshOptions
) => Promise<boolean>
refreshRuntimeEnvironmentStatus: (environmentId: string, timeoutMs?: number) => Promise<boolean>
hydrateRuntimeEnvironmentStatuses: () => Promise<void>
}
@@ -710,33 +710,37 @@ describe('runtime-status slice', () => {
clearRuntimeCompatibilityCacheForTests()
})
// Both directions of the failure-publication policy, from one failing probe. A user-initiated
// check publishes the outage it just observed; a caller holding live transport evidence must
// not, because status.get dials its own socket and its failure is unverifiable, not exited.
it.each([
{ name: 'a user-initiated check', options: undefined, publishes: true },
{ name: 'publishUnreachable defaulted', options: {}, publishes: true },
{
name: 'a caller that opted out of publishing',
options: { publishUnreachable: false },
publishes: false
}
])('records null and returns false when a runtime refresh fails: $name', async (scenario) => {
it('records null and returns false when a runtime refresh fails', async () => {
const getStatus = vi.fn().mockRejectedValue(new Error('closed'))
stubRuntimeEnvironmentApi({ getStatus })
const store = createSliceStore()
const cached = makeStatus()
store.getState().setRuntimeEnvironmentStatus('env-a', { status: cached, checkedAt: 1 })
const reachable = await store
.getState()
.refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options)
const reachable = await store.getState().refreshRuntimeEnvironmentStatus('env-a')
// The dial-answered contract the bridge's bounded retry chain reads is policy-independent.
expect(reachable).toBe(false)
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(
scenario.publishes ? null : cached
)
expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(null)
})
it('preserves successful reachability when reading its snapshot fails', async () => {
const log = vi.spyOn(console, 'error').mockImplementation(() => {})
vi.stubGlobal('window', {
api: {
runtimeEnvironments: {
getStatus: vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a')),
getStatusSnapshots: vi.fn().mockRejectedValue(new Error('IPC read failed'))
}
}
})
try {
const store = createSliceStore()
expect(await store.getState().refreshRuntimeEnvironmentStatus('env-a')).toBe(true)
expect(store.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false)
expect(log).toHaveBeenCalled()
} finally {
log.mockRestore()
}
})
it('hydrates saved environments through the single-environment refresh path', async () => {
+33 -42
View File
@@ -1,11 +1,7 @@
import type { StateCreator } from 'zustand'
import type { AppState } from '../types'
import type { RuntimeStatusSlice } from './runtime-status-types'
export type {
RuntimeEnvironmentStatus,
RuntimeStatusRefreshOptions,
RuntimeStatusSlice
} from './runtime-status-types'
export type { RuntimeEnvironmentStatus, RuntimeStatusSlice } from './runtime-status-types'
import { runtimeEnvironmentStatusesEqual } from './runtime-environment-status-equality'
import {
clearRecentRuntimeCompatibilityFailure,
@@ -20,21 +16,16 @@ import {
import { reconcileCatalogRows } from './repo-identity-reconcile'
import { createRuntimeStatusHydration } from './runtime-status-hydration'
import { refreshRuntimeEnvironmentStatus } from './runtime-status-refresh'
import * as runtimeStatusDiagnostics from './runtime-status-diagnostics-generation'
import * as runtimeStatusConnectionGeneration from './runtime-status-connection-generation'
import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay'
import {
ensureBrowserClientHostForRestartedRuntime,
ensureBrowserClientHostsForRestoredPages
} from '@/runtime/restored-client-hosted-browser-host-attach'
import * as runtimeStatusRecheck from './runtime-status-recheck'
import * as runtimeStatusDiagnosticsPublish from './runtime-status-diagnostics-publish'
import { applyRuntimeHostStatusSnapshot } from './runtime-status-snapshot'
export const clearRuntimeEnvironmentConnectionGenerationsForTests = (): void => {
runtimeStatusRecheck.cancelRuntimeStatusRechecks(
runtimeStatusConnectionGeneration.clearRuntimeEnvironmentConnectionGenerations()
)
runtimeStatusDiagnostics.clearRuntimeEnvironmentDiagnosticsGenerationsForTests()
runtimeStatusConnectionGeneration.clearRuntimeEnvironmentConnectionGenerations()
}
export {
@@ -52,6 +43,15 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
runtimeStatusByEnvironmentId: new Map(),
removedRuntimeEnvironmentIds: new Set(),
readRuntimeHostStatusSnapshots: async () => {
try {
const snapshots = await window.api.runtimeEnvironments.getStatusSnapshots()
snapshots.forEach((snapshot) => get().applyRuntimeHostStatusSnapshot(snapshot))
} catch (error) {
console.error('Failed to read runtime host status:', error)
}
},
setRuntimeEnvironments: (environments) => {
const previousRevisionById = new Map(
get().runtimeEnvironments.map((environment) => [
@@ -76,7 +76,6 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
const removedIds = get()
.runtimeEnvironments.map((environment) => environment.id)
.filter((id) => !nextIds.has(id))
runtimeStatusRecheck.cancelRuntimeStatusRechecks([...removedIds, ...replacedEnvironmentIds])
set((s) => {
const keep = new Set(environments.map((environment) => environment.id))
const nextStatuses = new Map(s.runtimeStatusByEnvironmentId)
@@ -155,15 +154,29 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
}
},
applyRuntimeHostStatusSnapshot: (snapshot) =>
applyRuntimeHostStatusSnapshot(snapshot, get(), (entry) => {
set((s) => ({
runtimeStatusByEnvironmentId: new Map(s.runtimeStatusByEnvironmentId).set(
snapshot.environmentId,
entry
)
}))
}),
setRuntimeEnvironmentStatus: (environmentId, status, options) => {
const previous = get().runtimeStatusByEnvironmentId.get(environmentId)
if (previous?.snapshot && !status.snapshot) {
return
}
const previousVerifiedStatus = previous?.snapshot?.status ?? previous?.status
const pairedDeviceId = status.status?.pairedDeviceId?.trim()
// A new runtime id under a known previous one is a restart, not a first connect: the guests are
// still ours to host, but only a fresh attach hands them back to the replacement runtime.
const runtimeRestarted = Boolean(
status.status !== null &&
previous?.status != null &&
previous.status.runtimeId !== status.status.runtimeId
previousVerifiedStatus != null &&
previousVerifiedStatus.runtimeId !== status.status.runtimeId
)
// Why: a non-null status proves the runtime just answered, so drop any stale
// "offline" compat failure before this online transition fires the
@@ -177,7 +190,8 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
// where the runtime id moved starts a runtime session.
const runtimeSessionStarted =
status.status !== null &&
(previous?.status == null || previous.status.runtimeId !== status.status.runtimeId)
(previousVerifiedStatus == null ||
previousVerifiedStatus.runtimeId !== status.status.runtimeId)
// Why narrower than the session start: a first publication has no prior connection to
// differ from, so it is not a reconnect. Advancing the generation there retires reads
// already issued against this very connection — a startup worktree scan that had
@@ -227,17 +241,6 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
...(environmentsChanged ? { runtimeEnvironments } : {})
}
})
runtimeStatusRecheck.reconcileRuntimeStatusRecheck({
environmentId,
status: status.status,
connectionGeneration:
runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration(environmentId),
environmentExists: () =>
get().runtimeEnvironments.some((environment) => environment.id === environmentId),
getConnectionGeneration: () =>
runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration(environmentId),
publish: (entry) => get().setRuntimeEnvironmentStatus(environmentId, entry)
})
if (runtimeRestarted) {
void ensureBrowserClientHostForRestartedRuntime(get(), environmentId)
}
@@ -250,18 +253,7 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
}
},
publishRuntimeEnvironmentDiagnostics:
runtimeStatusDiagnosticsPublish.createRuntimeEnvironmentDiagnosticsSlicePublisher({
getCurrent: (environmentId) => get().runtimeStatusByEnvironmentId.get(environmentId),
setState: (updater) =>
set((s) => runtimeStatusDiagnosticsPublish.updateRuntimeStatusStore(s, updater)),
getStore: get,
getConnectionGeneration:
runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration
}),
clearRuntimeEnvironmentStatus: (environmentId) => {
runtimeStatusRecheck.cancelRuntimeStatusRecheck(environmentId)
dismissRuntimeDisconnectedToast(environmentId)
set((s) => {
runtimeStatusConnectionGeneration.advanceRuntimeEnvironmentConnectionGeneration(environmentId)
@@ -278,7 +270,6 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
const keep = new Set(environmentIds)
for (const id of get().runtimeStatusByEnvironmentId.keys()) {
if (!keep.has(id)) {
runtimeStatusRecheck.cancelRuntimeStatusRecheck(id)
dismissRuntimeDisconnectedToast(id)
}
}
@@ -295,10 +286,10 @@ export const createRuntimeStatusSlice: StateCreator<AppState, [], [], RuntimeSta
})
},
refreshRuntimeEnvironmentStatus: (environmentId, timeoutMs = 10_000, options) =>
refreshRuntimeEnvironmentStatus: (environmentId, timeoutMs = 10_000) =>
refreshRuntimeEnvironmentStatus(environmentId, timeoutMs, (entry) => {
if (entry.status === null && options?.publishUnreachable === false) {
// Unverifiable, not exited: leave the cached verdict for the caller's retry to settle.
if (entry.snapshot) {
get().applyRuntimeHostStatusSnapshot(entry.snapshot)
return
}
// Why: setRuntimeEnvironmentStatus drops any stale compat failure on a non-null
@@ -16,6 +16,9 @@ import { translateHostAccessLinkError } from '@/lib/remote-pairing-copy'
import { callEnvironmentEnvelope } from './web-runtime-calls'
import {
closeActiveRuntimeClients,
subscribeWebRuntimeStatus,
readWebRuntimeStatusSnapshots,
observeWebRuntimeStatus,
disconnectActiveRuntimeEnvironment,
getClientForEnvironment,
manuallyDisconnectedEnvironmentIds,
@@ -29,6 +32,8 @@ export function createRuntimeEnvironmentsApi(): NonNullable<
Partial<PreloadApi>['runtimeEnvironments']
> {
return {
onStatusChanged: subscribeWebRuntimeStatus,
getStatusSnapshots: async () => readWebRuntimeStatusSnapshots(),
list: async () => {
const environment = requireActiveEnvironmentOrNull()
return environment ? [redactStoredWebRuntimeEnvironment(environment)] : []
@@ -146,6 +151,12 @@ export function createRuntimeEnvironmentsApi(): NonNullable<
manuallyDisconnectedEnvironmentIds.clear()
closeActiveRuntimeClients()
webRuntimeState.activeEnvironment = nextEnvironment
getClientForEnvironment(nextEnvironment).statusOwner?.acceptVerified({
id: 'status.get',
ok: true,
result: runtimeStatus,
_meta: { runtimeId: runtimeStatus.runtimeId }
})
return {
ok: true,
environment: redactStoredWebRuntimeEnvironment(nextEnvironment),
@@ -173,6 +184,7 @@ export function createRuntimeEnvironmentsApi(): NonNullable<
connect: ({ selector, timeoutMs }) => {
const environment = resolveEnvironment(selector)
manuallyDisconnectedEnvironmentIds.delete(environment.id)
closeActiveRuntimeClients()
return callEnvironmentEnvelope<RuntimeStatus>(
environment.id,
'status.get',
@@ -180,8 +192,10 @@ export function createRuntimeEnvironmentsApi(): NonNullable<
timeoutMs
)
},
getStatus: ({ selector, timeoutMs }) =>
callEnvironmentEnvelope<RuntimeStatus>(selector, 'status.get', undefined, timeoutMs),
getStatus: ({ selector, timeoutMs, observeOnly }) =>
observeOnly
? observeWebRuntimeStatus(selector, timeoutMs)
: callEnvironmentEnvelope<RuntimeStatus>(selector, 'status.get', undefined, timeoutMs),
retryControlConnection: () => Promise.resolve(),
prepareBrowserClientHostPlacement: async () => ({ kind: 'server' }),
call: ({ selector, method, params, timeoutMs }) =>
@@ -1,3 +1,7 @@
import type {
RuntimeHostStatusSnapshot,
RuntimeHostStatusResponse
} from '../../../../shared/runtime-host-status'
import type { WorktreeVisibilityDefaults } from '../../../../shared/global-settings-types'
import { RuntimeRpcCallQueuePool } from '../../../../shared/runtime-rpc-call-queue'
import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope'
@@ -30,6 +34,43 @@ export const webRuntimeState: {
cachedDetectedWorktrees: null
}
const statusListeners = new Set<(snapshot: RuntimeHostStatusSnapshot) => void>()
export function subscribeWebRuntimeStatus(
callback: (snapshot: RuntimeHostStatusSnapshot) => void
): () => void {
statusListeners.add(callback)
return () => {
statusListeners.delete(callback)
}
}
export function readWebRuntimeStatusSnapshots(): RuntimeHostStatusSnapshot[] {
const snapshot = webRuntimeState.activeClient?.statusOwner?.read()
return snapshot ? [snapshot] : []
}
export async function observeWebRuntimeStatus(
selector: string,
timeoutMs?: number
): Promise<RuntimeHostStatusResponse> {
const environment = resolveEnvironment(selector)
if (manuallyDisconnectedEnvironmentIds.has(environment.id)) {
return manuallyDisconnectedResponse(environment)
}
const existing = webRuntimeState.activeClient?.statusOwner
if (existing) {
return existing.refresh({ timeoutMs, observeOnly: true })
}
const transient = new WebRuntimeClient(getPreferredWebPairingOffer(environment), {
reconnect: false
})
try {
return (await transient.call('status.get', undefined, {
timeoutMs
})) as RuntimeHostStatusResponse
} finally {
transient.close()
}
}
export const manuallyDisconnectedEnvironmentIds = new Set<string>()
export const runtimeCallQueuePool = new RuntimeRpcCallQueuePool()
@@ -50,7 +91,18 @@ export function getClientForEnvironment(
webRuntimeState.activeClientEnvironmentId !== environment.id
) {
webRuntimeState.activeClient?.close()
webRuntimeState.activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment))
webRuntimeState.activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment), {
status: {
environmentId: environment.id,
pairingRevision: environment.pairingRevision ?? environment.createdAt,
publish: (snapshot) => {
for (const listener of statusListeners) {
listener(snapshot)
}
},
verified: (response) => updateEnvironmentFromResponse(environment, response)
}
})
webRuntimeState.activeClientEnvironmentId = environment.id
}
return webRuntimeState.activeClient
@@ -6,8 +6,13 @@ it('keeps the paired-web client public export surface exact', () => {
expectTypeOf<WebClient.SubscribeOptions>().toEqualTypeOf<WebClient.SubscribeOptions>()
expectTypeOf<WebClient.WebRuntimeSubscriptionHandle>().toEqualTypeOf<WebClient.WebRuntimeSubscriptionHandle>()
expectTypeOf<ConstructorParameters<typeof WebClient.WebRuntimeClient>>().toEqualTypeOf<
[pairing: WebPairingOffer]
[
pairing: WebPairingOffer,
options?: ConstructorParameters<typeof WebClient.WebRuntimeClient>[1]
]
>()
expectTypeOf<keyof WebClient.WebRuntimeClient>().toEqualTypeOf<
'call' | 'close' | 'subscribe' | 'statusOwner'
>()
expectTypeOf<keyof WebClient.WebRuntimeClient>().toEqualTypeOf<'call' | 'close' | 'subscribe'>()
expect(Object.keys(WebClient)).toEqual(['WebRuntimeClient'])
})
@@ -70,7 +70,7 @@ describe('WebRuntimeClient timeout budget', () => {
await vi.advanceTimersByTimeAsync(60_000)
expect(settled).toBe(false)
expect(waitForConnected).toHaveBeenCalledWith(25)
expect(waitForConnected).toHaveBeenCalledWith(25, undefined)
resolveConnection()
await Promise.resolve()
+65 -6
View File
@@ -1,3 +1,8 @@
import { RuntimeHostStatusOwner } from '../../../shared/runtime-host-status-owner'
import type {
RuntimeHostStatusSnapshot,
RuntimeHostStatusResponse
} from '../../../shared/runtime-host-status'
import type { RuntimeRpcResponse } from '../../../shared/runtime-rpc-envelope'
import { WebRuntimeConnectionTransport } from './web-runtime-connection-transport'
import { subscribeWebRuntimeFileWatch } from './web-runtime-file-watch-subscription'
@@ -24,11 +29,62 @@ export class WebRuntimeClient {
private readonly fileWatchTeardownRetries = new Map<string, Set<() => Promise<void>>>()
private readonly childClients = new Set<WebRuntimeClient>()
constructor(private readonly pairing: WebPairingOffer) {
this.transport = new WebRuntimeConnectionTransport(pairing, {
now: () => this.now(),
isDocumentVisible: () => this.isDocumentVisible()
})
readonly statusOwner?: RuntimeHostStatusOwner
constructor(
private readonly pairing: WebPairingOffer,
options: {
reconnect?: boolean
status?: {
environmentId: string
pairingRevision: number
publish: (snapshot: RuntimeHostStatusSnapshot) => void
verified: (response: RuntimeHostStatusResponse) => void
}
} = {}
) {
this.transport = new WebRuntimeConnectionTransport(
pairing,
{
now: () => this.now(),
isDocumentVisible: () => this.isDocumentVisible()
},
{
reconnect: options.reconnect,
onStateChanged: (state) => {
if (state === 'auth-failed') {
this.statusOwner?.authenticationRejected()
}
this.statusOwner?.connectionChanged(
state === 'connected'
? 'ready'
: state === 'disconnected' || state === 'auth-failed'
? 'disconnected'
: 'connecting'
)
}
}
)
if (options.status) {
const status = options.status
this.statusOwner = new RuntimeHostStatusOwner({
...status,
persistent: true,
request: (signal) =>
this.transport.call('status.get', undefined, {
timeoutMs: 15_000,
signal
}) as Promise<RuntimeHostStatusResponse>,
verified: (response) => {
status.verified(response)
return true
}
})
this.statusOwner.connectionChanged(
this.transport.state === 'connected' ? 'ready' : 'connecting'
)
this.statusOwner.activate()
}
}
call(
@@ -36,7 +92,9 @@ export class WebRuntimeClient {
params?: unknown,
options?: { timeoutMs?: number }
): Promise<RuntimeRpcResponse<unknown>> {
return this.transport.call(method, params, options)
return method === 'status.get' && this.statusOwner
? this.statusOwner.refresh(options)
: this.transport.call(method, params, options)
}
async subscribe(
@@ -94,6 +152,7 @@ export class WebRuntimeClient {
}
close(options: { notifySubscriptions?: boolean } = {}): void {
this.statusOwner?.dispose()
const shouldNotifySubscriptions = options.notifySubscriptions ?? true
for (const child of Array.from(this.childClients)) {
child.close({ notifySubscriptions: shouldNotifySubscriptions })
@@ -43,7 +43,11 @@ export class WebRuntimeConnectionTransport {
constructor(
private readonly pairing: WebPairingOffer,
clock: { now: () => number; isDocumentVisible: () => boolean }
clock: { now: () => number; isDocumentVisible: () => boolean },
private readonly lifecycle: {
onStateChanged?: (state: WebRuntimeConnectionState) => void
reconnect?: boolean
} = {}
) {
this.serverPublicKey = publicKeyFromBase64(pairing.publicKeyB64)
this.connectionWaiters = new WebRuntimeConnectionWaiters({
@@ -60,7 +64,7 @@ export class WebRuntimeConnectionTransport {
this.requestRegistry = new WebRuntimeRequestRegistry({
deviceToken: pairing.deviceToken,
nextId: () => this.nextId(),
waitForConnected: (timeoutMs) => this.connectionWaiters.wait(timeoutMs),
waitForConnected: (timeoutMs, signal) => this.connectionWaiters.wait(timeoutMs, signal),
sendEncrypted: (message) => this.sendEncrypted(message)
})
this.heartbeat = new WebRuntimeConnectionHeartbeat({
@@ -82,7 +86,7 @@ export class WebRuntimeConnectionTransport {
async call(
method: string,
params?: unknown,
options?: { timeoutMs?: number }
options?: { timeoutMs?: number; signal?: AbortSignal }
): Promise<RuntimeRpcResponse<unknown>> {
return this.requestRegistry.call(method, params, options)
}
@@ -153,6 +157,7 @@ export class WebRuntimeConnectionTransport {
} else if (next === 'auth-failed') {
this.connectionWaiters.rejectAll(createWebRuntimeUnauthorizedError())
}
this.lifecycle.onStateChanged?.(next)
}
private openConnection(): void {
@@ -232,7 +237,7 @@ export class WebRuntimeConnectionTransport {
}
private scheduleReconnect(): void {
if (this.reconnectTimer || this.intentionallyClosed) {
if (this.reconnectTimer || this.intentionallyClosed || this.lifecycle.reconnect === false) {
return
}
const delay = withReconnectJitter(
@@ -13,7 +13,10 @@ export class WebRuntimeConnectionWaiters {
constructor(private readonly options: WebRuntimeConnectionWaiterOptions) {}
wait(timeoutMs = 30_000): Promise<void> {
wait(timeoutMs = 30_000, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) {
return Promise.reject(signal.reason)
}
if (this.options.getState() === 'connected') {
return Promise.resolve()
}
@@ -24,11 +27,20 @@ export class WebRuntimeConnectionWaiters {
return Promise.reject(new Error('Remote Orca runtime connection closed.'))
}
return new Promise((resolve, reject) => {
const timeout = window.setTimeout(() => {
const index = this.waiters.findIndex((waiter) => waiter.resolve === resolve)
const cleanup = (): void => {
window.clearTimeout(timeout)
signal?.removeEventListener('abort', abort)
const index = this.waiters.indexOf(waiter)
if (index !== -1) {
this.waiters.splice(index, 1)
}
}
const abort = (): void => {
cleanup()
reject(signal?.reason)
}
const timeout = window.setTimeout(() => {
cleanup()
reject(
new Error(
withRemoteRuntimeTailscaleHint(
@@ -38,16 +50,18 @@ export class WebRuntimeConnectionWaiters {
)
)
}, timeoutMs)
this.waiters.push({
const waiter = {
resolve: () => {
window.clearTimeout(timeout)
cleanup()
resolve()
},
reject: (error) => {
window.clearTimeout(timeout)
reject: (error: Error) => {
cleanup()
reject(error)
}
})
}
this.waiters.push(waiter)
signal?.addEventListener('abort', abort, { once: true })
})
}
@@ -6,7 +6,7 @@ const REQUEST_TIMEOUT_MS = 30_000
type WebRuntimeRequestRegistryOptions = {
deviceToken: string
nextId: () => string
waitForConnected: (timeoutMs?: number) => Promise<void>
waitForConnected: (timeoutMs?: number, signal?: AbortSignal) => Promise<void>
sendEncrypted: (message: unknown) => boolean
}
@@ -18,17 +18,41 @@ export class WebRuntimeRequestRegistry {
async call(
method: string,
params?: unknown,
callOptions?: { timeoutMs?: number }
callOptions?: { timeoutMs?: number; signal?: AbortSignal }
): Promise<RuntimeRpcResponse<unknown>> {
await this.options.waitForConnected(callOptions?.timeoutMs)
const signal = callOptions?.signal
await this.options.waitForConnected(callOptions?.timeoutMs, signal)
signal?.throwIfAborted()
return new Promise((resolve, reject) => {
const id = this.options.nextId()
const timeoutMs = callOptions?.timeoutMs ?? REQUEST_TIMEOUT_MS
const timeout = window.setTimeout(() => {
this.pending.delete(id)
cleanup()
reject(new Error(`Request timed out: ${method}`))
}, timeoutMs)
this.pending.set(id, { method, resolve, reject, timeout })
const cleanup = (): void => {
signal?.removeEventListener('abort', abort)
}
const abort = (): void => {
this.pending.delete(id)
window.clearTimeout(timeout)
cleanup()
reject(signal?.reason)
}
signal?.addEventListener('abort', abort, { once: true })
this.pending.set(id, {
method,
resolve: (value) => {
cleanup()
resolve(value)
},
reject: (error) => {
cleanup()
reject(error)
},
timeout
})
if (
!this.options.sendEncrypted({
id,
@@ -39,6 +63,7 @@ export class WebRuntimeRequestRegistry {
) {
this.pending.delete(id)
window.clearTimeout(timeout)
cleanup()
reject(new Error('Remote Orca runtime is not connected.'))
}
})
@@ -0,0 +1,49 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import WebSocket from 'ws'
import {
createSharedControlTestServer,
closeSharedControlTestServers
} from '../../../shared/remote-runtime-shared-control-test-server'
import { WebRuntimeClient } from './web-runtime-client'
const clients: WebRuntimeClient[] = []
beforeEach(() => {
vi.stubGlobal('WebSocket', WebSocket)
vi.stubGlobal('window', {
setTimeout,
clearTimeout,
setInterval,
clearInterval,
atob: (value: string) => Buffer.from(value, 'base64').toString('binary'),
btoa: (value: string) => Buffer.from(value, 'binary').toString('base64')
})
})
afterEach(async () => {
clients.splice(0).forEach((client) => client.close())
await closeSharedControlTestServers()
vi.unstubAllGlobals()
})
it('primary browser status follows the authenticated socket and closing it retires the owner', async () => {
let runtimeId = 'before'
const server = await createSharedControlTestServer({
resultForRequest: () => ({ runtimeId, capabilities: [] })
})
const publish = vi.fn()
const client = new WebRuntimeClient(server.pairing, {
status: { environmentId: 'browser', pairingRevision: 1, publish, verified: vi.fn() }
})
clients.push(client)
await expect
.poll(() => client.statusOwner?.read().verification, { timeout: 3_000 })
.toBe('verified')
expect(client.statusOwner?.read().status?.runtimeId).toBe('before')
runtimeId = 'after'
server.closeClients()
await expect
.poll(() => client.statusOwner?.read().status?.runtimeId, { timeout: 3_000 })
.toBe('after')
expect(client.statusOwner?.read().transport).toBe('ready')
client.close()
expect(publish.mock.lastCall?.[0]).toMatchObject({ retired: true, verification: 'blocked' })
})
+28 -4
View File
@@ -320,17 +320,15 @@ describe('execution host registry', () => {
])
})
it('includes runtime hosts from repo ownership but marks them disconnected without live status', () => {
it('keeps runtime hosts checking before their first status result', () => {
const hosts = buildExecutionHostRegistry({
repos: [{ connectionId: null, executionHostId: 'runtime:env-2' }],
settings: { activeRuntimeEnvironmentId: null }
})
// No live status means no evidence the Orca server is reachable, so it must
// read 'disconnected' rather than defaulting to 'available'/"Connected".
expect(hosts).toMatchObject([
{ id: 'local', health: 'local' },
{ id: 'runtime:env-2', kind: 'runtime', label: 'env-2', health: 'disconnected' }
{ id: 'runtime:env-2', kind: 'runtime', label: 'env-2', health: 'connecting' }
])
})
@@ -373,3 +371,29 @@ describe('execution host registry', () => {
])
})
})
it('keeps an initial unknown-transport verification connecting', () => {
const hosts = buildExecutionHostRegistry({
repos: [],
settings: null,
runtimeEnvironments: [{ id: 'host', name: 'Host' }],
runtimeStatusByEnvironmentId: new Map([
[
'host',
{
status: null,
snapshot: {
environmentId: 'host',
pairingRevision: 1,
sequence: 1,
checkedAt: 0,
status: null,
verification: 'checking',
transport: 'unknown'
}
}
]
])
})
expect(hosts.find((host) => host.id === 'runtime:host')?.health).toBe('connecting')
})
+24 -7
View File
@@ -1,3 +1,4 @@
import type { RuntimeHostStatusSnapshot } from './runtime-host-status'
import {
LOCAL_EXECUTION_HOST_ID,
getLocalExecutionHostLabel,
@@ -49,6 +50,7 @@ type RuntimeEnvironmentSummary = {
}
type RuntimeHostStatus = {
snapshot?: RuntimeHostStatusSnapshot
status?: RuntimeStatus | null
remoteControl?: RuntimeStatus['remoteControl'] | null
appVersion?: string | null
@@ -158,9 +160,24 @@ function addRuntimeHost(
const hostId = toRuntimeExecutionHostId(environmentId)
const runtimeStatus = statusByEnvironmentId?.get(environmentId)
const status = runtimeStatus?.status
const compatibility = runtimeCompatibility(status)
const snapshot = runtimeStatus?.snapshot
const metadata = status ?? snapshot?.status
const compatibility = runtimeCompatibility(metadata)
const remoteControl = runtimeStatus?.remoteControl ?? status?.remoteControl
const controlHealth = runtimeControlHealth(remoteControl)
const controlHealth = snapshot?.retired
? 'disconnected'
: snapshot?.verification === 'blocked'
? 'blocked'
: !runtimeStatus ||
snapshot?.verification === 'checking' ||
snapshot?.transport === 'disconnected' ||
snapshot?.transport === 'connecting'
? 'connecting'
: snapshot?.transport === 'ready'
? compatibility?.kind === 'blocked'
? 'blocked'
: 'available'
: runtimeControlHealth(remoteControl)
setHost(hosts, {
id: hostId,
kind: 'runtime',
@@ -168,12 +185,12 @@ function addRuntimeHost(
detail: 'Orca server',
health: controlHealth ?? runtimeHealth(status, compatibility, remoteControl),
compatibility: compatibility ?? undefined,
capabilities: status?.capabilities,
appVersion: runtimeStatus?.appVersion ?? status?.appVersion ?? null,
protocolVersion: status?.runtimeProtocolVersion ?? status?.protocolVersion ?? null,
capabilities: metadata?.capabilities,
appVersion: runtimeStatus?.appVersion ?? metadata?.appVersion ?? null,
protocolVersion: metadata?.runtimeProtocolVersion ?? metadata?.protocolVersion ?? null,
minCompatibleClientVersion:
status?.minCompatibleRuntimeClientVersion ?? status?.minCompatibleMobileVersion ?? null,
platform: status?.hostPlatform ?? null,
metadata?.minCompatibleRuntimeClientVersion ?? metadata?.minCompatibleMobileVersion ?? null,
platform: metadata?.hostPlatform ?? null,
remoteControlState: remoteControl ?? null,
...(source ? { source } : {})
})
@@ -20,6 +20,7 @@ export type SharedControlTestServer = {
}
type ServerOptions = {
resultForRequest?: (method: string) => unknown
delaySubscriptionReady?: boolean
sendKeepaliveBeforeResponse?: boolean
keepaliveDelayMs?: number
@@ -174,7 +175,7 @@ function handleRequest(
const streaming = isStreamingMethod(request.method)
const result = streaming
? { type: 'ready', subscriptionId: `${request.method}:subscription` }
: { method: request.method }
: (options.resultForRequest?.(request.method) ?? { method: request.method })
const sendResponse = (): void => {
if (options.sendUnknownResponseBeforeResponse) {
sendEncrypted(ws, sharedKey, {
@@ -0,0 +1,207 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { RuntimeHostStatusOwner } from './runtime-host-status-owner'
import { runtimeHostStatusFailure, type RuntimeHostStatusResponse } from './runtime-host-status'
import type { RuntimeStatus } from './runtime-types'
const owners: RuntimeHostStatusOwner[] = []
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
owners.splice(0).forEach((owner) => owner.dispose())
vi.useRealTimers()
})
function success(runtimeId = 'host-1'): RuntimeHostStatusResponse & { ok: true } {
return {
id: 'status',
ok: true,
result: { runtimeId, capabilities: [] } as unknown as RuntimeStatus,
_meta: { runtimeId }
}
}
function deferred() {
let resolve!: (response: RuntimeHostStatusResponse) => void
const promise = new Promise<RuntimeHostStatusResponse>((done) => {
resolve = done
})
return { promise, resolve }
}
function createOwner(persistent = false) {
const request = vi
.fn<(signal: AbortSignal) => Promise<RuntimeHostStatusResponse>>()
.mockResolvedValue(success())
const publish = vi.fn()
const verified = vi.fn((_response: RuntimeHostStatusResponse, _active: boolean) => persistent)
const owner = new RuntimeHostStatusOwner({
environmentId: 'env-a',
pairingRevision: 1,
persistent,
request,
publish,
verified
})
owners.push(owner)
return { owner, request, publish, verified }
}
it('shares one verification between viewers with independent deadlines', async () => {
const { owner, request } = createOwner()
const pending = deferred()
request.mockReturnValue(pending.promise)
const impatient = owner.refresh({ timeoutMs: 100 })
const patient = owner.refresh({ timeoutMs: 1_000 })
await vi.advanceTimersByTimeAsync(100)
expect((await impatient).ok).toBe(false)
expect(request).toHaveBeenCalledOnce()
expect(request.mock.calls[0][0].aborted).toBe(false)
pending.resolve(success())
expect((await patient).ok).toBe(true)
})
it('uses ready transitions, not diagnostic updates or a healthy polling timer', async () => {
const { owner, request } = createOwner(true)
await owner.refresh()
owner.connectionChanged('ready')
await vi.advanceTimersByTimeAsync(0)
owner.connectionChanged('ready')
await vi.advanceTimersByTimeAsync(300_000)
expect(request).toHaveBeenCalledTimes(2)
owner.connectionChanged('disconnected')
await vi.advanceTimersByTimeAsync(300_000)
expect(request).toHaveBeenCalledTimes(2)
owner.connectionChanged('ready')
await vi.advanceTimersByTimeAsync(0)
expect(request).toHaveBeenCalledTimes(3)
})
it('retries a failed status operation while retaining healthy transport and last good metadata', async () => {
const { owner, request } = createOwner(true)
owner.connectionChanged('ready')
await owner.refresh()
request.mockResolvedValueOnce(runtimeHostStatusFailure('runtime_unavailable', 'status timed out'))
await owner.refresh()
expect(owner.read()).toMatchObject({
transport: 'ready',
verification: 'unavailable',
status: { runtimeId: 'host-1' }
})
await vi.advanceTimersByTimeAsync(3_000)
expect(owner.read().verification).toBe('verified')
expect(request).toHaveBeenCalledTimes(3)
})
it('retires a lost-socket request before explicit fallback and rejects its late result', async () => {
const { owner, request } = createOwner(true)
owner.connectionChanged('ready')
const old = deferred()
request.mockReturnValueOnce(old.promise)
const waiting = owner.refresh()
owner.connectionChanged('disconnected')
expect(request.mock.calls[0][0].aborted).toBe(true)
request.mockResolvedValueOnce(success('fallback-host'))
expect((await owner.refresh()).ok).toBe(true)
expect((await waiting).ok).toBe(true)
old.resolve(success('obsolete-host'))
await vi.advanceTimersByTimeAsync(0)
expect(owner.read().status?.runtimeId).toBe('fallback-host')
owner.connectionChanged('ready')
await vi.advanceTimersByTimeAsync(0)
expect(request).toHaveBeenCalledTimes(3)
})
it('a reconnect transfers waiting readers to a fresh verification', async () => {
const { owner, request } = createOwner(true)
owner.connectionChanged('ready')
const old = deferred()
request.mockReturnValueOnce(old.promise)
const waiting = owner.refresh()
owner.connectionChanged('disconnected')
owner.connectionChanged('ready')
expect((await waiting).ok).toBe(true)
old.resolve(success('old'))
await vi.advanceTimersByTimeAsync(0)
expect(owner.read().status?.runtimeId).toBe('host-1')
})
it('disconnect settles readers and prevents late results and retry resurrection', async () => {
const { owner, request, publish } = createOwner()
const old = deferred()
request.mockReturnValue(old.promise)
const waiting = owner.refresh()
owner.dispose()
expect((await waiting).ok).toBe(false)
const sequence = owner.read().sequence
old.resolve(success())
owner.connectionChanged('ready')
await vi.advanceTimersByTimeAsync(300_000)
expect(owner.read()).toMatchObject({ retired: true, sequence })
expect(publish.mock.lastCall?.[0].retired).toBe(true)
expect(request).toHaveBeenCalledOnce()
})
it('passive reads create neither standing retries nor connection intent', async () => {
const { owner, request, verified } = createOwner()
request.mockResolvedValueOnce(runtimeHostStatusFailure('runtime_unavailable', 'offline'))
await owner.refresh({ observeOnly: true })
await vi.advanceTimersByTimeAsync(300_000)
expect(request).toHaveBeenCalledOnce()
await owner.refresh({ observeOnly: true })
expect(verified.mock.lastCall?.[1]).toBe(false)
})
it('authentication rejection blocks automatic verification until explicit reconnect', async () => {
const { owner, request } = createOwner(true)
owner.connectionChanged('ready')
request.mockResolvedValueOnce(runtimeHostStatusFailure('unauthorized', 're-pair'))
await owner.refresh()
owner.connectionChanged('disconnected')
owner.connectionChanged('ready')
await vi.advanceTimersByTimeAsync(300_000)
expect(request).toHaveBeenCalledOnce()
expect((await owner.refresh({ reconnect: true })).ok).toBe(true)
})
it('blocks a rejected reconnect even without an outstanding status request', async () => {
const { owner, request } = createOwner(true)
owner.connectionChanged('ready')
await owner.refresh()
owner.connectionChanged('disconnected')
owner.authenticationRejected()
expect(owner.read()).toMatchObject({ verification: 'blocked', status: { runtimeId: 'host-1' } })
owner.connectionChanged('ready')
await vi.advanceTimersByTimeAsync(300_000)
expect(request).toHaveBeenCalledOnce()
})
it('cancelling one reader leaves the shared request available to other readers', async () => {
const { owner, request } = createOwner()
const pending = deferred()
request.mockReturnValue(pending.promise)
const controller = new AbortController()
const cancelled = owner.refresh({ signal: controller.signal })
const remaining = owner.refresh()
const rejection = expect(cancelled).rejects.toThrow('cancelled')
controller.abort(new Error('cancelled'))
await rejection
expect(request.mock.calls[0][0].aborted).toBe(false)
pending.resolve(success())
expect((await remaining).ok).toBe(true)
})
it.each(['unknown', 'ready'] as const)(
'distinguishes the caller deadline with %s transport',
async (transport) => {
const { owner, request } = createOwner()
owner.connectionChanged(transport)
request.mockReturnValue(deferred().promise)
const response = owner.refresh({ timeoutMs: 100 })
await vi.advanceTimersByTimeAsync(100)
expect(await response).toMatchObject({
ok: false,
error: {
message:
transport === 'ready'
? 'Status request timed out.'
: 'Timed out waiting for the remote Orca runtime.'
}
})
}
)
+274
View File
@@ -0,0 +1,274 @@
import {
isRuntimeHostStatusBlocked,
runtimeHostStatusError,
runtimeHostStatusFailure,
type RuntimeHostStatusResponse,
type RuntimeHostStatusSnapshot
} from './runtime-host-status'
const RETRY_DELAYS_MS = [3_000, 6_000, 12_000, 30_000, 60_000]
const REQUEST_TIMEOUT_MS = 15_000
let publicationSequence = 0
type Waiter = {
resolve: (response: RuntimeHostStatusResponse) => void
cleanup: () => void
}
type StatusOwnerOptions = {
environmentId: string
pairingRevision: number
persistent?: boolean
request: (signal: AbortSignal) => Promise<RuntimeHostStatusResponse>
verified: (response: Extract<RuntimeHostStatusResponse, { ok: true }>, active: boolean) => boolean
publish: (snapshot: RuntimeHostStatusSnapshot) => void
}
/** One verification and one retry slot, shared by all readers of this connection. */
export class RuntimeHostStatusOwner {
private active = false
private disposed = false
private persistent: boolean
private attempt = 0
private retry: ReturnType<typeof setTimeout> | null = null
private request: AbortController | null = null
private readonly waiters = new Set<Waiter>()
private response: RuntimeHostStatusResponse = runtimeHostStatusFailure(
'runtime_unavailable',
'Status has not been checked.'
)
private snapshot: RuntimeHostStatusSnapshot
constructor(private readonly options: StatusOwnerOptions) {
this.persistent = options.persistent ?? false
this.snapshot = {
environmentId: options.environmentId,
pairingRevision: options.pairingRevision,
sequence: ++publicationSequence,
checkedAt: 0,
status: null,
verification: 'checking',
transport: 'unknown'
}
}
read(): RuntimeHostStatusSnapshot {
return this.snapshot
}
activate(): void {
if (this.active || this.disposed) {
return
}
this.active = true
this.startRequest()
}
acceptVerified(response: Extract<RuntimeHostStatusResponse, { ok: true }>): void {
if (this.disposed) {
return
}
this.active = true
this.retireRequest()
this.clearRetry()
this.complete(response)
}
refresh(
options: { timeoutMs?: number; observeOnly?: true; reconnect?: true; signal?: AbortSignal } = {}
): Promise<RuntimeHostStatusResponse> {
if (options.signal?.aborted) {
return Promise.reject(options.signal.reason)
}
if (this.disposed) {
return Promise.resolve(this.response)
}
if (!options.observeOnly) {
this.active = true
}
if (options.reconnect) {
this.attempt = 0
this.update({ verification: 'checking' })
}
if (this.snapshot.verification === 'blocked') {
return Promise.resolve(this.response)
}
const result = new Promise<RuntimeHostStatusResponse>((resolve, reject) => {
const release = (): void => {
waiter.cleanup()
this.waiters.delete(waiter)
if (!this.active && this.waiters.size === 0) {
this.retireRequest()
}
}
const abort = (): void => {
release()
reject(options.signal?.reason)
}
const timer = setTimeout(() => {
release()
resolve(
runtimeHostStatusFailure(
'runtime_unavailable',
this.snapshot.transport === 'ready'
? 'Status request timed out.'
: 'Timed out waiting for the remote Orca runtime.'
)
)
}, options.timeoutMs ?? REQUEST_TIMEOUT_MS)
const waiter: Waiter = {
resolve,
cleanup: () => {
clearTimeout(timer)
options.signal?.removeEventListener('abort', abort)
}
}
this.waiters.add(waiter)
options.signal?.addEventListener('abort', abort, { once: true })
})
this.startRequest()
return result
}
connectionChanged(
transport: RuntimeHostStatusSnapshot['transport'],
remoteControl?: RuntimeHostStatusSnapshot['remoteControl']
): void {
if (this.disposed) {
return
}
const previous = this.snapshot.transport
this.update({ transport, ...(remoteControl !== undefined ? { remoteControl } : {}) })
if (transport === previous) {
return
}
if (previous === 'ready') {
this.retireRequest()
this.clearRetry()
if (this.snapshot.verification !== 'blocked') {
this.update({ verification: 'unavailable' })
}
}
if (transport === 'ready' && this.snapshot.verification !== 'blocked') {
// A pre-reconnect answer cannot verify the new socket's runtime.
this.retireRequest()
if (this.active || this.waiters.size > 0) {
this.startRequest()
}
}
}
authenticationRejected(): void {
if (this.disposed) {
return
}
this.retireRequest()
this.clearRetry()
this.complete(runtimeHostStatusFailure('unauthorized', 'Pair this client again.'))
}
dispose(): void {
if (this.disposed) {
return
}
this.disposed = true
this.active = false
this.retireRequest()
this.clearRetry()
this.response = runtimeHostStatusFailure(
'runtime_manually_disconnected',
'Runtime environment was disconnected or replaced.'
)
this.update({ retired: true, transport: 'disconnected', verification: 'blocked' })
this.settleWaiters()
}
private startRequest(): void {
if (this.disposed || this.request || this.snapshot.verification === 'blocked') {
return
}
this.clearRetry()
const controller = new AbortController()
this.request = controller
if (this.snapshot.verification !== 'verified') {
this.update({ verification: 'checking' })
}
void this.verify(controller)
}
private async verify(controller: AbortController): Promise<void> {
let response: RuntimeHostStatusResponse
try {
response = await this.options.request(controller.signal)
} catch (error) {
response = runtimeHostStatusError(error)
if (error instanceof TypeError || error instanceof SyntaxError) {
console.error('Runtime status verification failed:', error)
response = runtimeHostStatusFailure('invalid_runtime_response', error.message)
}
}
if (this.request !== controller || this.disposed) {
return
}
this.request = null
this.complete(response)
}
private complete(response: RuntimeHostStatusResponse): void {
this.response = response
if (response.ok) {
this.attempt = 0
this.update({ status: response.result, checkedAt: Date.now(), verification: 'verified' })
this.persistent = this.options.verified(response, this.active)
} else {
this.update({
checkedAt: Date.now(),
verification: isRuntimeHostStatusBlocked(response) ? 'blocked' : 'unavailable'
})
this.scheduleRetry()
}
this.settleWaiters()
}
private scheduleRetry(): void {
if (
!this.active ||
this.disposed ||
this.snapshot.verification === 'blocked' ||
(this.persistent && this.snapshot.transport !== 'ready')
) {
return
}
const delay = RETRY_DELAYS_MS[Math.min(this.attempt++, RETRY_DELAYS_MS.length - 1)]
this.retry = setTimeout(() => {
this.retry = null
this.startRequest()
}, delay)
}
private settleWaiters(): void {
for (const waiter of this.waiters) {
waiter.cleanup()
waiter.resolve(this.response)
}
this.waiters.clear()
}
private retireRequest(): void {
const request = this.request
this.request = null
request?.abort()
}
private clearRetry(): void {
if (this.retry) {
clearTimeout(this.retry)
}
this.retry = null
}
private update(patch: Partial<RuntimeHostStatusSnapshot>): void {
this.snapshot = { ...this.snapshot, ...patch, sequence: ++publicationSequence }
this.options.publish(this.snapshot)
}
}
+44
View File
@@ -0,0 +1,44 @@
import type { RemoteRuntimeSharedConnectionDiagnostics } from './remote-runtime-shared-control-types'
import type { RuntimeRpcFailure, RuntimeRpcResponse } from './runtime-rpc-envelope'
import type { RuntimeStatus } from './runtime-types'
export const RUNTIME_HOST_STATUS_CHANNEL = 'runtimeEnvironments:statusChanged'
/** Local client state; never exchanged with the paired host. */
export type RuntimeHostStatusSnapshot = {
environmentId: string
pairingRevision: number
sequence: number
checkedAt: number
status: RuntimeStatus | null
verification: 'checking' | 'verified' | 'unavailable' | 'blocked'
transport: 'unknown' | 'connecting' | 'ready' | 'disconnected'
remoteControl?: RemoteRuntimeSharedConnectionDiagnostics | null
retired?: true
}
export type RuntimeHostStatusResponse = RuntimeRpcResponse<RuntimeStatus>
export function runtimeHostStatusFailure(code: string, message: string): RuntimeRpcFailure {
return { id: 'status.get', ok: false, error: { code, message } }
}
export function runtimeHostStatusError(error: unknown): RuntimeRpcFailure {
const code =
error instanceof Error && 'code' in error && typeof error.code === 'string'
? error.code
: 'runtime_unavailable'
return runtimeHostStatusFailure(code, error instanceof Error ? error.message : String(error))
}
export function isRuntimeHostStatusBlocked(response: RuntimeRpcFailure): boolean {
return [
'unauthorized',
'forbidden',
'invalid_argument',
'invalid_runtime_response',
'protocol_version_mismatch',
'method_not_found',
'unsupported_method'
].includes(response.error.code)
}
@@ -0,0 +1,221 @@
import { createConnection, createServer, type Socket, type AddressInfo } from 'node:net'
import type { Page } from '@stablyai/playwright-test'
import { decodePairingOffer, encodePairingOffer } from '../../src/shared/pairing'
import { expect, test } from './helpers/orca-app'
import {
createRuntimeDesktopPairingOffer,
launchPairedElectronClient,
launchPairedWebClient,
type RuntimeDesktopPairingOffer
} from './helpers/paired-electron-client'
import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host'
async function interruptibleHost(offer: RuntimeDesktopPairingOffer) {
const pairing = decodePairingOffer(offer.pairingUrl)
const endpoint = new URL(pairing.endpoint)
const sockets = new Set<Socket>()
let online = true
const server = createServer((client) => {
if (!online) {
client.destroy()
return
}
const host = createConnection({ host: endpoint.hostname, port: Number(endpoint.port) })
for (const socket of [client, host]) {
sockets.add(socket)
socket.on('error', () => {
client.destroy()
host.destroy()
})
socket.on('close', () => {
sockets.delete(socket)
client.destroy()
host.destroy()
})
}
client.pipe(host).pipe(client)
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const address = server.address() as AddressInfo
const pairingUrl = encodePairingOffer({ ...pairing, endpoint: `ws://127.0.0.1:${address.port}` })
let webClientUrl: string | undefined
if (offer.webClientUrl) {
const url = new URL(offer.webClientUrl)
url.search = ''
url.hash = new URLSearchParams({ pairing: pairingUrl }).toString()
webClientUrl = url.href
}
return {
offer: { pairingUrl, webClientUrl },
setOnline(value: boolean) {
online = value
if (!online) {
sockets.forEach((socket) => socket.destroy())
}
},
async close() {
sockets.forEach((socket) => socket.destroy())
await new Promise<void>((resolve) => server.close(() => resolve()))
}
}
}
async function statusEvidence(page: Page, environmentId?: string) {
return page.evaluate((id) => {
const entries = window.__store?.getState().runtimeStatusByEnvironmentId
const entry = id ? entries?.get(id) : entries?.values().next().value
return entry?.snapshot
? {
verification: entry.snapshot.verification,
transport: entry.snapshot.transport,
runtimeId: entry.status?.runtimeId,
sequence: entry.snapshot.sequence
}
: null
}, environmentId)
}
async function expectWorkspaceHostAppearance(
page: Page,
disconnected: boolean,
hostLabel?: string
) {
const cards = page.locator('[data-worktree-card-surface="true"]')
const card = (
hostLabel ? cards.filter({ has: page.getByText(hostLabel, { exact: true }) }) : cards
).first()
await expect(card).toBeVisible()
await expect(card).toHaveCSS('opacity', disconnected ? '0.6' : '1')
const icon = card.locator(disconnected ? 'svg.lucide-server-off' : 'svg.lucide-server').first()
await expect(icon).toBeVisible()
await expect(
card.locator(disconnected ? 'svg.lucide-server' : 'svg.lucide-server-off')
).toHaveCount(0)
await expect(icon).toHaveClass(disconnected ? /text-destructive/ : /text-muted-foreground/)
await icon.hover()
await expect(
page.getByRole('tooltip', { name: disconnected ? /disconnected/i : /Project on/ })
).toBeVisible()
await page.mouse.move(900, 600)
}
for (const topology of ['desktop', 'headless'] as const) {
test(`connection-owned status recovers with a ${topology} host and independent viewers`, async ({
electronApp,
orcaPage: page,
testRepoPath
}, testInfo) => {
test.setTimeout(180_000)
let headless: Awaited<ReturnType<typeof launchHeadlessPairedRuntimeHost>> | null = null
let proxy: Awaited<ReturnType<typeof interruptibleHost>> | undefined
let client: Awaited<ReturnType<typeof launchPairedElectronClient>> | undefined
let browser: Awaited<ReturnType<typeof launchPairedWebClient>> | undefined
try {
headless =
topology === 'headless'
? await launchHeadlessPairedRuntimeHost({ pinnedServePort: true })
: null
const offer = headless?.offer ?? (await createRuntimeDesktopPairingOffer(page))
await (headless
? headless.client.call('repo.add', { path: testRepoPath })
: page.evaluate(async (path) => {
await window.api.repos.add({ path })
await window.__store?.getState().fetchRepos()
}, testRepoPath))
proxy = await interruptibleHost(offer)
client = await launchPairedElectronClient(offer, testInfo, 'Direct host')
proxy.setOnline(false)
const offlineId = await client.page.evaluate(async (pairingCode) => {
const { environment } = await window.api.runtimeEnvironments.addFromPairingCode({
name: 'Recovering host',
pairingCode
})
const store = window.__store!.getState()
store.setRuntimeEnvironments(await window.api.runtimeEnvironments.list())
await store.refreshRuntimeEnvironmentStatus(environment.id, 1_000)
return environment.id
}, proxy.offer.pairingUrl)
await expect
.poll(() => statusEvidence(client!.page, offlineId))
.toMatchObject({ verification: 'unavailable' })
expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({
verification: 'verified'
})
proxy.setOnline(true)
await expect
.poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
const initial = await statusEvidence(client!.page, offlineId)
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expect(client.page.getByText('Recovering host', { exact: true }).first()).toBeVisible()
await client.page.screenshot({ path: testInfo.outputPath(`${topology}-recovered.png`) })
browser = await launchPairedWebClient(electronApp, proxy.offer)
await expect
.poll(() => statusEvidence(browser!.page), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
await expectWorkspaceHostAppearance(browser.page, false)
proxy.setOnline(false)
await expect
.poll(() => statusEvidence(client!.page, offlineId))
.toMatchObject({ transport: 'disconnected' })
await expect
.poll(() => statusEvidence(browser!.page), { timeout: 30_000 })
.toMatchObject({ transport: 'disconnected' })
expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({
verification: 'verified',
transport: 'ready'
})
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expectWorkspaceHostAppearance(client.page, false, 'Direct host')
await expectWorkspaceHostAppearance(browser.page, false)
await client.page.screenshot({
path: testInfo.outputPath(`${topology}-sidebar-reconnecting.png`)
})
await browser.page.screenshot({
path: testInfo.outputPath(`${topology}-browser-reconnecting.png`)
})
proxy.setOnline(true)
await expect
.poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
await expect
.poll(() => statusEvidence(browser!.page), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
expect((await statusEvidence(client!.page, offlineId))!.sequence).toBeGreaterThan(
initial!.sequence
)
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expectWorkspaceHostAppearance(browser.page, false)
await browser.page.screenshot({
path: testInfo.outputPath(`${topology}-browser-recovered.png`)
})
await client.page.evaluate(async (selector) => {
await window.api.runtimeEnvironments.disconnect({ selector })
}, offlineId)
await expect
.poll(() => statusEvidence(client!.page, offlineId))
.toMatchObject({ verification: 'blocked', transport: 'disconnected' })
await expectWorkspaceHostAppearance(client.page, true, 'Recovering host')
await expectWorkspaceHostAppearance(client.page, false, 'Direct host')
await client.page.screenshot({
path: testInfo.outputPath(`${topology}-sidebar-disconnected.png`)
})
await client.page.evaluate(async (selector) => {
await window.api.runtimeEnvironments.connect({ selector })
}, offlineId)
await expect
.poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expectWorkspaceHostAppearance(browser.page, false)
await client.page.screenshot({
path: testInfo.outputPath(`${topology}-sidebar-restored.png`)
})
} finally {
await browser?.dispose()
await client?.dispose()
await proxy?.close()
await headless?.dispose()
}
})
}