From 3b82d8de6422ed415e80dd2fdb75d73bcae610ea Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Fri, 11 Sep 2026 03:19:01 -0400 Subject: [PATCH] fix(runtime): let connections own host status recovery (#20003) * fix(runtime): let connections own host status recovery Verify runtime status after authenticated connection recovery and publish ordered snapshots to desktop and browser viewers. Consolidate failed-status retries in the connection owner and remove renderer retry/diagnostics merging. Adapt sidebar host-state derivation and regression coverage from Omar Shahine's original fix in https://github.com/stablyai/orca/pull/19163. Co-authored-by: Omar Shahine <10343873+omarshahine@users.noreply.github.com> * fix(runtime): show blocked hosts honestly and remove obsolete status options * fix(runtime): preserve timeout guidance and update IPC test fixtures * fix(runtime): preserve status evidence and address review gaps * test(sidebar): assert workspace host icons dimming and recovery tooltips * fix(palette): require available hosts before adding implicit badges * fix: retain disconnected host snapshots for new renderers --------- Co-authored-by: Omar Shahine <10343873+omarshahine@users.noreply.github.com> --- config/reliability-gates.jsonc | 75 +++++ ...me-environment-capability-evidence.test.ts | 7 +- ...runtime-environment-capability-evidence.ts | 7 - ...ntime-environment-connectivity-handlers.ts | 19 +- ...environment-federated-read-routing.test.ts | 22 +- .../runtime-environment-handler-channels.ts | 1 + ...me-environment-request-connections.test.ts | 16 +- ...runtime-environment-request-connections.ts | 68 +++++ ...time-environment-shared-control-support.ts | 118 ++------ ...time-environment-status-connection.test.ts | 65 ++++ .../ipc/runtime-environment-status-owner.ts | 89 ++++++ ...untime-environment-status-recovery.test.ts | 89 ++++++ ...untime-environment-support-routing.test.ts | 1 - .../runtime-environment-support-routing.ts | 6 +- ...t-transport-routing-tailscale-hint.test.ts | 8 +- .../runtime-environment-transport-routing.ts | 102 ++----- .../runtime-environments-call-routing.test.ts | 43 +-- ...time-environments-capability-cache.test.ts | 39 ++- .../runtime-environments-ipc-test-harness.ts | 56 ++++ .../ipc/runtime-environments-pairing.test.ts | 46 ++- ...me-environments-status-diagnostics.test.ts | 99 +++--- ...nvironments-subscription-lifecycle.test.ts | 32 +- ...-environments-subscription-routing.test.ts | 32 +- ...environments-subscription-teardown.test.ts | 32 +- src/main/ipc/runtime-environments.ts | 14 +- src/preload/api/runtime-api.ts | 3 + .../api/runtime-environments-bridge.ts | 14 + .../components/NewWorkspaceComposerCard.tsx | 12 +- .../cmd-j/palette-host-badge.test.ts | 19 +- .../components/cmd-j/palette-host-badge.ts | 2 +- .../settings/RepositoryHostSetupsSection.tsx | 11 +- .../use-runtime-environment-catalog.ts | 16 +- ...-runtime-environment-connection-actions.ts | 20 +- .../sidebar/AddRemoteHostDialog.tsx | 7 +- .../sidebar/HostSectionHeaderMenu.tsx | 12 +- .../sidebar/NoticeHostGlyph.test.tsx | 14 +- .../components/sidebar/NoticeHostGlyph.tsx | 10 +- ...WorktreeCard.ssh-reconnect-prompt.test.tsx | 23 +- .../sidebar/sidebar-host-options.test.ts | 5 +- .../sidebar/use-worktree-card-foundation.ts | 13 +- .../status-bar/SshStatusSegment.tsx | 16 +- .../src/hooks/ipc-events-test-harness.ts | 3 + .../ipc-events/app-lifetime-ipc-bridge.ts | 23 +- .../ipc-events/runtime-client-ipc-bridge.ts | 136 +-------- .../runtime-reconnect-host-status.test.ts | 281 ++---------------- .../src/hooks/useIpcEvents-lifecycle.test.ts | 6 +- .../runtime-host-connection-state.test.ts | 71 +++++ .../runtime/runtime-host-connection-state.ts | 42 +++ .../runtime-status-diagnostics-generation.ts | 52 ---- .../runtime-status-diagnostics-publish.ts | 106 ------- .../slices/runtime-status-diagnostics.test.ts | 89 ------ .../slices/runtime-status-recheck.test.ts | 238 --------------- .../store/slices/runtime-status-recheck.ts | 167 ----------- .../store/slices/runtime-status-refresh.ts | 16 + ...tatus-restored-browser-host-attach.test.ts | 32 +- .../slices/runtime-status-snapshot.test.ts | 111 +++++++ .../store/slices/runtime-status-snapshot.ts | 43 +++ .../src/store/slices/runtime-status-types.ts | 20 +- .../src/store/slices/runtime-status.test.ts | 42 +-- .../src/store/slices/runtime-status.ts | 75 ++--- .../web-runtime-environments-api.ts | 18 +- .../web/preload-api/web-runtime-session.ts | 54 +++- .../web-runtime-client-export-parity.test.ts | 9 +- .../web-runtime-client-timeout-budget.test.ts | 2 +- src/renderer/src/web/web-runtime-client.ts | 71 ++++- .../web/web-runtime-connection-transport.ts | 13 +- .../src/web/web-runtime-connection-waiters.ts | 30 +- .../src/web/web-runtime-request-registry.ts | 33 +- .../src/web/web-runtime-status-owner.test.ts | 49 +++ src/shared/execution-host-registry.test.ts | 32 +- src/shared/execution-host-registry.ts | 31 +- ...mote-runtime-shared-control-test-server.ts | 3 +- src/shared/runtime-host-status-owner.test.ts | 207 +++++++++++++ src/shared/runtime-host-status-owner.ts | 274 +++++++++++++++++ src/shared/runtime-host-status.ts | 44 +++ .../e2e/runtime-host-status-recovery.spec.ts | 221 ++++++++++++++ 76 files changed, 2315 insertions(+), 1612 deletions(-) create mode 100644 src/main/ipc/runtime-environment-status-connection.test.ts create mode 100644 src/main/ipc/runtime-environment-status-owner.ts create mode 100644 src/main/ipc/runtime-environment-status-recovery.test.ts delete mode 100644 src/renderer/src/store/slices/runtime-status-diagnostics-generation.ts delete mode 100644 src/renderer/src/store/slices/runtime-status-diagnostics-publish.ts delete mode 100644 src/renderer/src/store/slices/runtime-status-diagnostics.test.ts delete mode 100644 src/renderer/src/store/slices/runtime-status-recheck.test.ts delete mode 100644 src/renderer/src/store/slices/runtime-status-recheck.ts create mode 100644 src/renderer/src/store/slices/runtime-status-snapshot.test.ts create mode 100644 src/renderer/src/store/slices/runtime-status-snapshot.ts create mode 100644 src/renderer/src/web/web-runtime-status-owner.test.ts create mode 100644 src/shared/runtime-host-status-owner.test.ts create mode 100644 src/shared/runtime-host-status-owner.ts create mode 100644 src/shared/runtime-host-status.ts create mode 100644 tests/e2e/runtime-host-status-recovery.spec.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 0ba70ddcf1e..4e1f44bd7c1 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10,6 +10,81 @@ } }, "gates": [ + { + "id": "runtime.connection-owned-host-status", + "title": "Host status recovers with its owning connection", + "maturity": "experimental", + "protection": "partial", + "owner": "runtime", + "layer": "service-integration-and-e2e", + "surfaces": [ + "sidebar host status", + "desktop runtime connection", + "browser primary connection" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["remote-runtime"], + "coverageNotes": "Real authenticated sockets plus isolated desktop and headless hosts with desktop and browser viewers; deterministic lifecycle tests cover stale results and reader deadlines.", + "motivatingLinks": ["https://github.com/stablyai/orca/pull/19163"], + "invariant": "Failed bootstrap and authenticated reconnect converge without UI triggers; one connection owner publishes verified status, with no independent healthy status polling.", + "oracle": "Observe automatic recovery, retained runtime identity on failure, ordered publications, exact request counts, isolated viewer outages, and retirement on disconnect.", + "commands": [ + "ORCA_BACKGROUND_LAUNCH=1 pnpm test src/shared/runtime-host-status-owner.test.ts src/main/ipc/runtime-environment-status-recovery.test.ts src/main/ipc/runtime-environment-status-connection.test.ts src/renderer/src/store/slices/runtime-status-snapshot.test.ts src/renderer/src/web/web-runtime-status-owner.test.ts", + "ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" + ], + "testFiles": [ + "src/shared/runtime-host-status-owner.test.ts", + "src/main/ipc/runtime-environment-status-recovery.test.ts", + "src/main/ipc/runtime-environment-status-connection.test.ts", + "src/renderer/src/store/slices/runtime-status-snapshot.test.ts", + "src/renderer/src/web/web-runtime-status-owner.test.ts", + "tests/e2e/runtime-host-status-recovery.spec.ts" + ], + "assertionRefs": [ + { + "file": "src/main/ipc/runtime-environment-status-recovery.test.ts", + "assertions": [ + "recovers a saved host after its first status check fails, without another UI request" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-09-10", + "runner": "local", + "platform": "macos", + "command": "ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "summary": "Desktop-host and headless-host journeys passed with desktop and browser viewers.", + "durationSeconds": 31.3 + } + ], + "runtimeBudget": { + "p95Seconds": 180, + "scope": "Target excluding builds; measured p95 not established." + }, + "flakeHistory": { + "status": "soaking", + "evidence": "Local candidate runs passed; no sustained CI history yet." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "First-status-failure oracle failed on main 58ff95becb40 (one request instead of two) and passes on the candidate. E2E verifies candidate recovery, not a baseline comparison." + }, + "performanceBudget": { + "required": false, + "evidence": "Deterministic tests assert one shared request and no healthy owner polling." + }, + "promotionCriteria": ["Collect repeated CI runs without unexplained flakes."], + "knownGaps": [ + "No live Linux, Windows, SSH, or mixed-version pair validation.", + "TCP interruption exercises reconnect, not a full real host process restart.", + "The outage begins on the first saved-host check, not by relaunching a preseeded desktop profile." + ], + "demotionRule": "Keep experimental until repeated runs establish reliability; preserve request-count and lifecycle assertions." + }, { "id": "mobile-push.headless-startup-and-policy", "title": "Headless push lifecycle and mobile delivery policy", diff --git a/src/main/ipc/runtime-environment-capability-evidence.test.ts b/src/main/ipc/runtime-environment-capability-evidence.test.ts index 4671326ef02..8c9bce5f272 100644 --- a/src/main/ipc/runtime-environment-capability-evidence.test.ts +++ b/src/main/ipc/runtime-environment-capability-evidence.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it } from 'vitest' import type { PairingOffer } from '../../shared/pairing' import { advanceRuntimeEnvironmentCapabilityIncarnation, @@ -17,7 +17,6 @@ describe('runtime environment capability evidence', () => { it('accepts evidence by dispatch order instead of completion order', () => { const older = captureRuntimeEnvironmentCapabilityEvidence('env', pairing()) const newer = captureRuntimeEnvironmentCapabilityEvidence('env', pairing()) - const pause = vi.fn() expect( applyRuntimeEnvironmentCapabilityVerdict({ @@ -30,12 +29,10 @@ describe('runtime environment capability evidence', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence: older, verdict: 'absent', - runtimeId: 'runtime-old', - onAbsent: pause + runtimeId: 'runtime-old' }) ).toBe(false) - expect(pause).not.toHaveBeenCalled() expect(isRuntimeEnvironmentCapabilityPaused('env')).toBe(false) }) diff --git a/src/main/ipc/runtime-environment-capability-evidence.ts b/src/main/ipc/runtime-environment-capability-evidence.ts index d32bda584e9..197ec71f4f8 100644 --- a/src/main/ipc/runtime-environment-capability-evidence.ts +++ b/src/main/ipc/runtime-environment-capability-evidence.ts @@ -68,8 +68,6 @@ export function applyRuntimeEnvironmentCapabilityVerdict(args: { evidence: RuntimeEnvironmentCapabilityEvidence verdict: RuntimeEnvironmentCapabilityVerdict runtimeId: string - onCapable?: () => void - onAbsent?: () => void }): boolean { const state = stateFor(args.evidence.environmentId) if ( @@ -83,11 +81,6 @@ export function applyRuntimeEnvironmentCapabilityVerdict(args: { verdict: args.verdict, runtimeId: args.runtimeId } - if (args.verdict === 'capable') { - args.onCapable?.() - } else { - args.onAbsent?.() - } return true } diff --git a/src/main/ipc/runtime-environment-connectivity-handlers.ts b/src/main/ipc/runtime-environment-connectivity-handlers.ts index 1e267d8675a..bfbc63847c4 100644 --- a/src/main/ipc/runtime-environment-connectivity-handlers.ts +++ b/src/main/ipc/runtime-environment-connectivity-handlers.ts @@ -20,6 +20,8 @@ import { verifyAndAddRuntimeEnvironmentFromPairingCode } from './runtime-environ import { clearRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence' import { closeRemoteRuntimeRequestConnection, + getRuntimeEnvironmentStatusOwner, + getRuntimeEnvironmentStatusSnapshots, retryRemoteRuntimeSharedControlConnectionNow } from './runtime-environment-request-connections' import { @@ -29,7 +31,6 @@ import { } from './runtime-environment-manual-disconnect' import { callRuntimeEnvironment, - clearSharedControlSupport, getRuntimeEnvironmentStatus } from './runtime-environment-transport-routing' @@ -60,6 +61,9 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ getUserDataPath, invalidateTransport }: ConnectivityHandlerOptions): void { + ipcMain.handle('runtimeEnvironments:getStatusSnapshots', () => + getRuntimeEnvironmentStatusSnapshots() + ) ipcMain.handle('runtimeEnvironments:list', () => listEnvironments(getUserDataPath()).map(redactRuntimeEnvironment) ) @@ -80,6 +84,12 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ const result = await verifyAndAddRuntimeEnvironmentFromPairingCode(getUserDataPath(), args) if (result.ok) { clearRuntimeEnvironmentManualDisconnect(result.environment.id) + getRuntimeEnvironmentStatusOwner(getUserDataPath(), result.environment.id).acceptVerified({ + id: 'status.get', + ok: true, + result: result.runtimeStatus, + _meta: { runtimeId: result.runtimeStatus.runtimeId } + }) } return result } @@ -121,6 +131,8 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ markRuntimeEnvironmentManuallyDisconnected(environment.id) invalidateTransport(environment.id) closeLegacySelectorTransport(args.selector, environment.id) + // Retain disconnected evidence for renderers that missed the teardown event. + getRuntimeEnvironmentStatusOwner(getUserDataPath(), environment.id) return { disconnected: redactRuntimeEnvironment(environment) } } ) @@ -132,7 +144,9 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ ): Promise> => { const environment = resolveEnvironment(getUserDataPath(), args.selector) clearRuntimeEnvironmentManualDisconnect(environment.id) - return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs) + return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs, { + reconnect: true + }) } ) ipcMain.handle( @@ -156,7 +170,6 @@ function closeLegacySelectorTransport(selector: string, environmentId: string): return } closeRemoteRuntimeRequestConnection(selector) - clearSharedControlSupport(selector) } function registerPassiveStatusHandler(getUserDataPath: () => string): void { diff --git a/src/main/ipc/runtime-environment-federated-read-routing.test.ts b/src/main/ipc/runtime-environment-federated-read-routing.test.ts index c58a404fd39..51c77510a01 100644 --- a/src/main/ipc/runtime-environment-federated-read-routing.test.ts +++ b/src/main/ipc/runtime-environment-federated-read-routing.test.ts @@ -1,3 +1,5 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -20,14 +22,17 @@ vi.mock('../../shared/remote-runtime-client', () => ({ sendRemoteRuntimeRequest: sendRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: vi.fn(), - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - reconnectRemoteRuntimeSharedControlConnection: vi.fn(), - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn() -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: vi.fn(), + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + reconnectRemoteRuntimeSharedControlConnection: vi.fn(), + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn() + }) +}) import { callRuntimeEnvironment, @@ -55,6 +60,7 @@ describe('federated read RPC transport routing', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environment-handler-channels.ts b/src/main/ipc/runtime-environment-handler-channels.ts index 0b63dea943a..23b40fe5183 100644 --- a/src/main/ipc/runtime-environment-handler-channels.ts +++ b/src/main/ipc/runtime-environment-handler-channels.ts @@ -9,6 +9,7 @@ export const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [ 'runtimeEnvironments:retryControlConnection', 'runtimeEnvironments:prepareBrowserClientHostPlacement', 'runtimeEnvironments:getStatus', + 'runtimeEnvironments:getStatusSnapshots', 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', 'runtimeEnvironments:unsubscribe' diff --git a/src/main/ipc/runtime-environment-request-connections.test.ts b/src/main/ipc/runtime-environment-request-connections.test.ts index 750d1becc6a..b02d1b06fb5 100644 --- a/src/main/ipc/runtime-environment-request-connections.test.ts +++ b/src/main/ipc/runtime-environment-request-connections.test.ts @@ -47,9 +47,9 @@ describe('runtime environment shared-control connection cache', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence: absent, verdict: 'absent', - runtimeId: 'runtime-test', - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) + runtimeId: 'runtime-test' }) + pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) expect(getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state).toBe('closed') await delay(400) expect(server.connectionCount()).toBe(1) @@ -58,12 +58,10 @@ describe('runtime environment shared-control connection cache', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence: capable, verdict: 'capable', - runtimeId: 'runtime-test', - onCapable: () => { - ensureRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID, server.pairing) - reconnectRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID) - } + runtimeId: 'runtime-test' }) + ensureRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID, server.pairing) + reconnectRemoteRuntimeSharedControlConnection(ENVIRONMENT_ID) await waitFor(() => server.connectionCount() === 2) }) @@ -119,9 +117,9 @@ describe('runtime environment shared-control connection cache', () => { applyRuntimeEnvironmentCapabilityVerdict({ evidence, verdict: 'absent', - runtimeId: 'runtime-test', - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) + runtimeId: 'runtime-test' }) + pauseRemoteRuntimeSharedControlRetry(ENVIRONMENT_ID) expect(getRemoteRuntimeSharedControlDiagnostics(ENVIRONMENT_ID)?.state).toBe('reconnecting') await waitFor(() => server.connectionCount() === 2) diff --git a/src/main/ipc/runtime-environment-request-connections.ts b/src/main/ipc/runtime-environment-request-connections.ts index c1f855697e5..6ba9f273c1e 100644 --- a/src/main/ipc/runtime-environment-request-connections.ts +++ b/src/main/ipc/runtime-environment-request-connections.ts @@ -1,4 +1,9 @@ import type { PairingOffer } from '../../shared/pairing' +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { getPreferredPairingOffer } from '../../shared/runtime-environments' +import type { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner' +import type { RuntimeStatus } from '../../shared/runtime-types' +import { createRuntimeEnvironmentStatusOwner } from './runtime-environment-status-owner' import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/protocol-version' import type { RuntimeOrchestrationEnvelope, @@ -30,6 +35,56 @@ type CachedSharedControlConnection = { const requestConnections = new Map() const sharedControlConnections = new Map() +const statusOwners = new Map() + +export function getRuntimeEnvironmentStatusOwner( + userDataPath: string, + selector: string +): RuntimeHostStatusOwner { + const environment = resolveEnvironment(userDataPath, selector) + const pairing = getPreferredPairingOffer(environment) + const key = `${userDataPath}\0${environment.pairingRevision ?? environment.createdAt}\0${getPairingKey(pairing)}` + let cached = statusOwners.get(environment.id) + if (!cached || cached.key !== key || cached.owner.read().retired) { + if (cached) { + closeRemoteRuntimeRequestConnection(environment.id) + } + const owner = createRuntimeEnvironmentStatusOwner(userDataPath, environment, { + isReady: () => getRemoteRuntimeSharedControlDiagnostics(environment.id)?.state === 'ready', + request: (signal) => + sendRemoteRuntimeSharedControlRequest( + environment.id, + pairing, + 'status.get', + undefined, + 15_000, + undefined, + signal + ), + establish: () => { + ensureRemoteRuntimeSharedControlConnection(environment.id, pairing) + reconnectRemoteRuntimeSharedControlConnection(environment.id) + }, + pause: () => pauseRemoteRuntimeSharedControlRetry(environment.id) + }) + cached = { key, owner } + statusOwners.set(environment.id, cached) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + owner.dispose() + } + } + return cached.owner +} + +export function resetRuntimeEnvironmentStatusOwners(): void { + for (const id of statusOwners.keys()) { + closeRemoteRuntimeRequestConnection(id) + } +} + +export function getRuntimeEnvironmentStatusSnapshots() { + return [...statusOwners.values()].map(({ owner }) => owner.read()) +} export function sendRemoteRuntimeConnectionRequest( environmentId: string, @@ -56,6 +111,9 @@ export function sendRemoteRuntimeConnectionRequest( } export function closeRemoteRuntimeRequestConnection(environmentId: string): void { + const status = statusOwners.get(environmentId) + statusOwners.delete(environmentId) + status?.owner.dispose() const cached = requestConnections.get(environmentId) requestConnections.delete(environmentId) cached?.connection.close() @@ -166,6 +224,16 @@ function getSharedControlConnection( transportGeneration, diagnostics }) + statusOwners + .get(environmentId) + ?.owner.connectionChanged( + diagnostics.state === 'ready' + ? 'ready' + : diagnostics.state === 'closed' || diagnostics.state === 'reconnecting' + ? 'disconnected' + : 'connecting', + diagnostics + ) } }) } diff --git a/src/main/ipc/runtime-environment-shared-control-support.ts b/src/main/ipc/runtime-environment-shared-control-support.ts index 29513e2970a..0de6a35e0f6 100644 --- a/src/main/ipc/runtime-environment-shared-control-support.ts +++ b/src/main/ipc/runtime-environment-shared-control-support.ts @@ -1,39 +1,23 @@ -import { - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY -} from '../../shared/protocol-version' -import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' -import { markEnvironmentUsed } from '../../shared/runtime-environment-store' import type { getPreferredPairingOffer, KnownRuntimeEnvironment } from '../../shared/runtime-environments' -import type { RuntimeStatus } from '../../shared/runtime-types' +import { RemoteRuntimeClientError } from '../../shared/remote-runtime-client-error' import { - applyRuntimeEnvironmentCapabilityVerdict, - captureRuntimeEnvironmentCapabilityEvidence, getAcceptedRuntimeEnvironmentCapabilityOutcome, - isRuntimeEnvironmentCapabilityOutcomeCurrent, - runtimeEnvironmentCapabilityOutcome, resetRuntimeEnvironmentCapabilityEvidence, type RuntimeEnvironmentCapabilityOutcome } from './runtime-environment-capability-evidence' -import { pauseRemoteRuntimeSharedControlRetry } from './runtime-environment-request-connections' - -const sharedControlSupport = new Map< - string, - { cacheKey: string; check: Promise } ->() +import { + getRuntimeEnvironmentStatusOwner, + resetRuntimeEnvironmentStatusOwners +} from './runtime-environment-request-connections' export function resetSharedControlSupport(): void { - sharedControlSupport.clear() + resetRuntimeEnvironmentStatusOwners() resetRuntimeEnvironmentCapabilityEvidence() } -export function clearSharedControlSupport(environmentId: string): void { - sharedControlSupport.delete(environmentId) -} - export async function supportsSharedControl( userDataPath: string, environment: KnownRuntimeEnvironment, @@ -48,85 +32,17 @@ export async function supportsSharedControl( if (accepted) { return accepted } - const cacheKey = getSharedControlSupportCacheKey(environment, pairing) - const cached = sharedControlSupport.get(environment.id) - if (cached?.cacheKey === cacheKey) { - const outcome = await cached.check - if (isRuntimeEnvironmentCapabilityOutcomeCurrent(outcome)) { - return outcome - } - if (sharedControlSupport.get(environment.id)?.check === cached.check) { - sharedControlSupport.delete(environment.id) - } - return { kind: 'stale_incarnation' } + const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({ + timeoutMs + }) + if (!response.ok) { + throw new RemoteRuntimeClientError(response.error.code, response.error.message) } - let resolvedCacheKey = cacheKey - const evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) - const check = (async () => { - const response = await sendRemoteRuntimeRequest( + return ( + getAcceptedRuntimeEnvironmentCapabilityOutcome( + environment.id, pairing, - 'status.get', - undefined, - timeoutMs, - undefined, - undefined, - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES - ) - if (response.ok === true) { - const verdict = response.result.capabilities?.includes( - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY - ) - ? 'capable' - : 'absent' - const acceptedEvidence = applyRuntimeEnvironmentCapabilityVerdict({ - evidence, - verdict, - runtimeId: response._meta.runtimeId, - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(environment.id) - }) - if (!acceptedEvidence) { - return { kind: 'stale_incarnation' } as const - } - markEnvironmentUsed(userDataPath, environment.id, { runtimeId: response._meta.runtimeId }) - resolvedCacheKey = getSharedControlSupportCacheKey( - environment, - pairing, - response._meta.runtimeId - ) - return runtimeEnvironmentCapabilityOutcome(evidence, verdict, response._meta.runtimeId) - } - return runtimeEnvironmentCapabilityOutcome( - evidence, - 'absent', - environment.runtimeId ?? 'unknown-runtime' - ) - })() - // Why: support belongs to the saved pairing/runtime identity, not its mutable display name. - sharedControlSupport.set(environment.id, { cacheKey, check }) - try { - const outcome = await check - const cachedAfterCheck = sharedControlSupport.get(environment.id) - if (cachedAfterCheck?.check === check && cachedAfterCheck.cacheKey !== resolvedCacheKey) { - sharedControlSupport.set(environment.id, { cacheKey: resolvedCacheKey, check }) - } - return outcome - } catch (error) { - if (sharedControlSupport.get(environment.id)?.check === check) { - sharedControlSupport.delete(environment.id) - } - throw error - } -} - -function getSharedControlSupportCacheKey( - environment: KnownRuntimeEnvironment, - pairing: ReturnType, - runtimeId = environment.runtimeId -): string { - return [ - runtimeId ?? 'unknown-runtime', - pairing.endpoint, - pairing.deviceToken, - pairing.publicKeyB64 - ].join('\0') + response._meta.runtimeId + ) ?? { kind: 'stale_incarnation' } + ) } diff --git a/src/main/ipc/runtime-environment-status-connection.test.ts b/src/main/ipc/runtime-environment-status-connection.test.ts new file mode 100644 index 00000000000..8d4fad6f9b9 --- /dev/null +++ b/src/main/ipc/runtime-environment-status-connection.test.ts @@ -0,0 +1,65 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import { encodePairingOffer } from '../../shared/pairing' +import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' +import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' +import { + createSharedControlTestServer, + closeSharedControlTestServers +} from '../../shared/remote-runtime-shared-control-test-server' +import { getRuntimeEnvironmentStatus } from './runtime-environment-transport-routing' +import { + getRuntimeEnvironmentStatusOwner, + resetRuntimeEnvironmentStatusOwners +} from './runtime-environment-request-connections' + +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) +const profiles: string[] = [] +afterEach(async () => { + resetRuntimeEnvironmentStatusOwners() + await closeSharedControlTestServers() + profiles.splice(0).forEach((profile) => rmSync(profile, { recursive: true, force: true })) +}) + +it('publishes real same-socket verification after every authenticated reconnect', async () => { + let runtimeId = 'host-before' + const server = await createSharedControlTestServer({ + resultForRequest: () => ({ + runtimeId, + capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] + }) + }) + const profile = mkdtempSync(join(tmpdir(), 'orca-status-socket-')) + profiles.push(profile) + const environment = addEnvironmentFromPairingCode(profile, { + name: 'host', + pairingCode: encodePairingOffer(server.pairing) + }) + await getRuntimeEnvironmentStatus(profile, environment.id) + const owner = getRuntimeEnvironmentStatusOwner(profile, environment.id) + await vi.waitFor( + () => { + expect(owner.read()).toMatchObject({ transport: 'ready', verification: 'verified' }) + expect(server.requests).toHaveLength(2) + }, + { timeout: 3_000 } + ) + expect(server.connectionCount()).toBe(2) // Bootstrap plus persistent control. + runtimeId = 'host-after' + server.closeClients() + await vi.waitFor( + () => { + expect(owner.read().status?.runtimeId).toBe('host-after') + expect(owner.read().verification).toBe('verified') + }, + { timeout: 3_000 } + ) + expect(server.connectionCount()).toBe(3) + expect(server.requests.map((request) => request.method)).toEqual([ + 'status.get', + 'status.get', + 'status.get' + ]) +}) diff --git a/src/main/ipc/runtime-environment-status-owner.ts b/src/main/ipc/runtime-environment-status-owner.ts new file mode 100644 index 00000000000..4ac3c067f74 --- /dev/null +++ b/src/main/ipc/runtime-environment-status-owner.ts @@ -0,0 +1,89 @@ +import { BrowserWindow } from 'electron' +import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' +import { + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, + REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY +} from '../../shared/protocol-version' +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { markEnvironmentUsed } from '../../shared/runtime-environment-store' +import { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner' +import { + RUNTIME_HOST_STATUS_CHANNEL, + type RuntimeHostStatusResponse +} from '../../shared/runtime-host-status' +import { + applyRuntimeEnvironmentCapabilityVerdict, + getAcceptedRuntimeEnvironmentCapabilityOutcome, + captureRuntimeEnvironmentCapabilityEvidence +} from './runtime-environment-capability-evidence' +import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' + +export function createRuntimeEnvironmentStatusOwner( + userDataPath: string, + environment: KnownRuntimeEnvironment, + transport: { + isReady: () => boolean + request: (signal: AbortSignal) => Promise + establish: () => void + pause: () => void + } +): RuntimeHostStatusOwner { + const pairing = getPreferredPairingOffer(environment) + let evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) + return new RuntimeHostStatusOwner({ + environmentId: environment.id, + pairingRevision: environment.pairingRevision ?? environment.createdAt, + request: (signal) => { + evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) + return transport.isReady() && + getAcceptedRuntimeEnvironmentCapabilityOutcome(environment.id, pairing, null)?.kind === + 'supported' + ? transport.request(signal) + : sendRemoteRuntimeRequest( + pairing, + 'status.get', + undefined, + 15_000, + undefined, + signal, + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES + ) + }, + verified: (response, active) => { + const capable = + response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) ?? false + const accepted = applyRuntimeEnvironmentCapabilityVerdict({ + evidence, + verdict: capable ? 'capable' : 'absent', + runtimeId: response._meta.runtimeId + }) + if (accepted && active && !isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + markEnvironmentUsed(userDataPath, environment.id, { + runtimeId: response._meta.runtimeId, + pairedDeviceId: response.result.pairedDeviceId + }) + if (capable) { + transport.establish() + } else { + transport.pause() + } + } + return capable && active + }, + publish: (snapshot) => { + for (const window of BrowserWindow.getAllWindows()) { + if (window.isDestroyed()) { + continue + } + try { + window.webContents.send(RUNTIME_HOST_STATUS_CHANNEL, snapshot) + } catch { + /* A renderer can close during publication. */ + } + } + } + }) +} diff --git a/src/main/ipc/runtime-environment-status-recovery.test.ts b/src/main/ipc/runtime-environment-status-recovery.test.ts new file mode 100644 index 00000000000..82e94ea62e0 --- /dev/null +++ b/src/main/ipc/runtime-environment-status-recovery.test.ts @@ -0,0 +1,89 @@ +import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' +import { pairingCode } from './runtime-environments-ipc-test-harness' +import { + getRuntimeEnvironmentStatus, + resetSharedControlSupport +} from './runtime-environment-transport-routing' + +const { request, publish } = vi.hoisted(() => ({ request: vi.fn(), publish: vi.fn() })) +vi.mock('../../shared/remote-runtime-client', () => ({ + sendRemoteRuntimeRequest: request, + subscribeRemoteRuntimeRequest: vi.fn() +})) +vi.mock('electron', () => ({ + BrowserWindow: { + getAllWindows: () => [ + { + isDestroyed: () => false, + webContents: { send: publish } + } + ] + } +})) + +let profile: string +beforeEach(() => { + vi.useFakeTimers() + request.mockReset() + publish.mockReset() + profile = mkdtempSync(join(tmpdir(), 'orca-status-recovery-')) +}) +afterEach(() => { + resetSharedControlSupport() + vi.useRealTimers() + rmSync(profile, { recursive: true, force: true }) +}) + +it('recovers a saved host after its first status check fails, without another UI request', async () => { + const environment = addEnvironmentFromPairingCode(profile, { + name: 'offline-at-startup', + pairingCode: pairingCode() + }) + request + .mockRejectedValueOnce( + Object.assign(new Error('host offline'), { code: 'runtime_unavailable' }) + ) + .mockResolvedValue({ + id: 'status', + ok: true, + result: { runtimeId: 'host-1', graphStatus: 'ready', capabilities: [] }, + _meta: { runtimeId: 'host-1' } + }) + expect((await getRuntimeEnvironmentStatus(profile, environment.id)).ok).toBe(false) + await vi.advanceTimersByTimeAsync(3_000) + expect(request).toHaveBeenCalledTimes(2) + expect(publish).toHaveBeenCalledWith( + 'runtimeEnvironments:statusChanged', + expect.objectContaining({ + environmentId: environment.id, + verification: 'verified', + status: expect.objectContaining({ runtimeId: 'host-1' }) + }) + ) + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledTimes(2) +}) + +it('a passive capability check does not strand later active bootstrap recovery', async () => { + const environment = addEnvironmentFromPairingCode(profile, { + name: 'passive-first', + pairingCode: pairingCode() + }) + request + .mockResolvedValueOnce({ + id: 'status', + ok: true, + result: { runtimeId: 'host-1', capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] }, + _meta: { runtimeId: 'host-1' } + }) + .mockRejectedValue(new Error('host offline')) + await getRuntimeEnvironmentStatus(profile, environment.id, undefined, { observeOnly: true }) + await getRuntimeEnvironmentStatus(profile, environment.id) + await vi.advanceTimersByTimeAsync(3_000) + expect(request).toHaveBeenCalledTimes(3) +}) diff --git a/src/main/ipc/runtime-environment-support-routing.test.ts b/src/main/ipc/runtime-environment-support-routing.test.ts index feb09ee481a..8fdde15a81c 100644 --- a/src/main/ipc/runtime-environment-support-routing.test.ts +++ b/src/main/ipc/runtime-environment-support-routing.test.ts @@ -57,7 +57,6 @@ describe('runtime environment support routing', () => { ).resolves.toMatchObject({ ok: true }) expect(supportsMock).toHaveBeenCalledTimes(2) - expect(clearSupportMock).toHaveBeenCalledOnce() expect(supported).toHaveBeenCalledOnce() expect(unsupported).not.toHaveBeenCalled() }) diff --git a/src/main/ipc/runtime-environment-support-routing.ts b/src/main/ipc/runtime-environment-support-routing.ts index e2503ad4445..9566b2fc1b7 100644 --- a/src/main/ipc/runtime-environment-support-routing.ts +++ b/src/main/ipc/runtime-environment-support-routing.ts @@ -18,10 +18,7 @@ import { type RuntimeEnvironmentCapabilityOutcome } from './runtime-environment-capability-evidence' import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' -import { - clearSharedControlSupport, - supportsSharedControl -} from './runtime-environment-shared-control-support' +import { supportsSharedControl } from './runtime-environment-shared-control-support' import { sendRemoteRuntimeRequestAbortable, sendRemoteRuntimeSharedControlRequestAbortable @@ -205,7 +202,6 @@ export async function routeRuntimeEnvironmentCallBySupport(args: { } return response } - clearSharedControlSupport(environment.id) environment = resolveEnvironment(args.userDataPath, environment.id) } return runtimeEnvironmentChangedFailure(environment, args.method) diff --git a/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts b/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts index a44f5df84c6..bb2bbdac322 100644 --- a/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts +++ b/src/main/ipc/runtime-environment-transport-routing-tailscale-hint.test.ts @@ -1,20 +1,23 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { generateKeyPair, publicKeyToBase64 } from '../../shared/e2ee-crypto' import { encodePairingOffer, type PairingOffer } from '../../shared/pairing' import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' import { callRuntimeEnvironment, getRuntimeEnvironmentStatus, - subscribeRuntimeEnvironment + subscribeRuntimeEnvironment, + resetSharedControlSupport } from './runtime-environment-transport-routing' // Why: prove the wiring, not just the helper — an unreachable endpoint exercises // the real WebSocket failure → reject → Tailscale-hint join points the settings // probe (returned ok:false) and in-use calls (thrown) actually use. +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) + let userDataPath: string function seedEnvironment(name: string, endpoint: string): string { @@ -39,6 +42,7 @@ beforeEach(() => { }) afterEach(() => { + resetSharedControlSupport() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environment-transport-routing.ts b/src/main/ipc/runtime-environment-transport-routing.ts index 70f49816603..b19b0d9e376 100644 --- a/src/main/ipc/runtime-environment-transport-routing.ts +++ b/src/main/ipc/runtime-environment-transport-routing.ts @@ -1,8 +1,5 @@ import { getPreferredPairingOffer } from '../../shared/runtime-environments' -import { - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY -} from '../../shared/protocol-version' +import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/protocol-version' import { resolveEnvironment, markEnvironmentUsed } from '../../shared/runtime-environment-store' import { isOrchestrationMutation } from '../../shared/orchestration-rpc-contract' import type { @@ -11,33 +8,22 @@ import type { } from '../../shared/runtime-rpc-envelope' import type { RuntimeStatus } from '../../shared/runtime-types' import { - sendRemoteRuntimeRequest, subscribeRemoteRuntimeRequest, type RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' import { withRemoteRuntimeTailscaleHint } from '../../shared/remote-runtime-tailscale-hint' import { enqueueRuntimeCall } from './runtime-environment-call-queue' -import { - ensureRemoteRuntimeSharedControlConnection, - pauseRemoteRuntimeSharedControlRetry, - reconnectRemoteRuntimeSharedControlConnection -} from './runtime-environment-request-connections' +import { getRuntimeEnvironmentStatusOwner } from './runtime-environment-request-connections' import { sendRemoteRuntimeConnectionRequestAbortable, sendRemoteRuntimeRequestAbortable } from './runtime-environment-abortable-requests' import { attachRemoteControlDiagnostics } from './runtime-environment-status-diagnostics' -import { - applyRuntimeEnvironmentCapabilityVerdict, - captureRuntimeEnvironmentCapabilityEvidence -} from './runtime-environment-capability-evidence' + import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' import { withTailscaleHintForResponse } from './runtime-environment-tailscale-response' -import { - clearSharedControlSupport, - resetSharedControlSupport -} from './runtime-environment-shared-control-support' +import { resetSharedControlSupport } from './runtime-environment-shared-control-support' import { executeSupportRoutedCall, shouldRouteCallBySupport, @@ -47,72 +33,31 @@ import { const DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS = 15_000 -export { clearSharedControlSupport, resetSharedControlSupport } +export { resetSharedControlSupport } export async function getRuntimeEnvironmentStatus( userDataPath: string, selector: string, timeoutMs?: number, - options?: { observeOnly?: true } + options?: { observeOnly?: true; signal?: AbortSignal; reconnect?: true } ): Promise> { const environment = resolveEnvironment(userDataPath, selector) - const pairing = getPreferredPairingOffer(environment) - const evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) - let response: RuntimeRpcResponse - try { - response = await sendRemoteRuntimeRequest( - pairing, - 'status.get', - undefined, - timeoutMs ?? DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS, - undefined, - undefined, - ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES - ) - } catch (error) { - // Why: the status UI needs shared-control diagnostics most when the - // fresh status probe failed and the host is reconnecting/offline. - return attachRemoteControlDiagnostics( - withTailscaleHintForResponse( - { - id: 'status.get', - ok: false, - error: { - code: 'runtime_unavailable', - message: error instanceof Error ? error.message : String(error) - }, - _meta: { runtimeId: environment.runtimeId } - }, - pairing.endpoint - ), - environment.id - ) - } - if (response.ok === true) { - const verdict = response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) - ? 'capable' - : 'absent' - const accepted = applyRuntimeEnvironmentCapabilityVerdict({ - evidence, - verdict, - runtimeId: response._meta.runtimeId, - onCapable: () => { - if (!options?.observeOnly && !isRuntimeEnvironmentManuallyDisconnected(environment.id)) { - ensureRemoteRuntimeSharedControlConnection(environment.id, pairing) - reconnectRemoteRuntimeSharedControlConnection(environment.id) - } - }, - onAbsent: () => pauseRemoteRuntimeSharedControlRetry(environment.id) - }) - if (accepted && !options?.observeOnly) { - markEnvironmentUsed(userDataPath, environment.id, { - runtimeId: response._meta.runtimeId, - pairedDeviceId: response.result.pairedDeviceId - }) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + return { + id: 'status.get', + ok: false, + error: { + code: 'runtime_manually_disconnected', + message: 'Runtime environment is manually disconnected.' + } } } + const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({ + timeoutMs, + ...options + }) return attachRemoteControlDiagnostics( - withTailscaleHintForResponse(response, pairing.endpoint), + withTailscaleHintForResponse(response, getPreferredPairingOffer(environment).endpoint), environment.id ) } @@ -127,6 +72,15 @@ export async function callRuntimeEnvironment( envelope?: RuntimeOrchestrationEnvelope, options?: { signal?: AbortSignal } ): Promise> { + if (method === 'status.get') { + const environment = resolveEnvironment(userDataPath, selector) + const failure = runtimeEnvironmentRevisionFailure( + environment, + expectedEnvironmentPairingRevision, + method + ) + return failure ?? getRuntimeEnvironmentStatus(userDataPath, selector, timeoutMs, options) + } const environment = resolveEnvironment(userDataPath, selector) // Why: connection failures reject (they don't resolve as ok:false), so the // Tailscale hint is applied to the thrown error here — wrapping the resolved diff --git a/src/main/ipc/runtime-environments-call-routing.test.ts b/src/main/ipc/runtime-environments-call-routing.test.ts index ef92dc66826..e6f8946527d 100644 --- a/src/main/ipc/runtime-environments-call-routing.test.ts +++ b/src/main/ipc/runtime-environments-call-routing.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -44,6 +45,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -58,18 +60,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -112,6 +119,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -339,7 +347,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { undefined, 15_000, undefined, - undefined, + expect.any(AbortSignal), ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES ) expect(sendRemoteRuntimeSharedControlRequestMock).toHaveBeenCalledWith( @@ -451,7 +459,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { } ) - it('keeps uncoded call failures on the rejected IPC fallback path', async () => { + it('returns uncoded status failures through the owner response', async () => { registerRuntimeEnvironmentHandlers(store as never) sendRemoteRuntimeRequestMock.mockRejectedValue(new Error('shared down')) @@ -464,9 +472,10 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:call' ) - await expect(call(null, { selector: 'desk', method: 'status.get' })).rejects.toThrow( - 'shared down' - ) + await expect(call(null, { selector: 'desk', method: 'status.get' })).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_unavailable', message: 'shared down' } + }) }) it('does not fall back after a shared-control request fails on a supported runtime', async () => { diff --git a/src/main/ipc/runtime-environments-capability-cache.test.ts b/src/main/ipc/runtime-environments-capability-cache.test.ts index 8ac11d69dd1..32f724df981 100644 --- a/src/main/ipc/runtime-environments-capability-cache.test.ts +++ b/src/main/ipc/runtime-environments-capability-cache.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -37,6 +38,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -51,18 +53,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -105,6 +112,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -182,9 +190,10 @@ describe('registerRuntimeEnvironmentHandlers', () => { { selector: string; method: string; params?: unknown; timeoutMs?: number }, { ok: true; result: unknown } >('runtimeEnvironments:call') - await expect(call(null, { selector: 'desk', method: 'repo.list' })).rejects.toThrow( - 'probe failed' - ) + await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_unavailable', message: 'probe failed' } + }) await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({ ok: true, result: { repos: [] } diff --git a/src/main/ipc/runtime-environments-ipc-test-harness.ts b/src/main/ipc/runtime-environments-ipc-test-harness.ts index 016352793cb..e97e45ede1a 100644 --- a/src/main/ipc/runtime-environments-ipc-test-harness.ts +++ b/src/main/ipc/runtime-environments-ipc-test-harness.ts @@ -1,6 +1,62 @@ import { expect } from 'vitest' import type { Mock } from 'vitest' +import { getPreferredPairingOffer } from '../../shared/runtime-environments' import { encodePairingOffer } from '../../shared/pairing' +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { createRuntimeEnvironmentStatusOwner } from './runtime-environment-status-owner' +import type { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner' +import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' + +/** Keep IPC tests on the production owner while replacing only its transport. */ +export function withRuntimeStatusOwners>(transport: T) { + const owners = new Map() + return { + ...transport, + getRuntimeEnvironmentStatusOwner: (profile: string, selector: string) => { + const environment = resolveEnvironment(profile, selector) + let owner = owners.get(environment.id) + if (!owner || owner.read().retired) { + owner = createRuntimeEnvironmentStatusOwner(profile, environment, { + isReady: () => + transport.getRemoteRuntimeSharedControlDiagnostics?.(environment.id)?.state === 'ready', + request: (signal) => + transport.sendRemoteRuntimeSharedControlRequest( + environment.id, + undefined, + 'status.get', + undefined, + 15_000, + undefined, + signal + ), + establish: () => { + transport.ensureRemoteRuntimeSharedControlConnection?.( + environment.id, + getPreferredPairingOffer(environment) + ) + transport.reconnectRemoteRuntimeSharedControlConnection?.(environment.id) + }, + pause: () => transport.pauseRemoteRuntimeSharedControlRetry?.(environment.id) + }) + owners.set(environment.id, owner) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + owner.dispose() + } + } + return owner + }, + getRuntimeEnvironmentStatusSnapshots: () => [...owners.values()].map((owner) => owner.read()), + resetRuntimeEnvironmentStatusOwners: () => { + owners.forEach((owner) => owner.dispose()) + owners.clear() + }, + closeRemoteRuntimeRequestConnection: (...args: unknown[]) => { + owners.get(args[0] as string)?.dispose() + owners.delete(args[0] as string) + transport.closeRemoteRuntimeRequestConnection(...args) + } + } +} export function pairingCode(endpoint = 'ws://127.0.0.1:6768'): string { return encodePairingOffer({ diff --git a/src/main/ipc/runtime-environments-pairing.test.ts b/src/main/ipc/runtime-environments-pairing.test.ts index 87d6c2698ab..ce492a7a740 100644 --- a/src/main/ipc/runtime-environments-pairing.test.ts +++ b/src/main/ipc/runtime-environments-pairing.test.ts @@ -1,3 +1,5 @@ +import type { RuntimeHostStatusSnapshot } from '../../shared/runtime-host-status' +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -44,6 +46,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -58,18 +61,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: retryRemoteRuntimeSharedControlConnectionNowMock, - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: retryRemoteRuntimeSharedControlConnectionNowMock, + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -125,6 +133,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -132,6 +141,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { registerRuntimeEnvironmentHandlers(store as never) expect(handleMock.mock.calls.map((call) => call[0])).toEqual([ + 'runtimeEnvironments:getStatusSnapshots', 'runtimeEnvironments:list', 'runtimeEnvironments:addFromPairingCode', 'runtimeEnvironments:verifyAndAddFromPairingCode', @@ -166,6 +176,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:retryControlConnection', 'runtimeEnvironments:prepareBrowserClientHostPlacement', 'runtimeEnvironments:getStatus', + 'runtimeEnvironments:getStatusSnapshots', 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', 'runtimeEnvironments:unsubscribe', @@ -467,6 +478,13 @@ describe('registerRuntimeEnvironmentHandlers', () => { ok: false, error: { code: 'runtime_manually_disconnected' } }) + const getSnapshots = handler( + 'runtimeEnvironments:getStatusSnapshots' + ) + // A new renderer only has the snapshot read, not the earlier disconnect event. + expect(await getSnapshots(null, undefined)).toMatchObject([ + { environmentId: added.environment.id, retired: true, transport: 'disconnected' } + ]) const call = handler< { selector: string; method: string }, { ok: boolean; error?: { code: string } } @@ -492,6 +510,10 @@ describe('registerRuntimeEnvironmentHandlers', () => { result: { runtimeId: 'runtime-remote' } }) expect(sendRemoteRuntimeRequestMock).toHaveBeenCalledOnce() + expect(await getSnapshots(null, undefined)).toMatchObject([ + { environmentId: added.environment.id, verification: 'verified' } + ]) + expect((await getSnapshots(null, undefined))[0].retired).not.toBe(true) }) it('marks environments owned by ephemeral VM runtimes in the public list', async () => { diff --git a/src/main/ipc/runtime-environments-status-diagnostics.test.ts b/src/main/ipc/runtime-environments-status-diagnostics.test.ts index b210e7c209b..9fe3d6baf0e 100644 --- a/src/main/ipc/runtime-environments-status-diagnostics.test.ts +++ b/src/main/ipc/runtime-environments-status-diagnostics.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -44,6 +45,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -58,18 +60,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: ensureRemoteRuntimeSharedControlConnectionMock, - pauseRemoteRuntimeSharedControlRetry: pauseRemoteRuntimeSharedControlRetryMock, - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: ensureRemoteRuntimeSharedControlConnectionMock, + pauseRemoteRuntimeSharedControlRetry: pauseRemoteRuntimeSharedControlRetryMock, + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -114,6 +121,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) @@ -148,9 +156,9 @@ describe('registerRuntimeEnvironmentHandlers', () => { expect.objectContaining({ endpoint: 'ws://127.0.0.1:6768', deviceToken: 'device-token' }), 'status.get', undefined, - 50, - undefined, + 15_000, undefined, + expect.any(AbortSignal), ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES ) expect(reconnectRemoteRuntimeSharedControlConnectionMock).toHaveBeenCalledWith( @@ -319,36 +327,41 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) }) - it('returns shared-control diagnostics when saved remote runtime status throws', async () => { - registerRuntimeEnvironmentHandlers(store as never) - getRemoteRuntimeSharedControlDiagnosticsMock.mockReturnValue({ - state: 'reconnecting', - pendingRequestCount: 0, - subscriptionCount: 1, - reconnectAttempt: 2, - lastConnectedAt: 123, - lastClose: { code: 1006, reason: '' }, - lastError: 'closed' - }) - sendRemoteRuntimeRequestMock.mockRejectedValue(new Error('socket closed')) + it.each(['runtimeEnvironments:getStatus', 'runtimeEnvironments:connect'])( + 'preserves failure diagnostics and guidance on %s', + async (channel) => { + registerRuntimeEnvironmentHandlers(store as never) + getRemoteRuntimeSharedControlDiagnosticsMock.mockReturnValue({ + state: 'reconnecting', + pendingRequestCount: 0, + subscriptionCount: 1, + reconnectAttempt: 2, + lastConnectedAt: 123, + lastClose: { code: 1006, reason: '' }, + lastError: 'closed' + }) + sendRemoteRuntimeRequestMock.mockRejectedValue( + new Error('Could not connect to the remote Orca runtime.') + ) - const add = handler< - { name: string; pairingCode: string }, - { environment: { id: string; name: string } } - >('runtimeEnvironments:addFromPairingCode') - await add(null, { name: 'desk', pairingCode: pairingCode() }) + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + await add(null, { name: 'desk', pairingCode: pairingCode() }) - const getStatus = handler< - { selector: string; timeoutMs?: number }, - { ok: false; error: { message: string; data?: { remoteControl?: { state: string } } } } - >('runtimeEnvironments:getStatus') + const getStatus = handler< + { selector: string; timeoutMs?: number }, + { ok: false; error: { message: string; data?: { remoteControl?: { state: string } } } } + >(channel) - await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({ - ok: false, - error: { - message: 'socket closed', - data: { remoteControl: { state: 'reconnecting' } } - } - }) - }) + await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({ + ok: false, + error: { + message: expect.stringContaining('connect both devices to Tailscale'), + data: { remoteControl: { state: 'reconnecting' } } + } + }) + } + ) }) diff --git a/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts b/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts index 53a07442c3b..ed4dcd62182 100644 --- a/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts +++ b/src/main/ipc/runtime-environments-subscription-lifecycle.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -38,6 +39,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -52,18 +54,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { invalidateRuntimeEnvironmentTransport, @@ -109,6 +116,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environments-subscription-routing.test.ts b/src/main/ipc/runtime-environments-subscription-routing.test.ts index 494f0b9ea6b..0ef70f7ac96 100644 --- a/src/main/ipc/runtime-environments-subscription-routing.test.ts +++ b/src/main/ipc/runtime-environments-subscription-routing.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -40,6 +41,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -54,18 +56,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) import { registerRuntimeEnvironmentHandlers } from './runtime-environments' import { channelHandlerLookup, pairingCode } from './runtime-environments-ipc-test-harness' @@ -108,6 +115,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environments-subscription-teardown.test.ts b/src/main/ipc/runtime-environments-subscription-teardown.test.ts index 13a98e1057a..afb1adf457a 100644 --- a/src/main/ipc/runtime-environments-subscription-teardown.test.ts +++ b/src/main/ipc/runtime-environments-subscription-teardown.test.ts @@ -1,3 +1,4 @@ +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -38,6 +39,7 @@ const { })) vi.mock('electron', () => ({ + BrowserWindow: { getAllWindows: () => [] }, app: { getPath: getPathMock }, ipcMain: { handle: handleMock, @@ -52,18 +54,23 @@ vi.mock('../../shared/remote-runtime-client', () => ({ subscribeRemoteRuntimeRequest: subscribeRemoteRuntimeRequestMock })) -vi.mock('./runtime-environment-request-connections', () => ({ - sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, - sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, - subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, - getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, - reconnectRemoteRuntimeSharedControlConnection: reconnectRemoteRuntimeSharedControlConnectionMock, - retryRemoteRuntimeSharedControlConnectionsNow: retryRemoteRuntimeSharedControlConnectionsNowMock, - retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), - ensureRemoteRuntimeSharedControlConnection: vi.fn(), - pauseRemoteRuntimeSharedControlRetry: vi.fn(), - closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock -})) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + sendRemoteRuntimeConnectionRequest: sendRemoteRuntimeConnectionRequestMock, + sendRemoteRuntimeSharedControlRequest: sendRemoteRuntimeSharedControlRequestMock, + subscribeRemoteRuntimeSharedControlRequest: subscribeRemoteRuntimeSharedControlRequestMock, + getRemoteRuntimeSharedControlDiagnostics: getRemoteRuntimeSharedControlDiagnosticsMock, + reconnectRemoteRuntimeSharedControlConnection: + reconnectRemoteRuntimeSharedControlConnectionMock, + retryRemoteRuntimeSharedControlConnectionsNow: + retryRemoteRuntimeSharedControlConnectionsNowMock, + retryRemoteRuntimeSharedControlConnectionNow: vi.fn(), + ensureRemoteRuntimeSharedControlConnection: vi.fn(), + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: closeRemoteRuntimeRequestConnectionMock + }) +}) vi.mock('../browser/paired-runtime-browser-client-host-runtime', () => ({ retirePairedRuntimeBrowserClientHostEnvironment: retirePairedRuntimeBrowserClientHostEnvironmentMock @@ -115,6 +122,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) afterEach(() => { + resetRuntimeEnvironmentStatusOwners() rmSync(userDataPath, { recursive: true, force: true }) }) diff --git a/src/main/ipc/runtime-environments.ts b/src/main/ipc/runtime-environments.ts index 7d9a261eef9..ac7a4107bc8 100644 --- a/src/main/ipc/runtime-environments.ts +++ b/src/main/ipc/runtime-environments.ts @@ -1,6 +1,6 @@ import { app, ipcMain } from 'electron' import { randomUUID } from 'node:crypto' -import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { listEnvironments, resolveEnvironment } from '../../shared/runtime-environment-store' import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' import type { Store } from '../persistence' import { @@ -8,14 +8,16 @@ import { registerRuntimeEnvironmentConnectivityHandlers, registerRuntimeEnvironmentPassiveHandlers } from './runtime-environment-connectivity-handlers' -import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections' +import { + closeRemoteRuntimeRequestConnection, + getRuntimeEnvironmentStatusOwner +} from './runtime-environment-request-connections' import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler' import { advanceRuntimeEnvironmentTransportGeneration, getRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' import { - clearSharedControlSupport, resetSharedControlSupport, subscribeRuntimeEnvironment } from './runtime-environment-transport-routing' @@ -64,7 +66,6 @@ export function invalidateRuntimeEnvironmentTransport(environmentId: string): Pr advanceRuntimeEnvironmentCapabilityIncarnation(environmentId) advanceRuntimeEnvironmentTransportGeneration(environmentId) closeRemoteRuntimeRequestConnection(environmentId) - clearSharedControlSupport(environmentId) closeSubscriptionsForEnvironment(environmentId) return retirePairedRuntimeBrowserClientHostEnvironment( environmentId, @@ -97,6 +98,11 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { }) registerRuntimeEnvironmentRecoveryHandler() registerRuntimeEnvironmentPassiveHandlers(getUserDataPath) + for (const environment of listEnvironments(getUserDataPath())) { + if (!isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + getRuntimeEnvironmentStatusOwner(getUserDataPath(), environment.id).activate() + } + } ipcMain.handle( 'runtimeEnvironments:subscribe', async ( diff --git a/src/preload/api/runtime-api.ts b/src/preload/api/runtime-api.ts index f7f553ec2c0..7fdf229dd88 100644 --- a/src/preload/api/runtime-api.ts +++ b/src/preload/api/runtime-api.ts @@ -1,3 +1,4 @@ +import type { RuntimeHostStatusSnapshot } from '../../shared/runtime-host-status' import type { RuntimeBrowserDriverState, RuntimeRendererSyncWindowGraph, @@ -77,6 +78,8 @@ export type RuntimeApi = { ) => () => void } runtimeEnvironments: { + getStatusSnapshots: () => Promise + onStatusChanged: (callback: (snapshot: RuntimeHostStatusSnapshot) => void) => () => void list: () => Promise addFromPairingCode: (args: { name: string diff --git a/src/preload/api/runtime-environments-bridge.ts b/src/preload/api/runtime-environments-bridge.ts index ddfa498dc74..63c31df52f8 100644 --- a/src/preload/api/runtime-environments-bridge.ts +++ b/src/preload/api/runtime-environments-bridge.ts @@ -1,4 +1,8 @@ import { ipcRenderer } from 'electron' +import { + RUNTIME_HOST_STATUS_CHANNEL, + type RuntimeHostStatusSnapshot +} from '../../shared/runtime-host-status' import type { VerifyAndAddRuntimeEnvironmentResult } from '../../shared/remote-pairing-verification' import type { RuntimeStatus } from '../../shared/runtime-types' import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' @@ -12,6 +16,16 @@ import { import type { PreloadApi } from '../api-types' export const runtimeEnvironmentsApi = { + getStatusSnapshots: (): Promise => + ipcRenderer.invoke('runtimeEnvironments:getStatusSnapshots'), + onStatusChanged: (callback: (snapshot: RuntimeHostStatusSnapshot) => void): (() => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + snapshot: RuntimeHostStatusSnapshot + ): void => callback(snapshot) + ipcRenderer.on(RUNTIME_HOST_STATUS_CHANNEL, listener) + return () => ipcRenderer.removeListener(RUNTIME_HOST_STATUS_CHANNEL, listener) + }, list: (): Promise => ipcRenderer.invoke('runtimeEnvironments:list'), addFromPairingCode: (args: { diff --git a/src/renderer/src/components/NewWorkspaceComposerCard.tsx b/src/renderer/src/components/NewWorkspaceComposerCard.tsx index d17bb8ba56d..7145bed732c 100644 --- a/src/renderer/src/components/NewWorkspaceComposerCard.tsx +++ b/src/renderer/src/components/NewWorkspaceComposerCard.tsx @@ -242,17 +242,11 @@ export default function NewWorkspaceComposerCard( selector: action.environmentId, timeoutMs: 15_000 }) - const runtimeStatus = unwrapRuntimeRpcResult(response) - useAppStore.getState().setRuntimeEnvironmentStatus(action.environmentId, { - status: runtimeStatus, - checkedAt: Date.now() - }) + unwrapRuntimeRpcResult(response) + await useAppStore.getState().readRuntimeHostStatusSnapshots() } catch (error) { if (action.kind === 'runtime') { - useAppStore.getState().setRuntimeEnvironmentStatus(action.environmentId, { - status: null, - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() } toast.error( error instanceof Error diff --git a/src/renderer/src/components/cmd-j/palette-host-badge.test.ts b/src/renderer/src/components/cmd-j/palette-host-badge.test.ts index cd6310c8580..8d15ff8d034 100644 --- a/src/renderer/src/components/cmd-j/palette-host-badge.test.ts +++ b/src/renderer/src/components/cmd-j/palette-host-badge.test.ts @@ -83,8 +83,7 @@ describe('getPaletteHostBadge', () => { repos: [{ executionHostId: 'runtime:env-1' }], sshTargetLabels: new Map(), settings: { activeRuntimeEnvironmentId: 'env-2' }, - // A live status makes the runtime 'available'; without it the host reads - // 'disconnected' and the badge is suppressed (covered below). + // Only verified availability enables unfiltered host badges. runtimeStatusByEnvironmentId: new Map([ [ 'env-1', @@ -145,3 +144,19 @@ describe('getPaletteHostBadge', () => { expect(getPaletteHostBadge(null, hosts)).toBeNull() }) }) + +it.each(['connecting', 'blocked', 'disconnected', 'error'] as const)( + 'does not infer reachability from %s health, but preserves explicit filter labels', + (health) => { + const hosts = buildSidebarHostOptions({ + repos: [{ executionHostId: 'runtime:env-1' }], + sshTargetLabels: new Map(), + settings: { activeRuntimeEnvironmentId: null } + }).map((host) => (host.kind === 'runtime' ? { ...host, health } : host)) + expect(getPaletteHostBadge({ connectionId: null }, hosts)).toBeNull() + expect(getPaletteHostBadge({ executionHostId: 'runtime:env-1' }, hosts, true)).toEqual({ + hostId: 'runtime:env-1', + label: 'env-1' + }) + } +) diff --git a/src/renderer/src/components/cmd-j/palette-host-badge.ts b/src/renderer/src/components/cmd-j/palette-host-badge.ts index 8ac94adeae4..01a9e33d3ac 100644 --- a/src/renderer/src/components/cmd-j/palette-host-badge.ts +++ b/src/renderer/src/components/cmd-j/palette-host-badge.ts @@ -17,7 +17,7 @@ export type PaletteHostBadge = { // unlike the sidebar gate, which lists disconnected hosts so users can connect. function hasActiveRemoteHost(hostOptions: readonly SidebarHostOption[]): boolean { return hostOptions.some( - (host) => host.id !== LOCAL_EXECUTION_HOST_ID && host.health !== 'disconnected' + (host) => host.id !== LOCAL_EXECUTION_HOST_ID && host.health === 'available' ) } diff --git a/src/renderer/src/components/settings/RepositoryHostSetupsSection.tsx b/src/renderer/src/components/settings/RepositoryHostSetupsSection.tsx index 94854bb623d..07bf5bfb54b 100644 --- a/src/renderer/src/components/settings/RepositoryHostSetupsSection.tsx +++ b/src/renderer/src/components/settings/RepositoryHostSetupsSection.tsx @@ -28,7 +28,7 @@ import { } from '@/store/slices/runtime-environment-ssh' import { isConnectedRuntimeHostState, - runtimeHostConnectionState + runtimeHostConnectionStateForEntry } from '@/runtime/runtime-host-connection-state' type RepositoryHostSetupsSectionProps = { @@ -227,14 +227,7 @@ export function RepositoryHostSetupsSection({ ? runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId) : undefined const runtimeOwnerState = runtimeOwnerEnvironmentId - ? runtimeHostConnectionState({ - hasStatusEntry: Boolean(runtimeOwnerStatusEntry), - status: runtimeOwnerStatusEntry?.status, - remoteControl: - runtimeOwnerStatusEntry?.remoteControl ?? - runtimeOwnerStatusEntry?.status?.remoteControl ?? - null - }) + ? runtimeHostConnectionStateForEntry(runtimeOwnerStatusEntry) : null const runtimeOwnerReachable = runtimeOwnerState === null || isConnectedRuntimeHostState(runtimeOwnerState) diff --git a/src/renderer/src/components/settings/use-runtime-environment-catalog.ts b/src/renderer/src/components/settings/use-runtime-environment-catalog.ts index a4acfb2b121..5470bf5b6e4 100644 --- a/src/renderer/src/components/settings/use-runtime-environment-catalog.ts +++ b/src/renderer/src/components/settings/use-runtime-environment-catalog.ts @@ -51,10 +51,7 @@ export function useRuntimeEnvironmentCatalog(): RuntimeEnvironmentCatalog { // linger in the sidebar registry. useAppStore.getState().setRuntimeEnvironments(nextEnvironments) if (verified) { - useAppStore.getState().setRuntimeEnvironmentStatus(verified.environmentId, { - status: verified.runtimeStatus, - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() } if (mountedRef.current) { setEnvironments(visibleEnvironments) @@ -93,10 +90,7 @@ export function useRuntimeEnvironmentCatalog(): RuntimeEnvironmentCatalog { const runtimeStatus = unwrapRuntimeRpcResult(response) // Why: feed the live status into the store so sidebar host pickers // reflect manual refreshes, not just the settings pane. - useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, { - status: runtimeStatus, - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() if (!mountedRef.current) { return } @@ -114,11 +108,7 @@ export function useRuntimeEnvironmentCatalog(): RuntimeEnvironmentCatalog { // Why: record the failed probe (null status) so the sidebar can // distinguish unreachable from never-checked. const remoteControl = extractRuntimeTransportDiagnostics(error) - useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, { - status: null, - ...(remoteControl ? { remoteControl } : {}), - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() if (!mountedRef.current) { return } diff --git a/src/renderer/src/components/settings/use-runtime-environment-connection-actions.ts b/src/renderer/src/components/settings/use-runtime-environment-connection-actions.ts index 062fde45ff4..2174c3633c6 100644 --- a/src/renderer/src/components/settings/use-runtime-environment-connection-actions.ts +++ b/src/renderer/src/components/settings/use-runtime-environment-connection-actions.ts @@ -40,14 +40,7 @@ export function useRuntimeEnvironmentConnectionActions({ await window.api.runtimeEnvironments.disconnect({ selector: environment.id }) // Why: disconnect is non-destructive; keep the saved server but show the // user that this live client is no longer attached to it. - useAppStore.getState().setRuntimeEnvironmentStatus( - environment.id, - { - status: null, - checkedAt: Date.now() - }, - { suppressDisconnectToast: true } - ) + await useAppStore.getState().readRuntimeHostStatusSnapshots() if (mountedRef.current) { setDetailsByEnvironmentId((current) => ({ ...current, @@ -96,10 +89,7 @@ export function useRuntimeEnvironmentConnectionActions({ const compatibility = evaluateHostDetails(runtimeStatus) // Why: row Connect is reachability only. The Advanced selector is the // explicit default-host control and should be the only active-server path. - useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, { - status: runtimeStatus, - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() if (mountedRef.current) { setDetailsByEnvironmentId((current) => ({ ...current, @@ -143,11 +133,7 @@ export function useRuntimeEnvironmentConnectionActions({ } catch (error) { const message = error instanceof Error ? error.message : 'Failed to connect server.' const remoteControl = extractRuntimeTransportDiagnostics(error) - useAppStore.getState().setRuntimeEnvironmentStatus(environment.id, { - status: null, - ...(remoteControl ? { remoteControl } : {}), - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() if (mountedRef.current) { setDetailsByEnvironmentId((current) => ({ ...current, diff --git a/src/renderer/src/components/sidebar/AddRemoteHostDialog.tsx b/src/renderer/src/components/sidebar/AddRemoteHostDialog.tsx index 03658792d94..1c6ba1ba75a 100644 --- a/src/renderer/src/components/sidebar/AddRemoteHostDialog.tsx +++ b/src/renderer/src/components/sidebar/AddRemoteHostDialog.tsx @@ -68,7 +68,7 @@ export function AddRemoteHostDialog({ const setSshTargetsMetadata = useAppStore((s) => s.setSshTargetsMetadata) const recordSshRepoReadoptions = useAppStore((s) => s.recordSshRepoReadoptions) const setRuntimeEnvironments = useAppStore((s) => s.setRuntimeEnvironments) - const setRuntimeEnvironmentStatus = useAppStore((s) => s.setRuntimeEnvironmentStatus) + const readRuntimeHostStatusSnapshots = useAppStore((s) => s.readRuntimeHostStatusSnapshots) const recordFeatureInteraction = useAppStore((s) => s.recordFeatureInteraction) const busy = isSaving || isBulkImporting || resolvingConfigAlias !== null @@ -283,10 +283,7 @@ export function AddRemoteHostDialog({ } const environments = await window.api.runtimeEnvironments.list() setRuntimeEnvironments(environments) - setRuntimeEnvironmentStatus(result.environment.id, { - status: result.runtimeStatus, - checkedAt: Date.now() - }) + await readRuntimeHostStatusSnapshots() toast.success( translate('auto.components.sidebar.AddRemoteHostDialog.serverSaved', 'Remote server added.') ) diff --git a/src/renderer/src/components/sidebar/HostSectionHeaderMenu.tsx b/src/renderer/src/components/sidebar/HostSectionHeaderMenu.tsx index 3aead74fd81..31d2fab5b91 100644 --- a/src/renderer/src/components/sidebar/HostSectionHeaderMenu.tsx +++ b/src/renderer/src/components/sidebar/HostSectionHeaderMenu.tsx @@ -141,13 +141,10 @@ export function HostSectionHeaderMenu({ row }: { row: HostHeaderRow }): React.JS selector: parsed.environmentId, timeoutMs: 10_000 }) - const runtimeStatus = unwrapRuntimeRpcResult(response) + unwrapRuntimeRpcResult(response) // Why: feed the probe result into the shared store so the host header and // other host pickers reflect this check without a separate fetch. - useAppStore.getState().setRuntimeEnvironmentStatus(parsed.environmentId, { - status: runtimeStatus, - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() toast.success( translate( 'auto.components.sidebar.HostSectionHeaderMenu.7f1a2b3c4d', @@ -160,10 +157,7 @@ export function HostSectionHeaderMenu({ row }: { row: HostHeaderRow }): React.JS } catch (err) { // Why: record the failed probe so the host registry can drop a previously // healthy verdict instead of showing stale "compatible" state. - useAppStore.getState().setRuntimeEnvironmentStatus(parsed.environmentId, { - status: null, - checkedAt: Date.now() - }) + await useAppStore.getState().readRuntimeHostStatusSnapshots() toast.error( err instanceof Error ? err.message diff --git a/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx b/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx index 46bf7cd371f..efd47b976ec 100644 --- a/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx +++ b/src/renderer/src/components/sidebar/NoticeHostGlyph.test.tsx @@ -83,7 +83,8 @@ describe('NoticeHostGlyph', () => { ) }) - it('marks a paired runtime with no live status as disconnected', async () => { + it('marks a paired runtime a probe found unreachable as disconnected', async () => { + runtimeStatusByEnvironmentId.set('openclaw-env', { status: null }) const container = await render('runtime:openclaw-env') expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe( @@ -91,6 +92,17 @@ describe('NoticeHostGlyph', () => { ) }) + it('does not call a host disconnected before its first probe answers', async () => { + // No entry means "not asked yet", not "asked and unreachable" — collapsing the two + // painted every remote row destructive between launch and the first probe. + const container = await render('runtime:openclaw-env') + + expect(container.querySelector('[data-testid="tooltip"]')?.textContent).toBe( + 'Project on openclaw' + ) + expect(container.querySelector('svg')?.getAttribute('class')).not.toContain('text-destructive') + }) + it('gives the local host the monitor glyph the run-target rows use', async () => { const container = await render('local', 'Local Mac') diff --git a/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx b/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx index db1616203dd..7c070450e7d 100644 --- a/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx +++ b/src/renderer/src/components/sidebar/NoticeHostGlyph.tsx @@ -5,6 +5,10 @@ import { HostRowIcon } from '../host-row-icon' import { useAppStore } from '@/store' import { parseExecutionHostId, type ExecutionHostId } from '../../../../shared/execution-host' import { translate } from '@/i18n/i18n' +import { + isDisconnectedRuntimeHostState, + runtimeHostConnectionStateForEntry +} from '@/runtime/runtime-host-connection-state' type NoticeHostGlyphProps = { hostId: ExecutionHostId @@ -26,11 +30,15 @@ export default function NoticeHostGlyph({ keyboardFocusable }: NoticeHostGlyphProps): React.JSX.Element | null { const host = parseExecutionHostId(hostId) + // Why the shared derivation, not raw truthiness: an absent entry means "not probed yet", + // which is not the same verdict as a probe that came back unreachable. const isDisconnected = useAppStore((s) => { if (host?.kind !== 'runtime') { return false } - return !s.runtimeStatusByEnvironmentId.get(host.environmentId)?.status + return isDisconnectedRuntimeHostState( + runtimeHostConnectionStateForEntry(s.runtimeStatusByEnvironmentId.get(host.environmentId)) + ) }) if (!host) { diff --git a/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx b/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx index 1fe2f4b9530..55ef3263ad4 100644 --- a/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx +++ b/src/renderer/src/components/sidebar/WorktreeCard.ssh-reconnect-prompt.test.tsx @@ -218,18 +218,35 @@ describe('WorktreeCard SSH reconnect prompt', () => { expect(markup).not.toContain('Retry SSH connection') }) - it('marks a runtime-host worktree disconnected when its environment has no status', () => { + it('marks a runtime-host worktree disconnected once a probe finds it unreachable', () => { + runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }] + runtimeStatusByEnvironmentId.set('env-1', { status: null }) + const runtimeRepo: Repo = { + ...makeRepo(), + connectionId: undefined, + executionHostId: 'runtime:env-1' + } + const markup = renderToStaticMarkup( + + ) + expect(markup).toContain('Remote Mac disconnected') + }) + + // Why: "not probed yet" is not "probed and unreachable" — collapsing them painted every + // remote card destructive and dimmed between launch and the first probe answering. + it('leaves a runtime-host worktree undimmed before its first probe answers', () => { runtimeEnvironments = [{ id: 'env-1', name: 'Remote Mac' }] const runtimeRepo: Repo = { ...makeRepo(), connectionId: undefined, executionHostId: 'runtime:env-1' } - // No status entry for env-1 → host is disconnected. const markup = renderToStaticMarkup( ) - expect(markup).toContain('Remote Mac disconnected') + expect(markup).not.toContain('Remote Mac disconnected') + expect(markup).toContain('Project on Remote Mac') + expect(markup).not.toContain('opacity-60') }) it('distinguishes connected worktrees on different Orca servers', () => { diff --git a/src/renderer/src/components/sidebar/sidebar-host-options.test.ts b/src/renderer/src/components/sidebar/sidebar-host-options.test.ts index 454268c24c4..6fe8d0d5901 100644 --- a/src/renderer/src/components/sidebar/sidebar-host-options.test.ts +++ b/src/renderer/src/components/sidebar/sidebar-host-options.test.ts @@ -77,11 +77,10 @@ describe('sidebar host options', () => { }) expect(hosts.map((host) => host.id)).toEqual(['local', 'runtime:runtime-1']) - // Without live status the focused runtime has no proof of reachability, so it - // reads 'disconnected' rather than defaulting to 'available'/"Connected". + // A first probe still in progress is not evidence of disconnection. expect(hosts.find((host) => host.id === 'runtime:runtime-1')).toMatchObject({ detail: 'Orca server', - health: 'disconnected' + health: 'connecting' }) }) diff --git a/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts b/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts index fc339bbe33f..d8ec20bb45a 100644 --- a/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts +++ b/src/renderer/src/components/sidebar/use-worktree-card-foundation.ts @@ -10,6 +10,10 @@ import { getHostDisplayLabelOverrides } from '../../../../shared/host-setting-ov import { getWorkspacePortsByWorktreeId } from '@/lib/workspace-port-groups' import { getExplicitRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner' import { hydrateRuntimeEnvironmentSshState } from '@/runtime/runtime-environment-ssh-state' +import { + isDisconnectedRuntimeHostState, + runtimeHostConnectionStateForEntry +} from '@/runtime/runtime-host-connection-state' import { useAppStore } from '@/store' import { selectRuntimeAwareSshStatus, @@ -177,12 +181,17 @@ export function useWorktreeCardFoundation({ const runtimeHostLabel = runtimeHostId ? (getHostDisplayLabelOverrides(settings).get(runtimeHostId) ?? runtimeEnvironmentName) : null - // Why: runtime ("Orca server") hosts get the same disconnected dimming as SSH when their environment has no live status. + // Why the shared derivation, not raw truthiness: an absent entry means "not probed yet", + // which is not the same verdict as a probe that came back unreachable. const isRuntimeDisconnected = useAppStore((s) => { if (!runtimeOwnerEnvironmentId) { return false } - return !s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId)?.status + return isDisconnectedRuntimeHostState( + runtimeHostConnectionStateForEntry( + s.runtimeStatusByEnvironmentId.get(runtimeOwnerEnvironmentId) + ) + ) }) const [titleRenaming, setTitleRenaming] = useState(false) const [showRenameErrorDialog, setShowRenameErrorDialog] = useState(false) diff --git a/src/renderer/src/components/status-bar/SshStatusSegment.tsx b/src/renderer/src/components/status-bar/SshStatusSegment.tsx index be74c0fd7f2..cfaeeeb6bdc 100644 --- a/src/renderer/src/components/status-bar/SshStatusSegment.tsx +++ b/src/renderer/src/components/status-bar/SshStatusSegment.tsx @@ -33,7 +33,7 @@ import { } from './remote-host-connection-status' import { isConnectedRuntimeHostState, - runtimeHostConnectionState, + runtimeHostConnectionStateForEntry, runtimeStatusForOverall } from '@/runtime/runtime-host-connection-state' import { refreshRuntimeProjectWorktreesAndLineage } from '@/hooks/runtime-project-refresh-scheduler' @@ -74,7 +74,7 @@ export function SshStatusSegment({ const settings = useAppStore((s) => s.settings) const runtimeEnvironments = useAppStore((s) => s.runtimeEnvironments) const runtimeStatusByEnvironmentId = useAppStore((s) => s.runtimeStatusByEnvironmentId) - const setRuntimeEnvironmentStatus = useAppStore((s) => s.setRuntimeEnvironmentStatus) + const readRuntimeHostStatusSnapshots = useAppStore((s) => s.readRuntimeHostStatusSnapshots) const hydrateRuntimeEnvironmentStatuses = useAppStore((s) => s.hydrateRuntimeEnvironmentStatuses) const remoteWorkspaceSyncStatusByTargetId = useAppStore( (s) => s.remoteWorkspaceSyncStatusByTargetId @@ -105,7 +105,7 @@ export function SshStatusSegment({ return { id: environment.id, label: override || environment.name || environment.id, - hasStatusEntry: Boolean(statusEntry), + snapshot: statusEntry?.snapshot, status: statusEntry?.status ?? null, active: settings?.activeRuntimeEnvironmentId === environment.id, remoteControl: statusEntry?.remoteControl ?? statusEntry?.status?.remoteControl ?? null @@ -113,7 +113,7 @@ export function SshStatusSegment({ }) const runtimeHostRows = runtimeHosts.map((host) => ({ ...host, - state: runtimeHostConnectionState(host) + state: runtimeHostConnectionStateForEntry(runtimeStatusByEnvironmentId.get(host.id)) })) // Available remote servers are online even when they are not the active runtime. // Keep host health separate from the advanced active-server selection. @@ -152,11 +152,7 @@ export function SshStatusSegment({ async (environmentId: string): Promise => { try { await window.api.runtimeEnvironments.disconnect({ selector: environmentId }) - setRuntimeEnvironmentStatus( - environmentId, - { status: null, checkedAt: Date.now() }, - { suppressDisconnectToast: true } - ) + await readRuntimeHostStatusSnapshots() recordFeatureInteraction('ssh') } catch (err) { toast.error( @@ -169,7 +165,7 @@ export function SshStatusSegment({ ) } }, - [recordFeatureInteraction, setRuntimeEnvironmentStatus] + [recordFeatureInteraction, readRuntimeHostStatusSnapshots] ) if (targets.length === 0 && runtimeHosts.length === 0) { diff --git a/src/renderer/src/hooks/ipc-events-test-harness.ts b/src/renderer/src/hooks/ipc-events-test-harness.ts index ae9efcc798e..b3d1ef71b98 100644 --- a/src/renderer/src/hooks/ipc-events-test-harness.ts +++ b/src/renderer/src/hooks/ipc-events-test-harness.ts @@ -140,6 +140,9 @@ export async function loadIpcEventsHarness( dispatchEvent: vi.fn(), api: new Proxy( { + runtimeEnvironments: createApiNamespaceStub({ + getStatusSnapshots: () => Promise.resolve([]) + }), ui: createApiNamespaceStub({ getZoomLevel: () => 0, consumePendingOpenSettings: () => Promise.resolve(false), diff --git a/src/renderer/src/hooks/ipc-events/app-lifetime-ipc-bridge.ts b/src/renderer/src/hooks/ipc-events/app-lifetime-ipc-bridge.ts index fa683b7188f..c5964d77fec 100644 --- a/src/renderer/src/hooks/ipc-events/app-lifetime-ipc-bridge.ts +++ b/src/renderer/src/hooks/ipc-events/app-lifetime-ipc-bridge.ts @@ -1,3 +1,4 @@ +import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status' import { getTabIdsAwaitingHostHydrationRemount } from '@/lib/parked-terminal-host-hydration' import { emitAutomationsChangedWindowEvent } from '@/lib/automations-changed-window-event' import { createBackgroundSleepingAgentWakeDispatcher } from '@/lib/wake-sleeping-agents-in-background' @@ -63,13 +64,23 @@ export function installAppLifetimeIpcEvents( ) const worktreeRuntime = createWorktreeEventRuntime(unsubs, isRuntimeEnvironmentActive) - const onSharedControlDiagnostics = window.api.runtimeEnvironments?.onSharedControlDiagnostics - if (onSharedControlDiagnostics) { - unsubs.push( - onSharedControlDiagnostics((event) => { - useAppStore.getState().publishRuntimeEnvironmentDiagnostics(event) + const statusApi = window.api.runtimeEnvironments + if (statusApi?.onStatusChanged) { + const apply = (snapshot: RuntimeHostStatusSnapshot): void => { + useAppStore.getState().applyRuntimeHostStatusSnapshot(snapshot) + } + let stopped = false + unsubs.push(statusApi.onStatusChanged(apply), () => { + stopped = true + }) + void statusApi + .getStatusSnapshots() + .then((snapshots) => { + if (!stopped) { + snapshots.forEach(apply) + } }) - ) + .catch((error) => console.error('Failed to read runtime status snapshots:', error)) } const unsubscribeRuntimeEnvironmentStore = registerRuntimeClientIpcBridge(unsubs, worktreeRuntime) registerProjectCatalogIpcBridge( diff --git a/src/renderer/src/hooks/ipc-events/runtime-client-ipc-bridge.ts b/src/renderer/src/hooks/ipc-events/runtime-client-ipc-bridge.ts index dc85e375124..a5db87bbce1 100644 --- a/src/renderer/src/hooks/ipc-events/runtime-client-ipc-bridge.ts +++ b/src/renderer/src/hooks/ipc-events/runtime-client-ipc-bridge.ts @@ -29,20 +29,6 @@ import { } from './runtime-environment-subscription-selection' import type { WorktreeEventRuntime } from './worktree-event-runtime' -/** Backoff for re-asking status.get after a probe that failed on its own socket. */ -const RUNTIME_STATUS_PROBE_RETRY_DELAYS_MS = [2_000, 10_000] - -/** - * Why a request carries its reason: `reconnected` must re-ask even when the cache reads - * reachable (a restart changes the runtimeId under an unchanged-looking status), while - * `recordedUnreachable` is satisfied by any answer that clears the offline verdict. - */ -type RuntimeStatusProbeTrigger = 'reconnected' | 'recordedUnreachable' - -function isRuntimeStatusRecordedUnreachable(environmentId: string): boolean { - return useAppStore.getState().runtimeStatusByEnvironmentId?.get(environmentId)?.status === null -} - export function registerRuntimeClientIpcBridge( unsubs: (() => void)[], worktreeRuntime: WorktreeEventRuntime @@ -148,107 +134,6 @@ export function registerRuntimeClientIpcBridge( }) } - const inFlightRuntimeStatusProbes = new Set() - const trailingRuntimeStatusProbes = new Map() - const runtimeStatusProbeRetryTimers = new Map>() - let runtimeStatusProbesStopped = false - // Why not the desired-subscription set alone: a host that is not the active environment - // drops out of it the moment anything records its status unreachable, so gating the - // retry on it cancels the retry exactly where recovery matters. Only removal - // (or a still-unhydrated catalog behind an active id) decides whether to keep asking, - // and the tombstone covers the window where settings still names a deleted host active. - const shouldProbeRuntimeStatus = (environmentId: string): boolean => { - const state = useAppStore.getState() - if (state.removedRuntimeEnvironmentIds?.has(environmentId)) { - return false - } - return ( - (state.runtimeEnvironments ?? []).some((environment) => environment.id === environmentId) || - getRuntimeClientEventEnvironmentIds(state).includes(environmentId) - ) - } - // Why: concurrent probes resolve in arbitrary order, so a slow one can publish its - // stale answer over a newer one and leave the sidebar naming a superseded runtime. - const probeRuntimeStatus = ( - environmentId: string, - attempt = 0, - trigger: RuntimeStatusProbeTrigger = 'reconnected' - ): void => { - if (runtimeStatusProbesStopped || !shouldProbeRuntimeStatus(environmentId)) { - return - } - if (inFlightRuntimeStatusProbes.has(environmentId)) { - // Serialize, don't drop: the in-flight answer predates this request, so a reconnect - // that lands mid-probe would otherwise go unasked — and a probe that succeeds - // schedules no retry to pick it up later. A reconnect outranks a queued - // recorded-unreachable request, which the in-flight answer may already settle. - if (trigger === 'reconnected' || !trailingRuntimeStatusProbes.has(environmentId)) { - trailingRuntimeStatusProbes.set(environmentId, trigger) - } - return - } - const pendingRetry = runtimeStatusProbeRetryTimers.get(environmentId) - if (pendingRetry !== undefined) { - clearTimeout(pendingRetry) - runtimeStatusProbeRetryTimers.delete(environmentId) - } - inFlightRuntimeStatusProbes.add(environmentId) - void useAppStore - .getState() - // publishUnreachable: false — the transport that just proved this host alive is not the - // socket status.get dials, so a failed probe here is unverifiable and must publish nothing. - .refreshRuntimeEnvironmentStatus(environmentId, undefined, { publishUnreachable: false }) - .catch(() => false) - .then((reachable) => { - inFlightRuntimeStatusProbes.delete(environmentId) - const trailingTrigger = trailingRuntimeStatusProbes.get(environmentId) - if (trailingTrigger !== undefined) { - trailingRuntimeStatusProbes.delete(environmentId) - // A newer reconnect asked while this one was dialing: restart the attempt chain. - // A resubscribe only asked because the cache read unreachable, so skip the extra - // socket + E2EE handshake when this answer already cleared that. - if ( - trailingTrigger === 'reconnected' || - isRuntimeStatusRecordedUnreachable(environmentId) - ) { - probeRuntimeStatus(environmentId, 0, trailingTrigger) - return - } - } - // Why: status.get dials its own short-lived socket, so it can fail while the - // control transport that just proved the host is up stays healthy. That failure - // is unverifiable and publishes nothing, so no store transition, resubscribe or - // further trigger follows — without this bounded retry one unlucky probe leaves - // a host already recorded offline stranded until the next transport gap. - const retryDelayMs = RUNTIME_STATUS_PROBE_RETRY_DELAYS_MS[attempt] - if ( - reachable || - retryDelayMs === undefined || - runtimeStatusProbesStopped || - !shouldProbeRuntimeStatus(environmentId) - ) { - return - } - runtimeStatusProbeRetryTimers.set( - environmentId, - setTimeout(() => { - runtimeStatusProbeRetryTimers.delete(environmentId) - probeRuntimeStatus(environmentId, attempt + 1) - }, retryDelayMs) - ) - }) - } - unsubs.push(() => { - // The flag, not just the timers: a probe still in flight at teardown would - // otherwise schedule a fresh retry chain after the bridge is gone. - runtimeStatusProbesStopped = true - trailingRuntimeStatusProbes.clear() - for (const retryTimer of runtimeStatusProbeRetryTimers.values()) { - clearTimeout(retryTimer) - } - runtimeStatusProbeRetryTimers.clear() - }) - const runtimeClientEventsSync = createRuntimeClientEventsSync({ getDesiredEnvironmentIds: () => getRuntimeClientEventEnvironmentIds(useAppStore.getState()), getSubscriptionKey: (environmentId) => buildRuntimeClientEventEnvironmentKey([environmentId]), @@ -271,7 +156,13 @@ export function registerRuntimeClientIpcBridge( () => { invalidateRuntimeClientEventReplay({ getSshStateReference: () => useAppStore.getState().sshStateByEnvironment, - refreshRuntimeStatus: () => probeRuntimeStatus(environmentId), + refreshRuntimeStatus: () => { + const state = useAppStore.getState() + const snapshot = state.runtimeStatusByEnvironmentId.get(environmentId)?.snapshot + if (!snapshot || snapshot.transport === 'unknown') { + void state.refreshRuntimeEnvironmentStatus(environmentId) + } + }, requestProjectRefresh: () => runtimeProjectRefreshScheduler.request(environmentId), markEnvironmentSshStateStale: () => useAppStore.getState().markEnvironmentSshStateStale(environmentId), @@ -281,19 +172,6 @@ export function registerRuntimeClientIpcBridge( }) } ) - // Why: only a reconnect of an already-ready transport replays with the tag above. - // A connection whose first ready lands after the host recovered (app started, or - // the env was added, while it was down) never replays, so the recorded-unreachable - // verdict this subscribe just disproved has to be re-asked here. Kept off the - // returned promise so subscription registration/teardown ordering is unchanged. - void subscription.then( - () => { - if (isRuntimeStatusRecordedUnreachable(environmentId)) { - probeRuntimeStatus(environmentId, 0, 'recordedUnreachable') - } - }, - () => {} - ) return subscription }, onEvent: handleRuntimeClientEvent diff --git a/src/renderer/src/hooks/ipc-events/runtime-reconnect-host-status.test.ts b/src/renderer/src/hooks/ipc-events/runtime-reconnect-host-status.test.ts index 0d605a3fab1..67050a77c19 100644 --- a/src/renderer/src/hooks/ipc-events/runtime-reconnect-host-status.test.ts +++ b/src/renderer/src/hooks/ipc-events/runtime-reconnect-host-status.test.ts @@ -67,16 +67,12 @@ describe('remote Orca server reconnect', () => { }[] = [] let liveRuntimeId = 'remote-runtime' let failingStatusProbes = 0 - let blockNextStatusProbe = false - let releaseBlockedStatusProbe: (() => void) | null = null beforeEach(() => { subscriptionResponders = [] unsubs = [] liveRuntimeId = 'remote-runtime' failingStatusProbes = 0 - blockNextStatusProbe = false - releaseBlockedStatusProbe = null // Module-level sonner double: without this a toast from an earlier test leaks into // the assertions below. vi.mocked(toast.warning).mockClear() @@ -88,12 +84,6 @@ describe('remote Orca server reconnect', () => { // Captured before the block so a probe that is still dialing answers with the // runtime it was dispatched against, not with whatever restarted meanwhile. const dispatchedRuntimeId = liveRuntimeId - if (blockNextStatusProbe) { - blockNextStatusProbe = false - await new Promise((resolve) => { - releaseBlockedStatusProbe = resolve - }) - } if (failingStatusProbes > 0) { failingStatusProbes -= 1 // status.get dials its own socket; it can fail while the control transport is up. @@ -170,261 +160,42 @@ describe('remote Orca server reconnect', () => { } as unknown as WorktreeEventRuntime) } - it('re-probes a replayed subscription while the cached status still looks reachable', async () => { + it('keeps legacy event recovery as a single request, with retries owned outside the renderer', async () => { + vi.useFakeTimers() startBridge() await settle() - expect(sidebarHostHealth()).toBe('available') - - // The gap was short enough that nothing probed during it, so the cached status still - // names the pre-restart runtime and the replay tag is the only evidence it is stale. - liveRuntimeId = 'remote-runtime-restarted' + failingStatusProbes = 1 replaySubscription() await settle() - - expect( - useAppStore.getState().runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status?.runtimeId - ).toBe('remote-runtime-restarted') - expect(sidebarHostHealth()).toBe('available') - }) - - it('returns the sidebar host to online when the first subscription lands untagged', async () => { - // A connection that was never ready does not replay: nothing tags its first - // response, so a client that booted while the host was down has only the - // successful subscribe as evidence that the recorded verdict is stale. - useAppStore.setState({ - runtimeStatusByEnvironmentId: new Map([ - [ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }] - ]) as never - }) - expect(sidebarHostHealth()).toBe('disconnected') - - startBridge() - await settle() - - expect(sidebarHostHealth()).toBe('available') - expect(subscriptionResponders.length).toBeGreaterThan(0) - }) - - it('re-asks after a probe that failed while the transport stayed up', async () => { - vi.useFakeTimers() - // Already recorded unreachable, so the failing probe's `null` is an unchanged - // re-publication: it writes nothing and leaves no store transition for the - // resubscribe path to key off. Only a retry can still recover this host. - useAppStore.setState({ - runtimeStatusByEnvironmentId: new Map([ - [ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }] - ]) as never - }) - failingStatusProbes = 1 - - startBridge() - await settle() - expect(sidebarHostHealth()).toBe('disconnected') - - await vi.advanceTimersByTimeAsync(2_000) - await settle() - - expect(sidebarHostHealth()).toBe('available') - }) - - it('stops re-asking a host that keeps refusing, instead of polling it forever', async () => { - vi.useFakeTimers() - useAppStore.setState({ - runtimeStatusByEnvironmentId: new Map([ - [ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }] - ]) as never - }) - failingStatusProbes = Number.POSITIVE_INFINITY - - startBridge() - await settle() + expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(120_000) - await settle() - - // One probe on the successful subscribe plus the two bounded retries. - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(3) + expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) }) - it('stops probing once the bridge is torn down mid-probe', async () => { - vi.useFakeTimers() + it('does not add a status request when a connection-owned subscription replays', async () => { + useAppStore.getState().applyRuntimeHostStatusSnapshot({ + environmentId: ENVIRONMENT_ID, + pairingRevision: 1, + sequence: 100, + checkedAt: 1, + transport: 'ready', + verification: 'verified', + status: liveRuntimeStatus() + }) + startBridge() + await settle() + replaySubscription() + await settle() + expect(window.api.runtimeEnvironments.getStatus).not.toHaveBeenCalled() + expect(sidebarHostHealth()).toBe('available') + }) + + it('does not start UI recovery machinery when an initial subscription attaches', async () => { useAppStore.setState({ - runtimeStatusByEnvironmentId: new Map([ - [ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }] - ]) as never - }) - failingStatusProbes = Number.POSITIVE_INFINITY - blockNextStatusProbe = true - - startBridge() - await settle() - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - - // Teardown clears scheduled retries, but this probe has not answered yet. - stopBridge?.() - stopBridge = null - for (const unsub of unsubs.splice(0)) { - unsub() - } - releaseBlockedStatusProbe?.() - await settle() - await vi.advanceTimersByTimeAsync(120_000) - await settle() - - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - }) - - it('re-asks for a saved host that is not the active environment', async () => { - vi.useFakeTimers() - // A non-active host is only in the desired-subscription set while its status is non-null, - // so gating the retry on that set would strand it offline with its subscription already - // torn down the moment anything else (an explicit disconnect, the toast's own retry) - // records the same outage. - useAppStore.setState({ - settings: { activeRuntimeEnvironmentId: 'env-laptop' } as never + runtimeStatusByEnvironmentId: new Map([[ENVIRONMENT_ID, { status: null, checkedAt: 1 }]]) }) startBridge() await settle() - expect(sidebarHostHealth()).toBe('available') - - failingStatusProbes = 1 - replaySubscription() - await settle() - useAppStore.getState().setRuntimeEnvironmentStatus(ENVIRONMENT_ID, { - status: null, - checkedAt: Date.now() - }) - expect(sidebarHostHealth()).toBe('disconnected') - - await vi.advanceTimersByTimeAsync(2_000) - await settle() - - expect(sidebarHostHealth()).toBe('available') - }) - - it('re-asks after a reconnect that lands while an earlier probe is still dialing', async () => { - blockNextStatusProbe = true - startBridge() - await settle() - replaySubscription() - await settle() - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - - // The host restarts while that probe is still on its own socket: the transport drops, - // reconnects and replays again. Serializing is right, dropping the request is not — - // the in-flight answer predates the restart this replay is reporting. - liveRuntimeId = 'remote-runtime-restarted' - replaySubscription() - await settle() - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - - releaseBlockedStatusProbe?.() - await settle() - - expect( - useAppStore.getState().runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status?.runtimeId - ).toBe('remote-runtime-restarted') - }) - - it('does not resurrect a host removed while a retry was pending', async () => { - vi.useFakeTimers() - useAppStore.setState({ - runtimeStatusByEnvironmentId: new Map([ - [ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }] - ]) as never - }) - failingStatusProbes = Number.POSITIVE_INFINITY - - startBridge() - await settle() - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - - // The user deletes the remote server before the first retry fires. Settings can still - // name it as active until the next settings read, so removal is what has to stop this. - useAppStore.getState().setRuntimeEnvironments([]) - await settle() - expect(useAppStore.getState().runtimeStatusByEnvironmentId.has(ENVIRONMENT_ID)).toBe(false) - - await vi.advanceTimersByTimeAsync(120_000) - await settle() - - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - // buildExecutionHostRegistry enumerates the status map, so a re-published entry for a - // deleted id puts that host back in the sidebar under its raw id. - expect(useAppStore.getState().runtimeStatusByEnvironmentId.has(ENVIRONMENT_ID)).toBe(false) - }) - - it('keeps a live cached status when the replay-triggered probe fails on its own socket', async () => { - startBridge() - await settle() - expect(sidebarHostHealth()).toBe('available') - // Asserted over every write, not just the end state: a demotion that a later probe - // undoes still flashed the sidebar offline and still fired the toast. - let recordedUnreachable = false - unsubs.push( - useAppStore.subscribe((state) => { - recordedUnreachable ||= - state.runtimeStatusByEnvironmentId.get(ENVIRONMENT_ID)?.status === null - }) - ) - - // status.get dials its own short-lived socket, so its failure is unverifiable — and the - // transport that just replayed is proof the host is up. Recording it as offline would - // manufacture the stuck-offline sidebar this re-probe exists to cure. - failingStatusProbes = 1 - replaySubscription() - await settle() - - expect(recordedUnreachable).toBe(false) - expect(sidebarHostHealth()).toBe('available') - expect(toast.warning).not.toHaveBeenCalled() - }) - - it('returns a stuck-offline host to online when the replayed probe answers', async () => { - // The reported bug: the sidebar stayed offline after the connection recovered. The first - // probe failing keeps the recorded verdict unreachable, so only the replay recovers it - // (its retry chain is still parked behind a 2s timer this test never advances). - useAppStore.setState({ - runtimeStatusByEnvironmentId: new Map([ - [ENVIRONMENT_ID, { status: null, checkedAt: Date.now() }] - ]) as never - }) - failingStatusProbes = 1 - startBridge() - await settle() - expect(sidebarHostHealth()).toBe('disconnected') - - replaySubscription() - await settle() - - expect(sidebarHostHealth()).toBe('available') - }) - - it('does not dial a second status socket when the in-flight probe already answered', async () => { - startBridge() - await settle() - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(0) - - // A reconnect re-probes unconditionally, and that probe is still on its own socket. - blockNextStatusProbe = true - replaySubscription() - await settle() - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - - // Meanwhile the outage's own failed probe is recorded, which resubscribes; that - // resubscribe resolves against a cache that still reads unreachable. - useAppStore.getState().setRuntimeEnvironmentStatus(ENVIRONMENT_ID, { - status: null, - checkedAt: Date.now() - }) - await settle() - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - - releaseBlockedStatusProbe?.() - await settle() - - // The resubscribe only wanted an answer for a host the cache called unreachable, and - // the probe it waited on gave one: a second status.get is a whole extra socket dial. - expect(window.api.runtimeEnvironments.getStatus).toHaveBeenCalledTimes(1) - expect(sidebarHostHealth()).toBe('available') + expect(window.api.runtimeEnvironments.getStatus).not.toHaveBeenCalled() }) }) diff --git a/src/renderer/src/hooks/useIpcEvents-lifecycle.test.ts b/src/renderer/src/hooks/useIpcEvents-lifecycle.test.ts index 5991c40c4de..2e13d9382e4 100644 --- a/src/renderer/src/hooks/useIpcEvents-lifecycle.test.ts +++ b/src/renderer/src/hooks/useIpcEvents-lifecycle.test.ts @@ -30,7 +30,7 @@ const EXPECTED_DIRECT_CALLBACK_METHODS = [ 'runtime.onNativeChatLaunchDraftResolved', 'runtime.onTerminalDriverChanged', 'runtime.onTerminalFitOverrideChanged', - 'runtimeEnvironments.onSharedControlDiagnostics', + 'runtimeEnvironments.onStatusChanged', 'settings.onChanged', 'ssh.onCredentialRequest', 'ssh.onCredentialResolved', @@ -106,7 +106,7 @@ const EXPECTED_DIRECT_CALLBACK_METHODS = [ const EXPECTED_CALLBACK_REGISTRATION_SEQUENCE = [ 'ui.onMobileMarkdownRequest', 'automations.onChanged', - 'runtimeEnvironments.onSharedControlDiagnostics', + 'runtimeEnvironments.onStatusChanged', 'repos.onChanged', 'worktrees.onChanged', 'worktrees.onHeadIdentitiesChanged', @@ -382,7 +382,7 @@ describe('useIpcEvents App-lifetime lifecycle', () => { ).toEqual([ 'ui.onMobileMarkdownRequest', 'automations.onChanged', - 'runtimeEnvironments.onSharedControlDiagnostics', + 'runtimeEnvironments.onStatusChanged', 'runtimeEnvironments.subscribe', ...EXPECTED_CALLBACK_REGISTRATION_SEQUENCE.slice(3) ]) diff --git a/src/renderer/src/runtime/runtime-host-connection-state.test.ts b/src/renderer/src/runtime/runtime-host-connection-state.test.ts index 8db0c0eb010..30ee06a5e98 100644 --- a/src/renderer/src/runtime/runtime-host-connection-state.test.ts +++ b/src/renderer/src/runtime/runtime-host-connection-state.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from 'vitest' import type { RuntimeStatus } from '../../../shared/runtime-types' import { isConnectedRuntimeHostState, + isDisconnectedRuntimeHostState, runtimeHostConnectionState, + runtimeHostConnectionStateForEntry, runtimeStatusForOverall } from './runtime-host-connection-state' @@ -177,3 +179,72 @@ describe('runtime host connection state', () => { ).toBe('disconnected') }) }) + +describe('runtime host connection state for a recorded status entry', () => { + it('separates a host that was never probed from one a probe found unreachable', () => { + // The sidebar read raw truthiness, which collapsed these two into the same red glyph. + expect(runtimeHostConnectionStateForEntry(undefined)).toBe('checking') + expect(runtimeHostConnectionStateForEntry({ status: null })).toBe('disconnected') + }) + + it('reads the remote-control diagnostics recorded beside a failed probe', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: null, + remoteControl: remoteControl('reconnecting') + }) + ).toBe('reconnecting') + }) + + it('agrees with the status bar that a closed control channel is disconnected', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: makeStatus({ remoteControl: remoteControl('closed') }) + }) + ).toBe('disconnected') + }) + + it('names only the disconnected verdict as disconnected', () => { + expect(isDisconnectedRuntimeHostState('disconnected')).toBe(true) + for (const state of [ + 'connected', + 'checking', + 'reconnecting', + 'runtime-unavailable', + 'workspace-window-closed' + ] as const) { + expect(isDisconnectedRuntimeHostState(state)).toBe(false) + } + }) +}) + +function remoteControl( + state: NonNullable['state'] +): NonNullable { + return { + state, + pendingRequestCount: 0, + subscriptionCount: 0, + reconnectAttempt: 1, + lastConnectedAt: null, + lastClose: null, + lastError: null + } +} + +it('does not report reconnecting after verification is terminally blocked', () => { + expect( + runtimeHostConnectionStateForEntry({ + status: null, + snapshot: { + environmentId: 'browser', + pairingRevision: 1, + sequence: 1, + checkedAt: 1, + status: null, + verification: 'blocked', + transport: 'disconnected' + } + }) + ).toBe('disconnected') +}) diff --git a/src/renderer/src/runtime/runtime-host-connection-state.ts b/src/renderer/src/runtime/runtime-host-connection-state.ts index 6094c995804..c10477415a1 100644 --- a/src/renderer/src/runtime/runtime-host-connection-state.ts +++ b/src/renderer/src/runtime/runtime-host-connection-state.ts @@ -1,3 +1,4 @@ +import type { RuntimeHostStatusSnapshot } from '../../../shared/runtime-host-status' import type { RuntimeStatus } from '../../../shared/runtime-types' import { isRuntimeWorkspaceWindowClosed } from '../../../shared/runtime-workspace-window-availability' @@ -97,3 +98,44 @@ export function isConnectedRuntimeHostState(state: RuntimeHostConnectionState): state === 'connected' || state === 'runtime-unavailable' || state === 'workspace-window-closed' ) } + +/** + * Only this verdict earns the destructive glyph. 'checking' and 'reconnecting' are + * unverifiable, not down, per docs/reference/ssh-execution-boundary.md. + */ +export function isDisconnectedRuntimeHostState(state: RuntimeHostConnectionState): boolean { + return state === 'disconnected' +} + +/** The same derivation, read straight off a recorded status entry. */ +export function runtimeHostConnectionStateForEntry( + entry: + | { + status: RuntimeStatus | null + remoteControl?: RuntimeStatus['remoteControl'] | null + snapshot?: RuntimeHostStatusSnapshot + } + | null + | undefined +): RuntimeHostConnectionState { + if (entry?.snapshot) { + const snapshot = entry.snapshot + if (snapshot.retired || snapshot.verification === 'blocked') { + return 'disconnected' + } + if (snapshot.transport === 'disconnected') { + return 'reconnecting' + } + if (snapshot.verification === 'checking' && !entry.status) { + return 'checking' + } + if (snapshot.transport === 'ready' && snapshot.verification !== 'verified') { + return 'runtime-unavailable' + } + } + return runtimeHostConnectionState({ + hasStatusEntry: Boolean(entry), + status: entry?.status ?? null, + remoteControl: entry?.remoteControl ?? entry?.status?.remoteControl ?? null + }) +} diff --git a/src/renderer/src/store/slices/runtime-status-diagnostics-generation.ts b/src/renderer/src/store/slices/runtime-status-diagnostics-generation.ts deleted file mode 100644 index 22ba23cbbb1..00000000000 --- a/src/renderer/src/store/slices/runtime-status-diagnostics-generation.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types' -import type { RuntimeEnvironmentStatus } from './runtime-status' - -const diagnosticsGenerationByEnvironment = new Map() - -export function updateRuntimeEnvironmentStatusOverlay( - state: Map, - environmentId: string, - status: RuntimeEnvironmentStatus -): Map { - const current = state.get(environmentId) - if (!current || current.status?.runtimeId !== status.status?.runtimeId) { - return state - } - return new Map(state).set(environmentId, status) -} - -export function acceptRuntimeEnvironmentDiagnosticsGeneration( - environmentId: string, - transportGeneration: number -): boolean { - const previous = diagnosticsGenerationByEnvironment.get(environmentId) - if (previous !== undefined && transportGeneration < previous) { - return false - } - diagnosticsGenerationByEnvironment.set(environmentId, transportGeneration) - return true -} - -export function clearRuntimeEnvironmentDiagnosticsGenerationsForTests(): void { - diagnosticsGenerationByEnvironment.clear() -} - -export function mergePushedRuntimeEnvironmentDiagnostics(args: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics - current: RuntimeEnvironmentStatus | undefined - publish: (status: RuntimeEnvironmentStatus) => void -}): void { - if ( - !args.current?.status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) || - !acceptRuntimeEnvironmentDiagnosticsGeneration(args.environmentId, args.transportGeneration) - ) { - return - } - args.publish({ - ...args.current, - status: { ...args.current.status, remoteControl: args.diagnostics } - }) -} diff --git a/src/renderer/src/store/slices/runtime-status-diagnostics-publish.ts b/src/renderer/src/store/slices/runtime-status-diagnostics-publish.ts deleted file mode 100644 index e51c8621bae..00000000000 --- a/src/renderer/src/store/slices/runtime-status-diagnostics-publish.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types' -import type { AppState } from '../types' -import type { RuntimeEnvironmentStatus } from './runtime-status' -import * as diagnosticsGeneration from './runtime-status-diagnostics-generation' -import * as runtimeStatusRecheck from './runtime-status-recheck' - -export function updateRuntimeStatusStore( - state: AppState, - updater: (state: Map) => Map -): AppState | Pick { - const next = updater(state.runtimeStatusByEnvironmentId) - return next === state.runtimeStatusByEnvironmentId - ? state - : { runtimeStatusByEnvironmentId: next } -} - -export function publishRuntimeEnvironmentDiagnostics(args: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics - getCurrent: () => RuntimeEnvironmentStatus | undefined - updateState: (status: RuntimeEnvironmentStatus) => boolean - afterPublish?: (status: RuntimeEnvironmentStatus) => void -}): void { - diagnosticsGeneration.mergePushedRuntimeEnvironmentDiagnostics({ - environmentId: args.environmentId, - transportGeneration: args.transportGeneration, - diagnostics: args.diagnostics, - current: args.getCurrent(), - publish: (status) => { - if (args.updateState(status)) { - args.afterPublish?.(status) - } - } - }) -} - -export function applyRuntimeEnvironmentStatusOverlay(args: { - environmentId: string - status: RuntimeEnvironmentStatus - setState: ( - updater: (state: Map) => Map - ) => void -}): boolean { - let updated = false - args.setState((state) => { - const next = diagnosticsGeneration.updateRuntimeEnvironmentStatusOverlay( - state, - args.environmentId, - args.status - ) - updated = next !== state - return next - }) - return updated -} - -export function createRuntimeEnvironmentDiagnosticsPublisher(args: { - getCurrent: (environmentId: string) => RuntimeEnvironmentStatus | undefined - setState: ( - updater: (state: Map) => Map - ) => void - afterPublish: (environmentId: string, status: RuntimeEnvironmentStatus) => void -}): (event: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics -}) => void { - return (event) => - publishRuntimeEnvironmentDiagnostics({ - ...event, - getCurrent: () => args.getCurrent(event.environmentId), - updateState: (status) => - applyRuntimeEnvironmentStatusOverlay({ - environmentId: event.environmentId, - status, - setState: args.setState - }), - afterPublish: (status) => args.afterPublish(event.environmentId, status) - }) -} - -export function createRuntimeEnvironmentDiagnosticsSlicePublisher(args: { - getCurrent: (environmentId: string) => RuntimeEnvironmentStatus | undefined - setState: ( - updater: (state: Map) => Map - ) => void - getStore: () => AppState - getConnectionGeneration: (environmentId: string) => number -}): (event: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics -}) => void { - return createRuntimeEnvironmentDiagnosticsPublisher({ - getCurrent: args.getCurrent, - setState: args.setState, - afterPublish: (environmentId, status) => - runtimeStatusRecheck.reconcileRuntimeStatusForSlice( - environmentId, - status.status, - args.getStore, - () => args.getConnectionGeneration(environmentId) - ) - }) -} diff --git a/src/renderer/src/store/slices/runtime-status-diagnostics.test.ts b/src/renderer/src/store/slices/runtime-status-diagnostics.test.ts deleted file mode 100644 index 9faecdcabf1..00000000000 --- a/src/renderer/src/store/slices/runtime-status-diagnostics.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { create } from 'zustand' -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { RuntimeStatus } from '../../../../shared/runtime-types' -import { createRuntimeStatusSlice, type RuntimeStatusSlice } from './runtime-status' - -function makeStatus(overrides: Partial = {}): RuntimeStatus { - return { - runtimeId: 'runtime-a', - rendererGraphEpoch: 0, - graphStatus: 'ready', - authoritativeWindowId: null, - liveTabCount: 3, - liveLeafCount: 0, - runtimeProtocolVersion: 3, - minCompatibleRuntimeClientVersion: 3, - capabilities: ['browser.screencast.v1'], - ...overrides - } as RuntimeStatus -} - -function createSliceStore() { - return create()((...a) => ({ - ...createRuntimeStatusSlice(...(a as unknown as Parameters)) - })) -} - -describe('runtime-status diagnostics', () => { - it('merges transport diagnostics into the complete status and fences stale pushes', () => { - const store = createSliceStore() - const status = makeStatus({ - capabilities: ['browser.screencast.v1', REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] - }) - store.getState().setRuntimeEnvironmentStatus('env-a', { status, checkedAt: 1 }) - const closed = { - state: 'closed' as const, - pendingRequestCount: 0, - subscriptionCount: 1, - reconnectAttempt: 2, - lastConnectedAt: 1, - lastClose: { code: 1006, reason: 'network' }, - lastError: 'connection lost' - } - store.getState().publishRuntimeEnvironmentDiagnostics({ - environmentId: 'env-a', - transportGeneration: 3, - diagnostics: closed - }) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toMatchObject({ - runtimeId: 'runtime-a', - capabilities: expect.arrayContaining([ - 'browser.screencast.v1', - REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY - ]), - liveTabCount: 3, - remoteControl: closed - }) - store.getState().publishRuntimeEnvironmentDiagnostics({ - environmentId: 'env-a', - transportGeneration: 2, - diagnostics: { ...closed, state: 'ready' } - }) - expect( - store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl?.state - ).toBe('closed') - }) - - it('ignores diagnostics after the latest status drops shared-control support', () => { - const store = createSliceStore() - const status = makeStatus({ capabilities: [] }) - store.getState().setRuntimeEnvironmentStatus('env-a', { status, checkedAt: 1 }) - - store.getState().publishRuntimeEnvironmentDiagnostics({ - environmentId: 'env-a', - transportGeneration: 3, - diagnostics: { - state: 'reconnecting', - pendingRequestCount: 0, - subscriptionCount: 1, - reconnectAttempt: 2, - lastConnectedAt: 1, - lastClose: { code: 1006, reason: 'network' }, - lastError: 'connection lost' - } - }) - - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(status) - }) -}) diff --git a/src/renderer/src/store/slices/runtime-status-recheck.test.ts b/src/renderer/src/store/slices/runtime-status-recheck.test.ts deleted file mode 100644 index 9f09580c390..00000000000 --- a/src/renderer/src/store/slices/runtime-status-recheck.test.ts +++ /dev/null @@ -1,238 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { create } from 'zustand' -import { toast } from 'sonner' -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments' -import type { RuntimeStatus } from '../../../../shared/runtime-types' -import { - clearRuntimeEnvironmentConnectionGenerationsForTests, - createRuntimeStatusSlice, - setRuntimeEnvironmentConnectionGenerationForTests, - type RuntimeStatusSlice -} from './runtime-status' -import { clearRuntimeStatusRechecksForTests } from './runtime-status-recheck' - -vi.mock('sonner', () => ({ toast: { warning: vi.fn(), dismiss: vi.fn() } })) - -beforeEach(() => { - vi.useFakeTimers() - clearRuntimeStatusRechecksForTests() - clearRuntimeEnvironmentConnectionGenerationsForTests() - vi.mocked(toast.warning).mockReset() -}) - -afterEach(() => { - clearRuntimeStatusRechecksForTests() - vi.useRealTimers() - vi.unstubAllGlobals() -}) - -describe('runtime status recheck', () => { - it('publishes an observe-only ready result through the setter', async () => { - const getStatus = vi.fn().mockResolvedValue(response(status('ready'))) - const store = createStore(getStatus) - - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - await vi.advanceTimersByTimeAsync(3_000) - - expect(getStatus).toHaveBeenCalledWith({ - selector: 'env-a', - timeoutMs: 10_000, - observeOnly: true - }) - expect( - store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl - ).toMatchObject({ - state: 'ready' - }) - await vi.advanceTimersByTimeAsync(120_000) - expect(getStatus).toHaveBeenCalledOnce() - }) - - it('continues indefinitely on the capped ladder, including unchanged publishes', async () => { - const getStatus = vi.fn().mockResolvedValue(response(status('reconnecting'))) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('reconnecting'), - checkedAt: 1 - }) - - await vi.advanceTimersByTimeAsync(3_000 + 6_000 + 12_000 + 30_000 + 60_000 + 60_000) - - expect(getStatus).toHaveBeenCalledTimes(6) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.checkedAt).toBe(1) - }) - - it('cancels on removal, capability loss, and null without probing again', async () => { - const getStatus = vi.fn() - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_authenticated'), - checkedAt: 1 - }) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: { ...status('awaiting_authenticated'), capabilities: [] }, - checkedAt: 2 - }) - await vi.advanceTimersByTimeAsync(60_000) - expect(getStatus).not.toHaveBeenCalled() - - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 3 - }) - store.getState().setRuntimeEnvironments([]) - await vi.advanceTimersByTimeAsync(60_000) - expect(getStatus).not.toHaveBeenCalled() - }) - - it('cancels an armed probe when the connection generation changes', async () => { - const getStatus = vi.fn() - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - - setRuntimeEnvironmentConnectionGenerationForTests('env-a', 2) - await vi.advanceTimersByTimeAsync(3_000) - - expect(getStatus).not.toHaveBeenCalled() - }) - - it('restarts the ladder for a newly published connection generation', async () => { - const getStatus = vi.fn().mockResolvedValue(response(status('ready', 'rt-next'))) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready', 'rt-next'), - checkedAt: 2 - }) - - await vi.advanceTimersByTimeAsync(3_000) - - expect(getStatus).toHaveBeenCalledOnce() - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.runtimeId).toBe( - 'rt-next' - ) - }) - - it('discards an in-flight result after a ready publish bumps the epoch', async () => { - const pending = deferred>() - const getStatus = vi.fn().mockReturnValue(pending.promise) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - await vi.advanceTimersByTimeAsync(3_000) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('ready'), - checkedAt: 2 - }) - - pending.resolve(response(status('reconnecting'))) - await Promise.resolve() - await Promise.resolve() - - expect( - store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.remoteControl - ).toMatchObject({ - state: 'ready' - }) - }) - - it('keeps setter side effects when a recheck discovers disconnection', async () => { - const getStatus = vi.fn().mockResolvedValue({ - id: 'status.get', - ok: false, - error: { - code: 'runtime_unavailable', - message: 'offline', - data: { remoteControl: status('reconnecting').remoteControl } - }, - _meta: { runtimeId: 'rt' } - }) - const store = createStore(getStatus) - store.getState().setRuntimeEnvironmentStatus('env-a', { - status: status('awaiting_ready'), - checkedAt: 1 - }) - - await vi.advanceTimersByTimeAsync(3_000) - - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBeNull() - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.remoteControl).toMatchObject( - { - state: 'reconnecting' - } - ) - expect(toast.warning).toHaveBeenCalledOnce() - }) -}) - -function createStore(getStatus: ReturnType) { - vi.stubGlobal('window', { - api: { runtimeEnvironments: { getStatus, list: vi.fn() } } - }) - const store = create()((...args) => ({ - ...createRuntimeStatusSlice(...(args as unknown as Parameters)) - })) - store.getState().setRuntimeEnvironments([environment()]) - return store -} - -function status( - controlState: NonNullable['state'], - runtimeId = 'rt' -): RuntimeStatus { - return { - runtimeId, - rendererGraphEpoch: 1, - graphStatus: 'ready', - authoritativeWindowId: null, - liveTabCount: 0, - liveLeafCount: 0, - capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY], - remoteControl: { - state: controlState, - pendingRequestCount: 0, - subscriptionCount: 0, - reconnectAttempt: 1, - lastConnectedAt: null, - lastClose: null, - lastError: null - } - } as RuntimeStatus -} - -function response(result: RuntimeStatus) { - return { id: 'status.get', ok: true as const, result, _meta: { runtimeId: result.runtimeId } } -} - -function environment(): PublicKnownRuntimeEnvironment { - return { - id: 'env-a', - name: 'Dev Box', - createdAt: 1, - updatedAt: 1, - lastUsedAt: null, - runtimeId: 'rt', - endpoints: [{ id: 'ws', kind: 'websocket', label: 'WebSocket', endpoint: 'ws://x' }], - preferredEndpointId: 'ws' - } -} - -function deferred() { - let resolve: (value: T) => void = () => {} - const promise = new Promise((done) => { - resolve = done - }) - return { promise, resolve } -} diff --git a/src/renderer/src/store/slices/runtime-status-recheck.ts b/src/renderer/src/store/slices/runtime-status-recheck.ts deleted file mode 100644 index 303e5b4318d..00000000000 --- a/src/renderer/src/store/slices/runtime-status-recheck.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../../../shared/protocol-version' -import type { RuntimeStatus } from '../../../../shared/runtime-types' -import { unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client' -import { extractRuntimeTransportDiagnostics } from '@/runtime/runtime-status-probe-diagnostics' -import type { RuntimeEnvironmentStatus } from './runtime-status' - -const RECHECK_DELAYS_MS = [3_000, 6_000, 12_000, 30_000, 60_000] - -type RecheckState = { - epoch: number - attempt: number - timer: ReturnType | null - inFlight: boolean - connectionGeneration: number - environmentExists: () => boolean - getConnectionGeneration: () => number - publish: (status: RuntimeEnvironmentStatus) => void -} - -type RuntimeStatusStore = { - runtimeEnvironments: readonly { id: string }[] - setRuntimeEnvironmentStatus: (environmentId: string, status: RuntimeEnvironmentStatus) => void -} - -const rechecks = new Map() - -export function reconcileRuntimeStatusRecheck(args: { - environmentId: string - status: RuntimeStatus | null - connectionGeneration: number - environmentExists: () => boolean - getConnectionGeneration: () => number - publish: (status: RuntimeEnvironmentStatus) => void -}): void { - if (!shouldRecheck(args.status)) { - cancelRuntimeStatusRecheck(args.environmentId) - return - } - let state = rechecks.get(args.environmentId) - if (state && state.connectionGeneration !== args.connectionGeneration) { - cancelRuntimeStatusRecheck(args.environmentId) - state = undefined - } - if (!state) { - state = { - epoch: 0, - attempt: 0, - timer: null, - inFlight: false, - connectionGeneration: args.connectionGeneration, - environmentExists: args.environmentExists, - getConnectionGeneration: args.getConnectionGeneration, - publish: args.publish - } - rechecks.set(args.environmentId, state) - } else { - state.connectionGeneration = args.connectionGeneration - state.environmentExists = args.environmentExists - state.getConnectionGeneration = args.getConnectionGeneration - state.publish = args.publish - } - armRuntimeStatusRecheck(args.environmentId, state) -} - -export function reconcileRuntimeStatusForSlice( - environmentId: string, - status: RuntimeStatus | null, - get: () => RuntimeStatusStore, - getConnectionGeneration: () => number -): void { - reconcileRuntimeStatusRecheck({ - environmentId, - status, - connectionGeneration: getConnectionGeneration(), - environmentExists: () => - get().runtimeEnvironments.some((environment) => environment.id === environmentId), - getConnectionGeneration, - publish: (nextStatus) => get().setRuntimeEnvironmentStatus(environmentId, nextStatus) - }) -} - -export function cancelRuntimeStatusRecheck(environmentId: string): void { - const state = rechecks.get(environmentId) - if (!state) { - return - } - state.epoch += 1 - if (state.timer) { - clearTimeout(state.timer) - } - rechecks.delete(environmentId) -} - -export function cancelRuntimeStatusRechecks(environmentIds: Iterable): void { - for (const environmentId of environmentIds) { - cancelRuntimeStatusRecheck(environmentId) - } -} - -export function clearRuntimeStatusRechecksForTests(): void { - cancelRuntimeStatusRechecks([...rechecks.keys()]) -} - -function shouldRecheck(status: RuntimeStatus | null): boolean { - return Boolean( - status?.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) && - status.remoteControl && - status.remoteControl.state !== 'ready' - ) -} - -function armRuntimeStatusRecheck(environmentId: string, state: RecheckState): void { - if (state.timer || state.inFlight) { - return - } - const delay = RECHECK_DELAYS_MS[Math.min(state.attempt, RECHECK_DELAYS_MS.length - 1)] - const generation = state.connectionGeneration - state.attempt += 1 - state.timer = setTimeout( - () => void fireRuntimeStatusRecheck(environmentId, state, generation), - delay - ) -} - -async function fireRuntimeStatusRecheck( - environmentId: string, - state: RecheckState, - generation: number -): Promise { - state.timer = null - const epoch = state.epoch - if ( - rechecks.get(environmentId) !== state || - !state.environmentExists() || - state.getConnectionGeneration() !== generation - ) { - cancelRuntimeStatusRecheck(environmentId) - return - } - state.inFlight = true - let nextEntry: RuntimeEnvironmentStatus - try { - const response = await window.api.runtimeEnvironments.getStatus({ - selector: environmentId, - timeoutMs: 10_000, - observeOnly: true - }) - nextEntry = { status: unwrapRuntimeRpcResult(response), checkedAt: Date.now() } - } catch (error: unknown) { - const remoteControl = extractRuntimeTransportDiagnostics(error) - nextEntry = { - status: null, - ...(remoteControl ? { remoteControl } : {}), - checkedAt: Date.now() - } - } - state.inFlight = false - if ( - rechecks.get(environmentId) !== state || - state.epoch !== epoch || - !state.environmentExists() || - state.getConnectionGeneration() !== generation - ) { - return - } - state.publish(nextEntry) -} diff --git a/src/renderer/src/store/slices/runtime-status-refresh.ts b/src/renderer/src/store/slices/runtime-status-refresh.ts index 638a3fae636..fb27eec6a43 100644 --- a/src/renderer/src/store/slices/runtime-status-refresh.ts +++ b/src/renderer/src/store/slices/runtime-status-refresh.ts @@ -15,6 +15,22 @@ export async function refreshRuntimeEnvironmentStatus( selector: environmentId, timeoutMs }) + if (window.api.runtimeEnvironments.getStatusSnapshots) { + try { + const snapshots = await window.api.runtimeEnvironments.getStatusSnapshots() + const snapshot = snapshots.find((entry) => entry.environmentId === environmentId) + if (snapshot) { + publish({ + snapshot, + status: snapshot.verification === 'verified' ? snapshot.status : null, + checkedAt: snapshot.checkedAt + }) + } + } catch (error) { + console.error('Failed to read runtime host status snapshot:', error) + } + return response.ok + } const status = unwrapRuntimeRpcResult(response) if (getRuntimeEnvironmentRevision(environmentId) !== expectedEnvironmentRevision) { return false diff --git a/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts b/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts index 0ba4d645026..d4b788cd49a 100644 --- a/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts +++ b/src/renderer/src/store/slices/runtime-status-restored-browser-host-attach.test.ts @@ -73,14 +73,10 @@ describe('restored client-hosted browser host attach on reachability', () => { }) }) - // The reconnect policy suppresses the *failure* publish only. A probe that answered still owes - // both recovery follow-ups, or a restored client-hosted page never comes back after the gap. - it('runs both recovery follow-ups on a success when the caller opted out of publishing failures', async () => { + it('runs both recovery follow-ups after a successful refresh', async () => { stubApi(vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a'))) - await storeWithRestoredHandles(true) - .getState() - .refreshRuntimeEnvironmentStatus('env-a', undefined, { publishUnreachable: false }) + await storeWithRestoredHandles(true).getState().refreshRuntimeEnvironmentStatus('env-a') expect(prepareBrowserClientHostPlacement).toHaveBeenCalledWith({ selector: 'env-a', @@ -89,24 +85,12 @@ describe('restored client-hosted browser host attach on reachability', () => { expect(replayClientHostedBrowserCloseIntents).toHaveBeenCalledWith('env-a', expect.anything()) }) - // Under either policy a failed probe owes *no* follow-ups: it verified nothing, so there is no - // recovered host to reattach restored pages to and no one to replay closes at. - it.each([ - { name: 'the default policy', options: undefined }, - { name: 'a caller that opted out of publishing', options: { publishUnreachable: false } } - ])( - 'starts no browser client host when the environment is unreachable: $name', - async (scenario) => { - stubApi(vi.fn().mockRejectedValue(new Error('unreachable'))) - - await storeWithRestoredHandles(true) - .getState() - .refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options) - - expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() - expect(replayClientHostedBrowserCloseIntents).not.toHaveBeenCalled() - } - ) + it('runs no recovery follow-ups when the environment is unreachable', async () => { + stubApi(vi.fn().mockRejectedValue(new Error('unreachable'))) + await storeWithRestoredHandles(true).getState().refreshRuntimeEnvironmentStatus('env-a') + expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() + expect(replayClientHostedBrowserCloseIntents).not.toHaveBeenCalled() + }) it('starts no browser client host for restored pages the server hosts', async () => { stubApi(vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a'))) diff --git a/src/renderer/src/store/slices/runtime-status-snapshot.test.ts b/src/renderer/src/store/slices/runtime-status-snapshot.test.ts new file mode 100644 index 00000000000..fa437065b28 --- /dev/null +++ b/src/renderer/src/store/slices/runtime-status-snapshot.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, expect, it, vi } from 'vitest' +import { create } from 'zustand' +import { toast } from 'sonner' +import { + createRuntimeStatusSlice, + clearRuntimeEnvironmentConnectionGenerationsForTests, + type RuntimeStatusSlice +} from './runtime-status' +import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status' +import type { RuntimeStatus } from '../../../../shared/runtime-types' +import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments' +import { runtimeHostConnectionStateForEntry } from '@/runtime/runtime-host-connection-state' + +vi.mock('sonner', () => ({ toast: { warning: vi.fn(), dismiss: vi.fn() } })) +vi.mock('@/runtime/restored-client-hosted-browser-host-attach', () => ({ + ensureBrowserClientHostsForRestoredPages: vi.fn(), + ensureBrowserClientHostForRestartedRuntime: vi.fn() +})) +vi.mock('@/runtime/client-hosted-browser-close-intent-replay', () => ({ + replayClientHostedBrowserCloseIntents: vi.fn() +})) + +beforeEach(() => { + clearRuntimeEnvironmentConnectionGenerationsForTests() + vi.clearAllMocks() +}) +const environment = { + id: 'env-a', + name: 'Host', + createdAt: 1, + pairingRevision: 1, + endpoints: [], + preferredEndpointId: '' +} as unknown as PublicKnownRuntimeEnvironment +function store() { + const value = create()((...args) => + createRuntimeStatusSlice(...(args as unknown as Parameters)) + ) + value.getState().setRuntimeEnvironments([environment]) + return value +} +function snapshot( + sequence: number, + patch: Partial = {} +): RuntimeHostStatusSnapshot { + return { + environmentId: 'env-a', + pairingRevision: 1, + sequence, + checkedAt: sequence, + transport: 'ready', + verification: 'verified', + status: { runtimeId: 'rt-1' } as RuntimeStatus, + ...patch + } +} + +it('hydrates both viewers and rejects an older read after a newer publication', () => { + for (const viewer of [store(), store()]) { + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2)) + viewer + .getState() + .applyRuntimeHostStatusSnapshot(snapshot(1, { status: null, verification: 'unavailable' })) + expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.runtimeId).toBe( + 'rt-1' + ) + } +}) + +it('represents failed verification honestly without manufacturing a session restart or toast', () => { + const viewer = store() + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(1)) + const generation = viewer + .getState() + .runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2, { verification: 'unavailable' })) + expect( + runtimeHostConnectionStateForEntry(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')) + ).toBe('runtime-unavailable') + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(3)) + expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration).toBe( + generation + ) + expect(toast.warning).not.toHaveBeenCalled() + viewer + .getState() + .applyRuntimeHostStatusSnapshot(snapshot(4, { status: { runtimeId: 'rt-2' } as RuntimeStatus })) + expect( + viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration + ).toBeGreaterThan(generation ?? 0) +}) + +it('retains disconnect ordering and rejects publications for removed or replaced pairings', () => { + const viewer = store() + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(1)) + viewer + .getState() + .applyRuntimeHostStatusSnapshot( + snapshot(3, { retired: true, verification: 'blocked', transport: 'disconnected' }) + ) + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(2)) + expect( + runtimeHostConnectionStateForEntry(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')) + ).toBe('disconnected') + viewer.getState().setRuntimeEnvironments([{ ...environment, pairingRevision: 2 }]) + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(4)) + expect(viewer.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false) + viewer.getState().setRuntimeEnvironments([]) + viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(5, { pairingRevision: 2 })) + expect(viewer.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false) +}) diff --git a/src/renderer/src/store/slices/runtime-status-snapshot.ts b/src/renderer/src/store/slices/runtime-status-snapshot.ts new file mode 100644 index 00000000000..b3543471d29 --- /dev/null +++ b/src/renderer/src/store/slices/runtime-status-snapshot.ts @@ -0,0 +1,43 @@ +import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status' +import type { AppState } from '../types' +import type { RuntimeEnvironmentStatus } from './runtime-status-types' +import { ensureBrowserClientHostsForRestoredPages } from '@/runtime/restored-client-hosted-browser-host-attach' +import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay' + +export function applyRuntimeHostStatusSnapshot( + snapshot: RuntimeHostStatusSnapshot, + state: AppState, + publishEvidence: (entry: RuntimeEnvironmentStatus) => void +): void { + const environment = state.runtimeEnvironments.find((entry) => entry.id === snapshot.environmentId) + if ( + !environment || + (environment.pairingRevision ?? environment.createdAt) !== snapshot.pairingRevision + ) { + return + } + const previous = state.runtimeStatusByEnvironmentId.get(snapshot.environmentId) + if (previous?.snapshot && previous.snapshot.sequence >= snapshot.sequence) { + return + } + const entry: RuntimeEnvironmentStatus = { + snapshot, + checkedAt: snapshot.checkedAt, + connectionGeneration: previous?.connectionGeneration, + status: snapshot.verification === 'verified' && !snapshot.retired ? snapshot.status : null, + remoteControl: snapshot.remoteControl + } + if (entry.status) { + if (snapshot.remoteControl) { + entry.status = { ...entry.status, remoteControl: snapshot.remoteControl } + } + state.setRuntimeEnvironmentStatus(snapshot.environmentId, entry) + if (previous?.status == null) { + void ensureBrowserClientHostsForRestoredPages(state) + void replayClientHostedBrowserCloseIntents(snapshot.environmentId, state) + } + } else { + // Lost contact or a failed method observes no runtime session ending. + publishEvidence(entry) + } +} diff --git a/src/renderer/src/store/slices/runtime-status-types.ts b/src/renderer/src/store/slices/runtime-status-types.ts index c34feaf07bf..78e47123cf5 100644 --- a/src/renderer/src/store/slices/runtime-status-types.ts +++ b/src/renderer/src/store/slices/runtime-status-types.ts @@ -1,8 +1,9 @@ +import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host-status' import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments' import type { RuntimeStatus } from '../../../../shared/runtime-types' -import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types' export type RuntimeEnvironmentStatus = { + snapshot?: RuntimeHostStatusSnapshot status: RuntimeStatus | null remoteControl?: RuntimeStatus['remoteControl'] | null appVersion?: string | null @@ -10,11 +11,9 @@ export type RuntimeEnvironmentStatus = { connectionGeneration?: number } -export type RuntimeStatusRefreshOptions = { - publishUnreachable?: boolean -} - export type RuntimeStatusSlice = { + readRuntimeHostStatusSnapshots: () => Promise + applyRuntimeHostStatusSnapshot: (snapshot: RuntimeHostStatusSnapshot) => void runtimeEnvironments: readonly PublicKnownRuntimeEnvironment[] runtimeEnvironmentCatalogHydrated: boolean runtimeEnvironmentCatalogSettled: boolean @@ -26,17 +25,8 @@ export type RuntimeStatusSlice = { status: RuntimeEnvironmentStatus, options?: { suppressDisconnectToast?: boolean } ) => void - publishRuntimeEnvironmentDiagnostics: (args: { - environmentId: string - transportGeneration: number - diagnostics: RemoteRuntimeSharedConnectionDiagnostics - }) => void clearRuntimeEnvironmentStatus: (environmentId: string) => void retainRuntimeEnvironmentStatuses: (environmentIds: Iterable) => void - refreshRuntimeEnvironmentStatus: ( - environmentId: string, - timeoutMs?: number, - options?: RuntimeStatusRefreshOptions - ) => Promise + refreshRuntimeEnvironmentStatus: (environmentId: string, timeoutMs?: number) => Promise hydrateRuntimeEnvironmentStatuses: () => Promise } diff --git a/src/renderer/src/store/slices/runtime-status.test.ts b/src/renderer/src/store/slices/runtime-status.test.ts index 8e3260957bf..fdc843c748a 100644 --- a/src/renderer/src/store/slices/runtime-status.test.ts +++ b/src/renderer/src/store/slices/runtime-status.test.ts @@ -710,33 +710,37 @@ describe('runtime-status slice', () => { clearRuntimeCompatibilityCacheForTests() }) - // Both directions of the failure-publication policy, from one failing probe. A user-initiated - // check publishes the outage it just observed; a caller holding live transport evidence must - // not, because status.get dials its own socket and its failure is unverifiable, not exited. - it.each([ - { name: 'a user-initiated check', options: undefined, publishes: true }, - { name: 'publishUnreachable defaulted', options: {}, publishes: true }, - { - name: 'a caller that opted out of publishing', - options: { publishUnreachable: false }, - publishes: false - } - ])('records null and returns false when a runtime refresh fails: $name', async (scenario) => { + it('records null and returns false when a runtime refresh fails', async () => { const getStatus = vi.fn().mockRejectedValue(new Error('closed')) stubRuntimeEnvironmentApi({ getStatus }) const store = createSliceStore() const cached = makeStatus() store.getState().setRuntimeEnvironmentStatus('env-a', { status: cached, checkedAt: 1 }) - const reachable = await store - .getState() - .refreshRuntimeEnvironmentStatus('env-a', undefined, scenario.options) + const reachable = await store.getState().refreshRuntimeEnvironmentStatus('env-a') - // The dial-answered contract the bridge's bounded retry chain reads is policy-independent. expect(reachable).toBe(false) - expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe( - scenario.publishes ? null : cached - ) + expect(store.getState().runtimeStatusByEnvironmentId.get('env-a')?.status).toBe(null) + }) + + it('preserves successful reachability when reading its snapshot fails', async () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.stubGlobal('window', { + api: { + runtimeEnvironments: { + getStatus: vi.fn().mockResolvedValue(createCompatibleRuntimeStatusResponse('runtime-a')), + getStatusSnapshots: vi.fn().mockRejectedValue(new Error('IPC read failed')) + } + } + }) + try { + const store = createSliceStore() + expect(await store.getState().refreshRuntimeEnvironmentStatus('env-a')).toBe(true) + expect(store.getState().runtimeStatusByEnvironmentId.has('env-a')).toBe(false) + expect(log).toHaveBeenCalled() + } finally { + log.mockRestore() + } }) it('hydrates saved environments through the single-environment refresh path', async () => { diff --git a/src/renderer/src/store/slices/runtime-status.ts b/src/renderer/src/store/slices/runtime-status.ts index 10985f389d8..c11fba3308a 100644 --- a/src/renderer/src/store/slices/runtime-status.ts +++ b/src/renderer/src/store/slices/runtime-status.ts @@ -1,11 +1,7 @@ import type { StateCreator } from 'zustand' import type { AppState } from '../types' import type { RuntimeStatusSlice } from './runtime-status-types' -export type { - RuntimeEnvironmentStatus, - RuntimeStatusRefreshOptions, - RuntimeStatusSlice -} from './runtime-status-types' +export type { RuntimeEnvironmentStatus, RuntimeStatusSlice } from './runtime-status-types' import { runtimeEnvironmentStatusesEqual } from './runtime-environment-status-equality' import { clearRecentRuntimeCompatibilityFailure, @@ -20,21 +16,16 @@ import { import { reconcileCatalogRows } from './repo-identity-reconcile' import { createRuntimeStatusHydration } from './runtime-status-hydration' import { refreshRuntimeEnvironmentStatus } from './runtime-status-refresh' -import * as runtimeStatusDiagnostics from './runtime-status-diagnostics-generation' import * as runtimeStatusConnectionGeneration from './runtime-status-connection-generation' import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay' import { ensureBrowserClientHostForRestartedRuntime, ensureBrowserClientHostsForRestoredPages } from '@/runtime/restored-client-hosted-browser-host-attach' -import * as runtimeStatusRecheck from './runtime-status-recheck' -import * as runtimeStatusDiagnosticsPublish from './runtime-status-diagnostics-publish' +import { applyRuntimeHostStatusSnapshot } from './runtime-status-snapshot' export const clearRuntimeEnvironmentConnectionGenerationsForTests = (): void => { - runtimeStatusRecheck.cancelRuntimeStatusRechecks( - runtimeStatusConnectionGeneration.clearRuntimeEnvironmentConnectionGenerations() - ) - runtimeStatusDiagnostics.clearRuntimeEnvironmentDiagnosticsGenerationsForTests() + runtimeStatusConnectionGeneration.clearRuntimeEnvironmentConnectionGenerations() } export { @@ -52,6 +43,15 @@ export const createRuntimeStatusSlice: StateCreator { + try { + const snapshots = await window.api.runtimeEnvironments.getStatusSnapshots() + snapshots.forEach((snapshot) => get().applyRuntimeHostStatusSnapshot(snapshot)) + } catch (error) { + console.error('Failed to read runtime host status:', error) + } + }, + setRuntimeEnvironments: (environments) => { const previousRevisionById = new Map( get().runtimeEnvironments.map((environment) => [ @@ -76,7 +76,6 @@ export const createRuntimeStatusSlice: StateCreator environment.id) .filter((id) => !nextIds.has(id)) - runtimeStatusRecheck.cancelRuntimeStatusRechecks([...removedIds, ...replacedEnvironmentIds]) set((s) => { const keep = new Set(environments.map((environment) => environment.id)) const nextStatuses = new Map(s.runtimeStatusByEnvironmentId) @@ -155,15 +154,29 @@ export const createRuntimeStatusSlice: StateCreator + applyRuntimeHostStatusSnapshot(snapshot, get(), (entry) => { + set((s) => ({ + runtimeStatusByEnvironmentId: new Map(s.runtimeStatusByEnvironmentId).set( + snapshot.environmentId, + entry + ) + })) + }), + setRuntimeEnvironmentStatus: (environmentId, status, options) => { const previous = get().runtimeStatusByEnvironmentId.get(environmentId) + if (previous?.snapshot && !status.snapshot) { + return + } + const previousVerifiedStatus = previous?.snapshot?.status ?? previous?.status const pairedDeviceId = status.status?.pairedDeviceId?.trim() // A new runtime id under a known previous one is a restart, not a first connect: the guests are // still ours to host, but only a fresh attach hands them back to the replacement runtime. const runtimeRestarted = Boolean( status.status !== null && - previous?.status != null && - previous.status.runtimeId !== status.status.runtimeId + previousVerifiedStatus != null && + previousVerifiedStatus.runtimeId !== status.status.runtimeId ) // Why: a non-null status proves the runtime just answered, so drop any stale // "offline" compat failure before this online transition fires the @@ -177,7 +190,8 @@ export const createRuntimeStatusSlice: StateCreator - get().runtimeEnvironments.some((environment) => environment.id === environmentId), - getConnectionGeneration: () => - runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration(environmentId), - publish: (entry) => get().setRuntimeEnvironmentStatus(environmentId, entry) - }) if (runtimeRestarted) { void ensureBrowserClientHostForRestartedRuntime(get(), environmentId) } @@ -250,18 +253,7 @@ export const createRuntimeStatusSlice: StateCreator get().runtimeStatusByEnvironmentId.get(environmentId), - setState: (updater) => - set((s) => runtimeStatusDiagnosticsPublish.updateRuntimeStatusStore(s, updater)), - getStore: get, - getConnectionGeneration: - runtimeStatusConnectionGeneration.getRuntimeEnvironmentConnectionGeneration - }), - clearRuntimeEnvironmentStatus: (environmentId) => { - runtimeStatusRecheck.cancelRuntimeStatusRecheck(environmentId) dismissRuntimeDisconnectedToast(environmentId) set((s) => { runtimeStatusConnectionGeneration.advanceRuntimeEnvironmentConnectionGeneration(environmentId) @@ -278,7 +270,6 @@ export const createRuntimeStatusSlice: StateCreator + refreshRuntimeEnvironmentStatus: (environmentId, timeoutMs = 10_000) => refreshRuntimeEnvironmentStatus(environmentId, timeoutMs, (entry) => { - if (entry.status === null && options?.publishUnreachable === false) { - // Unverifiable, not exited: leave the cached verdict for the caller's retry to settle. + if (entry.snapshot) { + get().applyRuntimeHostStatusSnapshot(entry.snapshot) return } // Why: setRuntimeEnvironmentStatus drops any stale compat failure on a non-null diff --git a/src/renderer/src/web/preload-api/web-runtime-environments-api.ts b/src/renderer/src/web/preload-api/web-runtime-environments-api.ts index e6ce2fd5a5d..8e9b0ecc245 100644 --- a/src/renderer/src/web/preload-api/web-runtime-environments-api.ts +++ b/src/renderer/src/web/preload-api/web-runtime-environments-api.ts @@ -16,6 +16,9 @@ import { translateHostAccessLinkError } from '@/lib/remote-pairing-copy' import { callEnvironmentEnvelope } from './web-runtime-calls' import { closeActiveRuntimeClients, + subscribeWebRuntimeStatus, + readWebRuntimeStatusSnapshots, + observeWebRuntimeStatus, disconnectActiveRuntimeEnvironment, getClientForEnvironment, manuallyDisconnectedEnvironmentIds, @@ -29,6 +32,8 @@ export function createRuntimeEnvironmentsApi(): NonNullable< Partial['runtimeEnvironments'] > { return { + onStatusChanged: subscribeWebRuntimeStatus, + getStatusSnapshots: async () => readWebRuntimeStatusSnapshots(), list: async () => { const environment = requireActiveEnvironmentOrNull() return environment ? [redactStoredWebRuntimeEnvironment(environment)] : [] @@ -146,6 +151,12 @@ export function createRuntimeEnvironmentsApi(): NonNullable< manuallyDisconnectedEnvironmentIds.clear() closeActiveRuntimeClients() webRuntimeState.activeEnvironment = nextEnvironment + getClientForEnvironment(nextEnvironment).statusOwner?.acceptVerified({ + id: 'status.get', + ok: true, + result: runtimeStatus, + _meta: { runtimeId: runtimeStatus.runtimeId } + }) return { ok: true, environment: redactStoredWebRuntimeEnvironment(nextEnvironment), @@ -173,6 +184,7 @@ export function createRuntimeEnvironmentsApi(): NonNullable< connect: ({ selector, timeoutMs }) => { const environment = resolveEnvironment(selector) manuallyDisconnectedEnvironmentIds.delete(environment.id) + closeActiveRuntimeClients() return callEnvironmentEnvelope( environment.id, 'status.get', @@ -180,8 +192,10 @@ export function createRuntimeEnvironmentsApi(): NonNullable< timeoutMs ) }, - getStatus: ({ selector, timeoutMs }) => - callEnvironmentEnvelope(selector, 'status.get', undefined, timeoutMs), + getStatus: ({ selector, timeoutMs, observeOnly }) => + observeOnly + ? observeWebRuntimeStatus(selector, timeoutMs) + : callEnvironmentEnvelope(selector, 'status.get', undefined, timeoutMs), retryControlConnection: () => Promise.resolve(), prepareBrowserClientHostPlacement: async () => ({ kind: 'server' }), call: ({ selector, method, params, timeoutMs }) => diff --git a/src/renderer/src/web/preload-api/web-runtime-session.ts b/src/renderer/src/web/preload-api/web-runtime-session.ts index 17a19136002..15b6cb63ee7 100644 --- a/src/renderer/src/web/preload-api/web-runtime-session.ts +++ b/src/renderer/src/web/preload-api/web-runtime-session.ts @@ -1,3 +1,7 @@ +import type { + RuntimeHostStatusSnapshot, + RuntimeHostStatusResponse +} from '../../../../shared/runtime-host-status' import type { WorktreeVisibilityDefaults } from '../../../../shared/global-settings-types' import { RuntimeRpcCallQueuePool } from '../../../../shared/runtime-rpc-call-queue' import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope' @@ -30,6 +34,43 @@ export const webRuntimeState: { cachedDetectedWorktrees: null } +const statusListeners = new Set<(snapshot: RuntimeHostStatusSnapshot) => void>() +export function subscribeWebRuntimeStatus( + callback: (snapshot: RuntimeHostStatusSnapshot) => void +): () => void { + statusListeners.add(callback) + return () => { + statusListeners.delete(callback) + } +} +export function readWebRuntimeStatusSnapshots(): RuntimeHostStatusSnapshot[] { + const snapshot = webRuntimeState.activeClient?.statusOwner?.read() + return snapshot ? [snapshot] : [] +} +export async function observeWebRuntimeStatus( + selector: string, + timeoutMs?: number +): Promise { + const environment = resolveEnvironment(selector) + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return manuallyDisconnectedResponse(environment) + } + const existing = webRuntimeState.activeClient?.statusOwner + if (existing) { + return existing.refresh({ timeoutMs, observeOnly: true }) + } + const transient = new WebRuntimeClient(getPreferredWebPairingOffer(environment), { + reconnect: false + }) + try { + return (await transient.call('status.get', undefined, { + timeoutMs + })) as RuntimeHostStatusResponse + } finally { + transient.close() + } +} + export const manuallyDisconnectedEnvironmentIds = new Set() export const runtimeCallQueuePool = new RuntimeRpcCallQueuePool() @@ -50,7 +91,18 @@ export function getClientForEnvironment( webRuntimeState.activeClientEnvironmentId !== environment.id ) { webRuntimeState.activeClient?.close() - webRuntimeState.activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment)) + webRuntimeState.activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment), { + status: { + environmentId: environment.id, + pairingRevision: environment.pairingRevision ?? environment.createdAt, + publish: (snapshot) => { + for (const listener of statusListeners) { + listener(snapshot) + } + }, + verified: (response) => updateEnvironmentFromResponse(environment, response) + } + }) webRuntimeState.activeClientEnvironmentId = environment.id } return webRuntimeState.activeClient diff --git a/src/renderer/src/web/web-runtime-client-export-parity.test.ts b/src/renderer/src/web/web-runtime-client-export-parity.test.ts index 27dce452549..6ba87eaad36 100644 --- a/src/renderer/src/web/web-runtime-client-export-parity.test.ts +++ b/src/renderer/src/web/web-runtime-client-export-parity.test.ts @@ -6,8 +6,13 @@ it('keeps the paired-web client public export surface exact', () => { expectTypeOf().toEqualTypeOf() expectTypeOf().toEqualTypeOf() expectTypeOf>().toEqualTypeOf< - [pairing: WebPairingOffer] + [ + pairing: WebPairingOffer, + options?: ConstructorParameters[1] + ] + >() + expectTypeOf().toEqualTypeOf< + 'call' | 'close' | 'subscribe' | 'statusOwner' >() - expectTypeOf().toEqualTypeOf<'call' | 'close' | 'subscribe'>() expect(Object.keys(WebClient)).toEqual(['WebRuntimeClient']) }) diff --git a/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts b/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts index 71568adc4da..95417748271 100644 --- a/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts +++ b/src/renderer/src/web/web-runtime-client-timeout-budget.test.ts @@ -70,7 +70,7 @@ describe('WebRuntimeClient timeout budget', () => { await vi.advanceTimersByTimeAsync(60_000) expect(settled).toBe(false) - expect(waitForConnected).toHaveBeenCalledWith(25) + expect(waitForConnected).toHaveBeenCalledWith(25, undefined) resolveConnection() await Promise.resolve() diff --git a/src/renderer/src/web/web-runtime-client.ts b/src/renderer/src/web/web-runtime-client.ts index 7bc756ca0c0..8a3b944f701 100644 --- a/src/renderer/src/web/web-runtime-client.ts +++ b/src/renderer/src/web/web-runtime-client.ts @@ -1,3 +1,8 @@ +import { RuntimeHostStatusOwner } from '../../../shared/runtime-host-status-owner' +import type { + RuntimeHostStatusSnapshot, + RuntimeHostStatusResponse +} from '../../../shared/runtime-host-status' import type { RuntimeRpcResponse } from '../../../shared/runtime-rpc-envelope' import { WebRuntimeConnectionTransport } from './web-runtime-connection-transport' import { subscribeWebRuntimeFileWatch } from './web-runtime-file-watch-subscription' @@ -24,11 +29,62 @@ export class WebRuntimeClient { private readonly fileWatchTeardownRetries = new Map Promise>>() private readonly childClients = new Set() - constructor(private readonly pairing: WebPairingOffer) { - this.transport = new WebRuntimeConnectionTransport(pairing, { - now: () => this.now(), - isDocumentVisible: () => this.isDocumentVisible() - }) + readonly statusOwner?: RuntimeHostStatusOwner + + constructor( + private readonly pairing: WebPairingOffer, + options: { + reconnect?: boolean + status?: { + environmentId: string + pairingRevision: number + publish: (snapshot: RuntimeHostStatusSnapshot) => void + verified: (response: RuntimeHostStatusResponse) => void + } + } = {} + ) { + this.transport = new WebRuntimeConnectionTransport( + pairing, + { + now: () => this.now(), + isDocumentVisible: () => this.isDocumentVisible() + }, + { + reconnect: options.reconnect, + onStateChanged: (state) => { + if (state === 'auth-failed') { + this.statusOwner?.authenticationRejected() + } + this.statusOwner?.connectionChanged( + state === 'connected' + ? 'ready' + : state === 'disconnected' || state === 'auth-failed' + ? 'disconnected' + : 'connecting' + ) + } + } + ) + if (options.status) { + const status = options.status + this.statusOwner = new RuntimeHostStatusOwner({ + ...status, + persistent: true, + request: (signal) => + this.transport.call('status.get', undefined, { + timeoutMs: 15_000, + signal + }) as Promise, + verified: (response) => { + status.verified(response) + return true + } + }) + this.statusOwner.connectionChanged( + this.transport.state === 'connected' ? 'ready' : 'connecting' + ) + this.statusOwner.activate() + } } call( @@ -36,7 +92,9 @@ export class WebRuntimeClient { params?: unknown, options?: { timeoutMs?: number } ): Promise> { - return this.transport.call(method, params, options) + return method === 'status.get' && this.statusOwner + ? this.statusOwner.refresh(options) + : this.transport.call(method, params, options) } async subscribe( @@ -94,6 +152,7 @@ export class WebRuntimeClient { } close(options: { notifySubscriptions?: boolean } = {}): void { + this.statusOwner?.dispose() const shouldNotifySubscriptions = options.notifySubscriptions ?? true for (const child of Array.from(this.childClients)) { child.close({ notifySubscriptions: shouldNotifySubscriptions }) diff --git a/src/renderer/src/web/web-runtime-connection-transport.ts b/src/renderer/src/web/web-runtime-connection-transport.ts index d62cf3b48ba..1ead257bbc5 100644 --- a/src/renderer/src/web/web-runtime-connection-transport.ts +++ b/src/renderer/src/web/web-runtime-connection-transport.ts @@ -43,7 +43,11 @@ export class WebRuntimeConnectionTransport { constructor( private readonly pairing: WebPairingOffer, - clock: { now: () => number; isDocumentVisible: () => boolean } + clock: { now: () => number; isDocumentVisible: () => boolean }, + private readonly lifecycle: { + onStateChanged?: (state: WebRuntimeConnectionState) => void + reconnect?: boolean + } = {} ) { this.serverPublicKey = publicKeyFromBase64(pairing.publicKeyB64) this.connectionWaiters = new WebRuntimeConnectionWaiters({ @@ -60,7 +64,7 @@ export class WebRuntimeConnectionTransport { this.requestRegistry = new WebRuntimeRequestRegistry({ deviceToken: pairing.deviceToken, nextId: () => this.nextId(), - waitForConnected: (timeoutMs) => this.connectionWaiters.wait(timeoutMs), + waitForConnected: (timeoutMs, signal) => this.connectionWaiters.wait(timeoutMs, signal), sendEncrypted: (message) => this.sendEncrypted(message) }) this.heartbeat = new WebRuntimeConnectionHeartbeat({ @@ -82,7 +86,7 @@ export class WebRuntimeConnectionTransport { async call( method: string, params?: unknown, - options?: { timeoutMs?: number } + options?: { timeoutMs?: number; signal?: AbortSignal } ): Promise> { return this.requestRegistry.call(method, params, options) } @@ -153,6 +157,7 @@ export class WebRuntimeConnectionTransport { } else if (next === 'auth-failed') { this.connectionWaiters.rejectAll(createWebRuntimeUnauthorizedError()) } + this.lifecycle.onStateChanged?.(next) } private openConnection(): void { @@ -232,7 +237,7 @@ export class WebRuntimeConnectionTransport { } private scheduleReconnect(): void { - if (this.reconnectTimer || this.intentionallyClosed) { + if (this.reconnectTimer || this.intentionallyClosed || this.lifecycle.reconnect === false) { return } const delay = withReconnectJitter( diff --git a/src/renderer/src/web/web-runtime-connection-waiters.ts b/src/renderer/src/web/web-runtime-connection-waiters.ts index c16e30cd779..a8f3d081e86 100644 --- a/src/renderer/src/web/web-runtime-connection-waiters.ts +++ b/src/renderer/src/web/web-runtime-connection-waiters.ts @@ -13,7 +13,10 @@ export class WebRuntimeConnectionWaiters { constructor(private readonly options: WebRuntimeConnectionWaiterOptions) {} - wait(timeoutMs = 30_000): Promise { + wait(timeoutMs = 30_000, signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(signal.reason) + } if (this.options.getState() === 'connected') { return Promise.resolve() } @@ -24,11 +27,20 @@ export class WebRuntimeConnectionWaiters { return Promise.reject(new Error('Remote Orca runtime connection closed.')) } return new Promise((resolve, reject) => { - const timeout = window.setTimeout(() => { - const index = this.waiters.findIndex((waiter) => waiter.resolve === resolve) + const cleanup = (): void => { + window.clearTimeout(timeout) + signal?.removeEventListener('abort', abort) + const index = this.waiters.indexOf(waiter) if (index !== -1) { this.waiters.splice(index, 1) } + } + const abort = (): void => { + cleanup() + reject(signal?.reason) + } + const timeout = window.setTimeout(() => { + cleanup() reject( new Error( withRemoteRuntimeTailscaleHint( @@ -38,16 +50,18 @@ export class WebRuntimeConnectionWaiters { ) ) }, timeoutMs) - this.waiters.push({ + const waiter = { resolve: () => { - window.clearTimeout(timeout) + cleanup() resolve() }, - reject: (error) => { - window.clearTimeout(timeout) + reject: (error: Error) => { + cleanup() reject(error) } - }) + } + this.waiters.push(waiter) + signal?.addEventListener('abort', abort, { once: true }) }) } diff --git a/src/renderer/src/web/web-runtime-request-registry.ts b/src/renderer/src/web/web-runtime-request-registry.ts index 1347d580f00..0329e208ba3 100644 --- a/src/renderer/src/web/web-runtime-request-registry.ts +++ b/src/renderer/src/web/web-runtime-request-registry.ts @@ -6,7 +6,7 @@ const REQUEST_TIMEOUT_MS = 30_000 type WebRuntimeRequestRegistryOptions = { deviceToken: string nextId: () => string - waitForConnected: (timeoutMs?: number) => Promise + waitForConnected: (timeoutMs?: number, signal?: AbortSignal) => Promise sendEncrypted: (message: unknown) => boolean } @@ -18,17 +18,41 @@ export class WebRuntimeRequestRegistry { async call( method: string, params?: unknown, - callOptions?: { timeoutMs?: number } + callOptions?: { timeoutMs?: number; signal?: AbortSignal } ): Promise> { - await this.options.waitForConnected(callOptions?.timeoutMs) + const signal = callOptions?.signal + await this.options.waitForConnected(callOptions?.timeoutMs, signal) + signal?.throwIfAborted() return new Promise((resolve, reject) => { const id = this.options.nextId() const timeoutMs = callOptions?.timeoutMs ?? REQUEST_TIMEOUT_MS const timeout = window.setTimeout(() => { this.pending.delete(id) + cleanup() reject(new Error(`Request timed out: ${method}`)) }, timeoutMs) - this.pending.set(id, { method, resolve, reject, timeout }) + const cleanup = (): void => { + signal?.removeEventListener('abort', abort) + } + const abort = (): void => { + this.pending.delete(id) + window.clearTimeout(timeout) + cleanup() + reject(signal?.reason) + } + signal?.addEventListener('abort', abort, { once: true }) + this.pending.set(id, { + method, + resolve: (value) => { + cleanup() + resolve(value) + }, + reject: (error) => { + cleanup() + reject(error) + }, + timeout + }) if ( !this.options.sendEncrypted({ id, @@ -39,6 +63,7 @@ export class WebRuntimeRequestRegistry { ) { this.pending.delete(id) window.clearTimeout(timeout) + cleanup() reject(new Error('Remote Orca runtime is not connected.')) } }) diff --git a/src/renderer/src/web/web-runtime-status-owner.test.ts b/src/renderer/src/web/web-runtime-status-owner.test.ts new file mode 100644 index 00000000000..7abfe6f19f4 --- /dev/null +++ b/src/renderer/src/web/web-runtime-status-owner.test.ts @@ -0,0 +1,49 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import WebSocket from 'ws' +import { + createSharedControlTestServer, + closeSharedControlTestServers +} from '../../../shared/remote-runtime-shared-control-test-server' +import { WebRuntimeClient } from './web-runtime-client' + +const clients: WebRuntimeClient[] = [] +beforeEach(() => { + vi.stubGlobal('WebSocket', WebSocket) + vi.stubGlobal('window', { + setTimeout, + clearTimeout, + setInterval, + clearInterval, + atob: (value: string) => Buffer.from(value, 'base64').toString('binary'), + btoa: (value: string) => Buffer.from(value, 'binary').toString('base64') + }) +}) +afterEach(async () => { + clients.splice(0).forEach((client) => client.close()) + await closeSharedControlTestServers() + vi.unstubAllGlobals() +}) + +it('primary browser status follows the authenticated socket and closing it retires the owner', async () => { + let runtimeId = 'before' + const server = await createSharedControlTestServer({ + resultForRequest: () => ({ runtimeId, capabilities: [] }) + }) + const publish = vi.fn() + const client = new WebRuntimeClient(server.pairing, { + status: { environmentId: 'browser', pairingRevision: 1, publish, verified: vi.fn() } + }) + clients.push(client) + await expect + .poll(() => client.statusOwner?.read().verification, { timeout: 3_000 }) + .toBe('verified') + expect(client.statusOwner?.read().status?.runtimeId).toBe('before') + runtimeId = 'after' + server.closeClients() + await expect + .poll(() => client.statusOwner?.read().status?.runtimeId, { timeout: 3_000 }) + .toBe('after') + expect(client.statusOwner?.read().transport).toBe('ready') + client.close() + expect(publish.mock.lastCall?.[0]).toMatchObject({ retired: true, verification: 'blocked' }) +}) diff --git a/src/shared/execution-host-registry.test.ts b/src/shared/execution-host-registry.test.ts index e509fe05bfc..e397e8ed5bf 100644 --- a/src/shared/execution-host-registry.test.ts +++ b/src/shared/execution-host-registry.test.ts @@ -320,17 +320,15 @@ describe('execution host registry', () => { ]) }) - it('includes runtime hosts from repo ownership but marks them disconnected without live status', () => { + it('keeps runtime hosts checking before their first status result', () => { const hosts = buildExecutionHostRegistry({ repos: [{ connectionId: null, executionHostId: 'runtime:env-2' }], settings: { activeRuntimeEnvironmentId: null } }) - // No live status means no evidence the Orca server is reachable, so it must - // read 'disconnected' rather than defaulting to 'available'/"Connected". expect(hosts).toMatchObject([ { id: 'local', health: 'local' }, - { id: 'runtime:env-2', kind: 'runtime', label: 'env-2', health: 'disconnected' } + { id: 'runtime:env-2', kind: 'runtime', label: 'env-2', health: 'connecting' } ]) }) @@ -373,3 +371,29 @@ describe('execution host registry', () => { ]) }) }) + +it('keeps an initial unknown-transport verification connecting', () => { + const hosts = buildExecutionHostRegistry({ + repos: [], + settings: null, + runtimeEnvironments: [{ id: 'host', name: 'Host' }], + runtimeStatusByEnvironmentId: new Map([ + [ + 'host', + { + status: null, + snapshot: { + environmentId: 'host', + pairingRevision: 1, + sequence: 1, + checkedAt: 0, + status: null, + verification: 'checking', + transport: 'unknown' + } + } + ] + ]) + }) + expect(hosts.find((host) => host.id === 'runtime:host')?.health).toBe('connecting') +}) diff --git a/src/shared/execution-host-registry.ts b/src/shared/execution-host-registry.ts index a970f9da45c..bad6b40f257 100644 --- a/src/shared/execution-host-registry.ts +++ b/src/shared/execution-host-registry.ts @@ -1,3 +1,4 @@ +import type { RuntimeHostStatusSnapshot } from './runtime-host-status' import { LOCAL_EXECUTION_HOST_ID, getLocalExecutionHostLabel, @@ -49,6 +50,7 @@ type RuntimeEnvironmentSummary = { } type RuntimeHostStatus = { + snapshot?: RuntimeHostStatusSnapshot status?: RuntimeStatus | null remoteControl?: RuntimeStatus['remoteControl'] | null appVersion?: string | null @@ -158,9 +160,24 @@ function addRuntimeHost( const hostId = toRuntimeExecutionHostId(environmentId) const runtimeStatus = statusByEnvironmentId?.get(environmentId) const status = runtimeStatus?.status - const compatibility = runtimeCompatibility(status) + const snapshot = runtimeStatus?.snapshot + const metadata = status ?? snapshot?.status + const compatibility = runtimeCompatibility(metadata) const remoteControl = runtimeStatus?.remoteControl ?? status?.remoteControl - const controlHealth = runtimeControlHealth(remoteControl) + const controlHealth = snapshot?.retired + ? 'disconnected' + : snapshot?.verification === 'blocked' + ? 'blocked' + : !runtimeStatus || + snapshot?.verification === 'checking' || + snapshot?.transport === 'disconnected' || + snapshot?.transport === 'connecting' + ? 'connecting' + : snapshot?.transport === 'ready' + ? compatibility?.kind === 'blocked' + ? 'blocked' + : 'available' + : runtimeControlHealth(remoteControl) setHost(hosts, { id: hostId, kind: 'runtime', @@ -168,12 +185,12 @@ function addRuntimeHost( detail: 'Orca server', health: controlHealth ?? runtimeHealth(status, compatibility, remoteControl), compatibility: compatibility ?? undefined, - capabilities: status?.capabilities, - appVersion: runtimeStatus?.appVersion ?? status?.appVersion ?? null, - protocolVersion: status?.runtimeProtocolVersion ?? status?.protocolVersion ?? null, + capabilities: metadata?.capabilities, + appVersion: runtimeStatus?.appVersion ?? metadata?.appVersion ?? null, + protocolVersion: metadata?.runtimeProtocolVersion ?? metadata?.protocolVersion ?? null, minCompatibleClientVersion: - status?.minCompatibleRuntimeClientVersion ?? status?.minCompatibleMobileVersion ?? null, - platform: status?.hostPlatform ?? null, + metadata?.minCompatibleRuntimeClientVersion ?? metadata?.minCompatibleMobileVersion ?? null, + platform: metadata?.hostPlatform ?? null, remoteControlState: remoteControl ?? null, ...(source ? { source } : {}) }) diff --git a/src/shared/remote-runtime-shared-control-test-server.ts b/src/shared/remote-runtime-shared-control-test-server.ts index 0e4adb1a69c..701ea0e5893 100644 --- a/src/shared/remote-runtime-shared-control-test-server.ts +++ b/src/shared/remote-runtime-shared-control-test-server.ts @@ -20,6 +20,7 @@ export type SharedControlTestServer = { } type ServerOptions = { + resultForRequest?: (method: string) => unknown delaySubscriptionReady?: boolean sendKeepaliveBeforeResponse?: boolean keepaliveDelayMs?: number @@ -174,7 +175,7 @@ function handleRequest( const streaming = isStreamingMethod(request.method) const result = streaming ? { type: 'ready', subscriptionId: `${request.method}:subscription` } - : { method: request.method } + : (options.resultForRequest?.(request.method) ?? { method: request.method }) const sendResponse = (): void => { if (options.sendUnknownResponseBeforeResponse) { sendEncrypted(ws, sharedKey, { diff --git a/src/shared/runtime-host-status-owner.test.ts b/src/shared/runtime-host-status-owner.test.ts new file mode 100644 index 00000000000..32331991988 --- /dev/null +++ b/src/shared/runtime-host-status-owner.test.ts @@ -0,0 +1,207 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { RuntimeHostStatusOwner } from './runtime-host-status-owner' +import { runtimeHostStatusFailure, type RuntimeHostStatusResponse } from './runtime-host-status' +import type { RuntimeStatus } from './runtime-types' + +const owners: RuntimeHostStatusOwner[] = [] +beforeEach(() => vi.useFakeTimers()) +afterEach(() => { + owners.splice(0).forEach((owner) => owner.dispose()) + vi.useRealTimers() +}) +function success(runtimeId = 'host-1'): RuntimeHostStatusResponse & { ok: true } { + return { + id: 'status', + ok: true, + result: { runtimeId, capabilities: [] } as unknown as RuntimeStatus, + _meta: { runtimeId } + } +} +function deferred() { + let resolve!: (response: RuntimeHostStatusResponse) => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} +function createOwner(persistent = false) { + const request = vi + .fn<(signal: AbortSignal) => Promise>() + .mockResolvedValue(success()) + const publish = vi.fn() + const verified = vi.fn((_response: RuntimeHostStatusResponse, _active: boolean) => persistent) + const owner = new RuntimeHostStatusOwner({ + environmentId: 'env-a', + pairingRevision: 1, + persistent, + request, + publish, + verified + }) + owners.push(owner) + return { owner, request, publish, verified } +} + +it('shares one verification between viewers with independent deadlines', async () => { + const { owner, request } = createOwner() + const pending = deferred() + request.mockReturnValue(pending.promise) + const impatient = owner.refresh({ timeoutMs: 100 }) + const patient = owner.refresh({ timeoutMs: 1_000 }) + await vi.advanceTimersByTimeAsync(100) + expect((await impatient).ok).toBe(false) + expect(request).toHaveBeenCalledOnce() + expect(request.mock.calls[0][0].aborted).toBe(false) + pending.resolve(success()) + expect((await patient).ok).toBe(true) +}) + +it('uses ready transitions, not diagnostic updates or a healthy polling timer', async () => { + const { owner, request } = createOwner(true) + await owner.refresh() + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(0) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledTimes(2) + owner.connectionChanged('disconnected') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledTimes(2) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(3) +}) + +it('retries a failed status operation while retaining healthy transport and last good metadata', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + await owner.refresh() + request.mockResolvedValueOnce(runtimeHostStatusFailure('runtime_unavailable', 'status timed out')) + await owner.refresh() + expect(owner.read()).toMatchObject({ + transport: 'ready', + verification: 'unavailable', + status: { runtimeId: 'host-1' } + }) + await vi.advanceTimersByTimeAsync(3_000) + expect(owner.read().verification).toBe('verified') + expect(request).toHaveBeenCalledTimes(3) +}) + +it('retires a lost-socket request before explicit fallback and rejects its late result', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + const old = deferred() + request.mockReturnValueOnce(old.promise) + const waiting = owner.refresh() + owner.connectionChanged('disconnected') + expect(request.mock.calls[0][0].aborted).toBe(true) + request.mockResolvedValueOnce(success('fallback-host')) + expect((await owner.refresh()).ok).toBe(true) + expect((await waiting).ok).toBe(true) + old.resolve(success('obsolete-host')) + await vi.advanceTimersByTimeAsync(0) + expect(owner.read().status?.runtimeId).toBe('fallback-host') + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(3) +}) + +it('a reconnect transfers waiting readers to a fresh verification', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + const old = deferred() + request.mockReturnValueOnce(old.promise) + const waiting = owner.refresh() + owner.connectionChanged('disconnected') + owner.connectionChanged('ready') + expect((await waiting).ok).toBe(true) + old.resolve(success('old')) + await vi.advanceTimersByTimeAsync(0) + expect(owner.read().status?.runtimeId).toBe('host-1') +}) + +it('disconnect settles readers and prevents late results and retry resurrection', async () => { + const { owner, request, publish } = createOwner() + const old = deferred() + request.mockReturnValue(old.promise) + const waiting = owner.refresh() + owner.dispose() + expect((await waiting).ok).toBe(false) + const sequence = owner.read().sequence + old.resolve(success()) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(owner.read()).toMatchObject({ retired: true, sequence }) + expect(publish.mock.lastCall?.[0].retired).toBe(true) + expect(request).toHaveBeenCalledOnce() +}) + +it('passive reads create neither standing retries nor connection intent', async () => { + const { owner, request, verified } = createOwner() + request.mockResolvedValueOnce(runtimeHostStatusFailure('runtime_unavailable', 'offline')) + await owner.refresh({ observeOnly: true }) + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledOnce() + await owner.refresh({ observeOnly: true }) + expect(verified.mock.lastCall?.[1]).toBe(false) +}) + +it('authentication rejection blocks automatic verification until explicit reconnect', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + request.mockResolvedValueOnce(runtimeHostStatusFailure('unauthorized', 're-pair')) + await owner.refresh() + owner.connectionChanged('disconnected') + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledOnce() + expect((await owner.refresh({ reconnect: true })).ok).toBe(true) +}) + +it('blocks a rejected reconnect even without an outstanding status request', async () => { + const { owner, request } = createOwner(true) + owner.connectionChanged('ready') + await owner.refresh() + owner.connectionChanged('disconnected') + owner.authenticationRejected() + expect(owner.read()).toMatchObject({ verification: 'blocked', status: { runtimeId: 'host-1' } }) + owner.connectionChanged('ready') + await vi.advanceTimersByTimeAsync(300_000) + expect(request).toHaveBeenCalledOnce() +}) + +it('cancelling one reader leaves the shared request available to other readers', async () => { + const { owner, request } = createOwner() + const pending = deferred() + request.mockReturnValue(pending.promise) + const controller = new AbortController() + const cancelled = owner.refresh({ signal: controller.signal }) + const remaining = owner.refresh() + const rejection = expect(cancelled).rejects.toThrow('cancelled') + controller.abort(new Error('cancelled')) + await rejection + expect(request.mock.calls[0][0].aborted).toBe(false) + pending.resolve(success()) + expect((await remaining).ok).toBe(true) +}) + +it.each(['unknown', 'ready'] as const)( + 'distinguishes the caller deadline with %s transport', + async (transport) => { + const { owner, request } = createOwner() + owner.connectionChanged(transport) + request.mockReturnValue(deferred().promise) + const response = owner.refresh({ timeoutMs: 100 }) + await vi.advanceTimersByTimeAsync(100) + expect(await response).toMatchObject({ + ok: false, + error: { + message: + transport === 'ready' + ? 'Status request timed out.' + : 'Timed out waiting for the remote Orca runtime.' + } + }) + } +) diff --git a/src/shared/runtime-host-status-owner.ts b/src/shared/runtime-host-status-owner.ts new file mode 100644 index 00000000000..d9c52395c89 --- /dev/null +++ b/src/shared/runtime-host-status-owner.ts @@ -0,0 +1,274 @@ +import { + isRuntimeHostStatusBlocked, + runtimeHostStatusError, + runtimeHostStatusFailure, + type RuntimeHostStatusResponse, + type RuntimeHostStatusSnapshot +} from './runtime-host-status' + +const RETRY_DELAYS_MS = [3_000, 6_000, 12_000, 30_000, 60_000] +const REQUEST_TIMEOUT_MS = 15_000 +let publicationSequence = 0 + +type Waiter = { + resolve: (response: RuntimeHostStatusResponse) => void + cleanup: () => void +} + +type StatusOwnerOptions = { + environmentId: string + pairingRevision: number + persistent?: boolean + request: (signal: AbortSignal) => Promise + verified: (response: Extract, active: boolean) => boolean + publish: (snapshot: RuntimeHostStatusSnapshot) => void +} + +/** One verification and one retry slot, shared by all readers of this connection. */ +export class RuntimeHostStatusOwner { + private active = false + private disposed = false + private persistent: boolean + private attempt = 0 + private retry: ReturnType | null = null + private request: AbortController | null = null + private readonly waiters = new Set() + private response: RuntimeHostStatusResponse = runtimeHostStatusFailure( + 'runtime_unavailable', + 'Status has not been checked.' + ) + private snapshot: RuntimeHostStatusSnapshot + + constructor(private readonly options: StatusOwnerOptions) { + this.persistent = options.persistent ?? false + this.snapshot = { + environmentId: options.environmentId, + pairingRevision: options.pairingRevision, + sequence: ++publicationSequence, + checkedAt: 0, + status: null, + verification: 'checking', + transport: 'unknown' + } + } + + read(): RuntimeHostStatusSnapshot { + return this.snapshot + } + + activate(): void { + if (this.active || this.disposed) { + return + } + this.active = true + this.startRequest() + } + + acceptVerified(response: Extract): void { + if (this.disposed) { + return + } + this.active = true + this.retireRequest() + this.clearRetry() + this.complete(response) + } + + refresh( + options: { timeoutMs?: number; observeOnly?: true; reconnect?: true; signal?: AbortSignal } = {} + ): Promise { + if (options.signal?.aborted) { + return Promise.reject(options.signal.reason) + } + if (this.disposed) { + return Promise.resolve(this.response) + } + if (!options.observeOnly) { + this.active = true + } + if (options.reconnect) { + this.attempt = 0 + this.update({ verification: 'checking' }) + } + if (this.snapshot.verification === 'blocked') { + return Promise.resolve(this.response) + } + const result = new Promise((resolve, reject) => { + const release = (): void => { + waiter.cleanup() + this.waiters.delete(waiter) + if (!this.active && this.waiters.size === 0) { + this.retireRequest() + } + } + const abort = (): void => { + release() + reject(options.signal?.reason) + } + const timer = setTimeout(() => { + release() + resolve( + runtimeHostStatusFailure( + 'runtime_unavailable', + this.snapshot.transport === 'ready' + ? 'Status request timed out.' + : 'Timed out waiting for the remote Orca runtime.' + ) + ) + }, options.timeoutMs ?? REQUEST_TIMEOUT_MS) + const waiter: Waiter = { + resolve, + cleanup: () => { + clearTimeout(timer) + options.signal?.removeEventListener('abort', abort) + } + } + this.waiters.add(waiter) + options.signal?.addEventListener('abort', abort, { once: true }) + }) + this.startRequest() + return result + } + + connectionChanged( + transport: RuntimeHostStatusSnapshot['transport'], + remoteControl?: RuntimeHostStatusSnapshot['remoteControl'] + ): void { + if (this.disposed) { + return + } + const previous = this.snapshot.transport + this.update({ transport, ...(remoteControl !== undefined ? { remoteControl } : {}) }) + if (transport === previous) { + return + } + if (previous === 'ready') { + this.retireRequest() + this.clearRetry() + if (this.snapshot.verification !== 'blocked') { + this.update({ verification: 'unavailable' }) + } + } + if (transport === 'ready' && this.snapshot.verification !== 'blocked') { + // A pre-reconnect answer cannot verify the new socket's runtime. + this.retireRequest() + if (this.active || this.waiters.size > 0) { + this.startRequest() + } + } + } + + authenticationRejected(): void { + if (this.disposed) { + return + } + this.retireRequest() + this.clearRetry() + this.complete(runtimeHostStatusFailure('unauthorized', 'Pair this client again.')) + } + + dispose(): void { + if (this.disposed) { + return + } + this.disposed = true + this.active = false + this.retireRequest() + this.clearRetry() + this.response = runtimeHostStatusFailure( + 'runtime_manually_disconnected', + 'Runtime environment was disconnected or replaced.' + ) + this.update({ retired: true, transport: 'disconnected', verification: 'blocked' }) + this.settleWaiters() + } + + private startRequest(): void { + if (this.disposed || this.request || this.snapshot.verification === 'blocked') { + return + } + this.clearRetry() + const controller = new AbortController() + this.request = controller + if (this.snapshot.verification !== 'verified') { + this.update({ verification: 'checking' }) + } + void this.verify(controller) + } + + private async verify(controller: AbortController): Promise { + let response: RuntimeHostStatusResponse + try { + response = await this.options.request(controller.signal) + } catch (error) { + response = runtimeHostStatusError(error) + if (error instanceof TypeError || error instanceof SyntaxError) { + console.error('Runtime status verification failed:', error) + response = runtimeHostStatusFailure('invalid_runtime_response', error.message) + } + } + if (this.request !== controller || this.disposed) { + return + } + this.request = null + this.complete(response) + } + + private complete(response: RuntimeHostStatusResponse): void { + this.response = response + if (response.ok) { + this.attempt = 0 + this.update({ status: response.result, checkedAt: Date.now(), verification: 'verified' }) + this.persistent = this.options.verified(response, this.active) + } else { + this.update({ + checkedAt: Date.now(), + verification: isRuntimeHostStatusBlocked(response) ? 'blocked' : 'unavailable' + }) + this.scheduleRetry() + } + this.settleWaiters() + } + + private scheduleRetry(): void { + if ( + !this.active || + this.disposed || + this.snapshot.verification === 'blocked' || + (this.persistent && this.snapshot.transport !== 'ready') + ) { + return + } + const delay = RETRY_DELAYS_MS[Math.min(this.attempt++, RETRY_DELAYS_MS.length - 1)] + this.retry = setTimeout(() => { + this.retry = null + this.startRequest() + }, delay) + } + + private settleWaiters(): void { + for (const waiter of this.waiters) { + waiter.cleanup() + waiter.resolve(this.response) + } + this.waiters.clear() + } + + private retireRequest(): void { + const request = this.request + this.request = null + request?.abort() + } + + private clearRetry(): void { + if (this.retry) { + clearTimeout(this.retry) + } + this.retry = null + } + + private update(patch: Partial): void { + this.snapshot = { ...this.snapshot, ...patch, sequence: ++publicationSequence } + this.options.publish(this.snapshot) + } +} diff --git a/src/shared/runtime-host-status.ts b/src/shared/runtime-host-status.ts new file mode 100644 index 00000000000..4dd7ff7cc22 --- /dev/null +++ b/src/shared/runtime-host-status.ts @@ -0,0 +1,44 @@ +import type { RemoteRuntimeSharedConnectionDiagnostics } from './remote-runtime-shared-control-types' +import type { RuntimeRpcFailure, RuntimeRpcResponse } from './runtime-rpc-envelope' +import type { RuntimeStatus } from './runtime-types' + +export const RUNTIME_HOST_STATUS_CHANNEL = 'runtimeEnvironments:statusChanged' + +/** Local client state; never exchanged with the paired host. */ +export type RuntimeHostStatusSnapshot = { + environmentId: string + pairingRevision: number + sequence: number + checkedAt: number + status: RuntimeStatus | null + verification: 'checking' | 'verified' | 'unavailable' | 'blocked' + transport: 'unknown' | 'connecting' | 'ready' | 'disconnected' + remoteControl?: RemoteRuntimeSharedConnectionDiagnostics | null + retired?: true +} + +export type RuntimeHostStatusResponse = RuntimeRpcResponse + +export function runtimeHostStatusFailure(code: string, message: string): RuntimeRpcFailure { + return { id: 'status.get', ok: false, error: { code, message } } +} + +export function runtimeHostStatusError(error: unknown): RuntimeRpcFailure { + const code = + error instanceof Error && 'code' in error && typeof error.code === 'string' + ? error.code + : 'runtime_unavailable' + return runtimeHostStatusFailure(code, error instanceof Error ? error.message : String(error)) +} + +export function isRuntimeHostStatusBlocked(response: RuntimeRpcFailure): boolean { + return [ + 'unauthorized', + 'forbidden', + 'invalid_argument', + 'invalid_runtime_response', + 'protocol_version_mismatch', + 'method_not_found', + 'unsupported_method' + ].includes(response.error.code) +} diff --git a/tests/e2e/runtime-host-status-recovery.spec.ts b/tests/e2e/runtime-host-status-recovery.spec.ts new file mode 100644 index 00000000000..09707606da1 --- /dev/null +++ b/tests/e2e/runtime-host-status-recovery.spec.ts @@ -0,0 +1,221 @@ +import { createConnection, createServer, type Socket, type AddressInfo } from 'node:net' +import type { Page } from '@stablyai/playwright-test' +import { decodePairingOffer, encodePairingOffer } from '../../src/shared/pairing' +import { expect, test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient, + launchPairedWebClient, + type RuntimeDesktopPairingOffer +} from './helpers/paired-electron-client' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' + +async function interruptibleHost(offer: RuntimeDesktopPairingOffer) { + const pairing = decodePairingOffer(offer.pairingUrl) + const endpoint = new URL(pairing.endpoint) + const sockets = new Set() + let online = true + const server = createServer((client) => { + if (!online) { + client.destroy() + return + } + const host = createConnection({ host: endpoint.hostname, port: Number(endpoint.port) }) + for (const socket of [client, host]) { + sockets.add(socket) + socket.on('error', () => { + client.destroy() + host.destroy() + }) + socket.on('close', () => { + sockets.delete(socket) + client.destroy() + host.destroy() + }) + } + client.pipe(host).pipe(client) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() as AddressInfo + const pairingUrl = encodePairingOffer({ ...pairing, endpoint: `ws://127.0.0.1:${address.port}` }) + let webClientUrl: string | undefined + if (offer.webClientUrl) { + const url = new URL(offer.webClientUrl) + url.search = '' + url.hash = new URLSearchParams({ pairing: pairingUrl }).toString() + webClientUrl = url.href + } + return { + offer: { pairingUrl, webClientUrl }, + setOnline(value: boolean) { + online = value + if (!online) { + sockets.forEach((socket) => socket.destroy()) + } + }, + async close() { + sockets.forEach((socket) => socket.destroy()) + await new Promise((resolve) => server.close(() => resolve())) + } + } +} + +async function statusEvidence(page: Page, environmentId?: string) { + return page.evaluate((id) => { + const entries = window.__store?.getState().runtimeStatusByEnvironmentId + const entry = id ? entries?.get(id) : entries?.values().next().value + return entry?.snapshot + ? { + verification: entry.snapshot.verification, + transport: entry.snapshot.transport, + runtimeId: entry.status?.runtimeId, + sequence: entry.snapshot.sequence + } + : null + }, environmentId) +} + +async function expectWorkspaceHostAppearance( + page: Page, + disconnected: boolean, + hostLabel?: string +) { + const cards = page.locator('[data-worktree-card-surface="true"]') + const card = ( + hostLabel ? cards.filter({ has: page.getByText(hostLabel, { exact: true }) }) : cards + ).first() + await expect(card).toBeVisible() + await expect(card).toHaveCSS('opacity', disconnected ? '0.6' : '1') + const icon = card.locator(disconnected ? 'svg.lucide-server-off' : 'svg.lucide-server').first() + await expect(icon).toBeVisible() + await expect( + card.locator(disconnected ? 'svg.lucide-server' : 'svg.lucide-server-off') + ).toHaveCount(0) + await expect(icon).toHaveClass(disconnected ? /text-destructive/ : /text-muted-foreground/) + await icon.hover() + await expect( + page.getByRole('tooltip', { name: disconnected ? /disconnected/i : /Project on/ }) + ).toBeVisible() + await page.mouse.move(900, 600) +} + +for (const topology of ['desktop', 'headless'] as const) { + test(`connection-owned status recovers with a ${topology} host and independent viewers`, async ({ + electronApp, + orcaPage: page, + testRepoPath + }, testInfo) => { + test.setTimeout(180_000) + let headless: Awaited> | null = null + let proxy: Awaited> | undefined + let client: Awaited> | undefined + let browser: Awaited> | undefined + try { + headless = + topology === 'headless' + ? await launchHeadlessPairedRuntimeHost({ pinnedServePort: true }) + : null + const offer = headless?.offer ?? (await createRuntimeDesktopPairingOffer(page)) + await (headless + ? headless.client.call('repo.add', { path: testRepoPath }) + : page.evaluate(async (path) => { + await window.api.repos.add({ path }) + await window.__store?.getState().fetchRepos() + }, testRepoPath)) + proxy = await interruptibleHost(offer) + client = await launchPairedElectronClient(offer, testInfo, 'Direct host') + proxy.setOnline(false) + const offlineId = await client.page.evaluate(async (pairingCode) => { + const { environment } = await window.api.runtimeEnvironments.addFromPairingCode({ + name: 'Recovering host', + pairingCode + }) + const store = window.__store!.getState() + store.setRuntimeEnvironments(await window.api.runtimeEnvironments.list()) + await store.refreshRuntimeEnvironmentStatus(environment.id, 1_000) + return environment.id + }, proxy.offer.pairingUrl) + await expect + .poll(() => statusEvidence(client!.page, offlineId)) + .toMatchObject({ verification: 'unavailable' }) + expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({ + verification: 'verified' + }) + proxy.setOnline(true) + await expect + .poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + const initial = await statusEvidence(client!.page, offlineId) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expect(client.page.getByText('Recovering host', { exact: true }).first()).toBeVisible() + await client.page.screenshot({ path: testInfo.outputPath(`${topology}-recovered.png`) }) + browser = await launchPairedWebClient(electronApp, proxy.offer) + await expect + .poll(() => statusEvidence(browser!.page), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + await expectWorkspaceHostAppearance(browser.page, false) + proxy.setOnline(false) + await expect + .poll(() => statusEvidence(client!.page, offlineId)) + .toMatchObject({ transport: 'disconnected' }) + await expect + .poll(() => statusEvidence(browser!.page), { timeout: 30_000 }) + .toMatchObject({ transport: 'disconnected' }) + expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({ + verification: 'verified', + transport: 'ready' + }) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expectWorkspaceHostAppearance(client.page, false, 'Direct host') + await expectWorkspaceHostAppearance(browser.page, false) + await client.page.screenshot({ + path: testInfo.outputPath(`${topology}-sidebar-reconnecting.png`) + }) + await browser.page.screenshot({ + path: testInfo.outputPath(`${topology}-browser-reconnecting.png`) + }) + proxy.setOnline(true) + await expect + .poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + await expect + .poll(() => statusEvidence(browser!.page), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + expect((await statusEvidence(client!.page, offlineId))!.sequence).toBeGreaterThan( + initial!.sequence + ) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expectWorkspaceHostAppearance(browser.page, false) + await browser.page.screenshot({ + path: testInfo.outputPath(`${topology}-browser-recovered.png`) + }) + await client.page.evaluate(async (selector) => { + await window.api.runtimeEnvironments.disconnect({ selector }) + }, offlineId) + await expect + .poll(() => statusEvidence(client!.page, offlineId)) + .toMatchObject({ verification: 'blocked', transport: 'disconnected' }) + await expectWorkspaceHostAppearance(client.page, true, 'Recovering host') + await expectWorkspaceHostAppearance(client.page, false, 'Direct host') + await client.page.screenshot({ + path: testInfo.outputPath(`${topology}-sidebar-disconnected.png`) + }) + await client.page.evaluate(async (selector) => { + await window.api.runtimeEnvironments.connect({ selector }) + }, offlineId) + await expect + .poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 }) + .toMatchObject({ verification: 'verified', transport: 'ready' }) + await expectWorkspaceHostAppearance(client.page, false, 'Recovering host') + await expectWorkspaceHostAppearance(browser.page, false) + await client.page.screenshot({ + path: testInfo.outputPath(`${topology}-sidebar-restored.png`) + }) + } finally { + await browser?.dispose() + await client?.dispose() + await proxy?.close() + await headless?.dispose() + } + }) +}