From 38c2d1dcb9dc079bba6f59bdaa179d6af0584f09 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Thu, 1 Oct 2026 15:53:24 -0700 Subject: [PATCH] feat(ssh): update, roll back, recover and stop a managed orcad server (#16741 T6-5 follow-up) (#24463) * feat(ssh): update, roll back, recover and stop a managed orcad server (#16741 T6-5 follow-up) Builds managed-server maintenance on T6-2's deploy, rollback and recovery and T6-4's journaled decommission. Each step reads a terminal census through the server's tunnel; orcad answers it through a new capability-gated orcad.terminalCensus RPC, and an older host or lost answer is unverifiable. Update defers over live or uncounted terminals and status reports the last deferral. A stop unlinks the server (deployment record, tunnel, SSH claim) only after a proven exit. Inert until the T6-6 settings UI. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(rpc): load the orcad terminal census lazily so the dispatcher does not pull in the xterm window polyfill Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: m4air Co-authored-by: Claude Opus 5.5 (1M context) --- .../daemon/daemon-provider-census.test.ts | 12 + src/main/daemon/daemon-provider-state.ts | 15 +- .../ipc/orcad-runtime-lifecycle-handlers.ts | 2 +- ...orcad-runtime-maintenance-handlers.test.ts | 87 +++++++ .../ipc/orcad-runtime-maintenance-handlers.ts | 83 +++++++ ...ntime-environment-connectivity-handlers.ts | 20 +- .../runtime-environment-handler-channels.ts | 5 + .../runtime-environment-removal-cleanup.ts | 27 +++ .../ipc/runtime-environments-pairing.test.ts | 10 + src/main/ipc/runtime-environments.ts | 6 + src/main/orcad/orcad-terminal-census.test.ts | 51 +++++ src/main/orcad/orcad-terminal-census.ts | 39 ++++ src/main/runtime/rpc/methods/index.ts | 2 + .../rpc/methods/orcad-terminal-census.ts | 18 ++ .../orcad-managed-lifecycle-test-fixture.ts | 103 +++++++++ src/main/ssh/orcad-managed-runtime-context.ts | 14 ++ .../ssh/orcad-managed-update-deferrals.ts | 22 ++ .../ssh/orcad-runtime-decommission.test.ts | 216 ++++++++++++++++++ src/main/ssh/orcad-runtime-decommission.ts | 202 ++++++++++++++++ src/main/ssh/orcad-runtime-deployment.test.ts | 11 +- src/main/ssh/orcad-runtime-deployment.ts | 14 +- src/main/ssh/orcad-runtime-lifecycle.ts | 6 + .../ssh/orcad-runtime-maintenance.test.ts | 197 ++++++++++++++++ src/main/ssh/orcad-runtime-maintenance.ts | 203 ++++++++++++++++ src/main/ssh/orcad-runtime-status.ts | 6 +- .../ssh/orcad-terminal-census-client.test.ts | 101 ++++++++ src/main/ssh/orcad-terminal-census-client.ts | 84 +++++++ src/shared/orcad-managed-runtime.ts | 61 ++++- src/shared/orcad-terminal-census.ts | 8 + src/shared/protocol-version.ts | 5 + .../rpc-params-catalog.generated.ts | 2 + ...me-environment-managed-orcad-store.test.ts | 35 ++- ...runtime-environment-managed-orcad-store.ts | 85 ++++++- 33 files changed, 1707 insertions(+), 45 deletions(-) create mode 100644 src/main/ipc/orcad-runtime-maintenance-handlers.test.ts create mode 100644 src/main/ipc/orcad-runtime-maintenance-handlers.ts create mode 100644 src/main/ipc/runtime-environment-removal-cleanup.ts create mode 100644 src/main/orcad/orcad-terminal-census.test.ts create mode 100644 src/main/orcad/orcad-terminal-census.ts create mode 100644 src/main/runtime/rpc/methods/orcad-terminal-census.ts create mode 100644 src/main/ssh/orcad-managed-lifecycle-test-fixture.ts create mode 100644 src/main/ssh/orcad-managed-update-deferrals.ts create mode 100644 src/main/ssh/orcad-runtime-decommission.test.ts create mode 100644 src/main/ssh/orcad-runtime-decommission.ts create mode 100644 src/main/ssh/orcad-runtime-maintenance.test.ts create mode 100644 src/main/ssh/orcad-runtime-maintenance.ts create mode 100644 src/main/ssh/orcad-terminal-census-client.test.ts create mode 100644 src/main/ssh/orcad-terminal-census-client.ts diff --git a/src/main/daemon/daemon-provider-census.test.ts b/src/main/daemon/daemon-provider-census.test.ts index 2b0d6f88ac8..30aade980fd 100644 --- a/src/main/daemon/daemon-provider-census.test.ts +++ b/src/main/daemon/daemon-provider-census.test.ts @@ -7,6 +7,7 @@ import { createAdapter } from './daemon-pty-router-test-fixture' import { disconnectDaemon, listLiveDaemonSessions, + listLiveDaemonSessionsWithProtocol, replaceDaemonProvider, requestIdleDaemonRetirement } from './daemon-provider-state' @@ -30,6 +31,17 @@ it('reads a census with an unanswered generation as unverifiable', async () => { await expect(listLiveDaemonSessions()).resolves.toBeNull() }) +it('labels each live session with the protocol of the generation that owns it', async () => { + const current = createAdapter('current', ['live-1'], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', ['live-2'], undefined, PROTOCOL_VERSION - 1) + replaceDaemonProvider(new DaemonPtyRouter({ current, legacy: [legacy] })) + + await expect(listLiveDaemonSessionsWithProtocol()).resolves.toEqual([ + { sessionId: 'live-1', isAlive: true, protocolVersion: PROTOCOL_VERSION }, + { sessionId: 'live-2', isAlive: true, protocolVersion: PROTOCOL_VERSION - 1 } + ]) +}) + it('lists every generation when each one answers', async () => { const current = createAdapter('current', ['live-1'], undefined, PROTOCOL_VERSION) const legacy = createAdapter('legacy', ['live-2'], undefined, PROTOCOL_VERSION) diff --git a/src/main/daemon/daemon-provider-state.ts b/src/main/daemon/daemon-provider-state.ts index f32b8a3c204..f54d886206d 100644 --- a/src/main/daemon/daemon-provider-state.ts +++ b/src/main/daemon/daemon-provider-state.ts @@ -11,7 +11,7 @@ import { getMacDaemonTccAttributionHealth, type MacDaemonTccAttributionHealth } from './daemon-tcc-attribution' -import { PROTOCOL_VERSION, type SessionInfo } from './types' +import { PROTOCOL_VERSION, type DaemonSessionInfo, type SessionInfo } from './types' import type { DaemonIdleRetirementResult } from './daemon-pty-runtime-state' let spawner: DaemonSpawner | null = null @@ -147,6 +147,12 @@ export async function listLiveDaemonPtyIds(): Promise { /** Returns null unless every daemon generation supplied an authoritative session inventory. */ export async function listLiveDaemonSessions(): Promise { + const sessions = await listLiveDaemonSessionsWithProtocol() + return sessions?.map(({ protocolVersion: _protocolVersion, ...session }) => session) ?? null +} + +/** Like listLiveDaemonSessions, with the protocol of the daemon generation owning each session. */ +export async function listLiveDaemonSessionsWithProtocol(): Promise { if (!adapter) { return null } @@ -155,7 +161,12 @@ export async function listLiveDaemonSessions(): Promise { ? adapter.getAllAdapters() : [adapter] const inventories = await Promise.allSettled( - adapters.map((daemonAdapter) => daemonAdapter.listSessions()) + adapters.map(async (daemonAdapter) => + (await daemonAdapter.listSessions()).map((session) => ({ + ...session, + protocolVersion: daemonAdapter.protocolVersion + })) + ) ) if (inventories.some((inventory) => inventory.status === 'rejected')) { return null diff --git a/src/main/ipc/orcad-runtime-lifecycle-handlers.ts b/src/main/ipc/orcad-runtime-lifecycle-handlers.ts index 35801d8de77..4d6e23833b1 100644 --- a/src/main/ipc/orcad-runtime-lifecycle-handlers.ts +++ b/src/main/ipc/orcad-runtime-lifecycle-handlers.ts @@ -29,7 +29,7 @@ export function registerOrcadRuntimeLifecycleHandlers(options: { ) } -function requiredString(value: unknown, label: string): string { +export function requiredString(value: unknown, label: string): string { if (typeof value !== 'string' || !value.trim()) { throw new Error(`${label} is required.`) } diff --git a/src/main/ipc/orcad-runtime-maintenance-handlers.test.ts b/src/main/ipc/orcad-runtime-maintenance-handlers.test.ts new file mode 100644 index 00000000000..ad87a398cc7 --- /dev/null +++ b/src/main/ipc/orcad-runtime-maintenance-handlers.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + handle: vi.fn(), + update: vi.fn(), + rollback: vi.fn(), + recover: vi.fn(), + stop: vi.fn(), + cancel: vi.fn(), + retire: vi.fn() +})) + +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../ssh/orcad-runtime-lifecycle', () => ({ + updateManagedOrcadEnvironment: mocks.update, + rollbackManagedOrcadEnvironment: mocks.rollback, + recoverManagedOrcadEnvironment: mocks.recover, + stopManagedOrcadEnvironment: mocks.stop, + cancelManagedOrcadStop: mocks.cancel +})) +vi.mock('./runtime-environment-removal-cleanup', () => ({ + retireRemovedRuntimeEnvironment: mocks.retire +})) + +const { registerOrcadRuntimeMaintenanceHandlers } = + await import('./orcad-runtime-maintenance-handlers') + +const invalidateTransport = vi.fn() + +function handler(channel: string): (_event: unknown, args: unknown) => Promise { + const registration = mocks.handle.mock.calls.find(([name]) => name === channel) + if (!registration) { + throw new Error(`${channel} handler was not registered`) + } + return registration[1] +} + +describe('managed orcad maintenance IPC', () => { + beforeEach(() => { + vi.clearAllMocks() + registerOrcadRuntimeMaintenanceHandlers({ + getUserDataPath: () => '/profile', + getActiveEnvironmentId: () => 'active-environment', + invalidateTransport + }) + }) + + it('reconnects after an update restarts orcad, but not after a deferral', async () => { + mocks.update.mockResolvedValueOnce({ outcome: 'deferred', code: 'busy' }) + await handler('runtimeEnvironments:updateOrcad')(null, { selector: 'Managed', force: 'yes' }) + expect(mocks.update).toHaveBeenCalledWith('/profile', { selector: 'Managed', force: false }) + expect(invalidateTransport).not.toHaveBeenCalled() + mocks.update.mockResolvedValueOnce({ outcome: 'updated', environment: { id: 'e-1' } }) + await handler('runtimeEnvironments:updateOrcad')(null, { selector: 'Managed', force: true }) + expect(invalidateTransport).toHaveBeenCalledWith('e-1') + }) + + it('reconnects after rollback and after recovery restores a serving slot', async () => { + mocks.rollback.mockResolvedValueOnce({ outcome: 'rolled-back', environment: { id: 'e-1' } }) + await handler('runtimeEnvironments:rollbackOrcad')(null, { selector: 'Managed' }) + mocks.recover.mockResolvedValueOnce({ + outcome: 'recovered', + activeVersion: null, + environment: { id: 'e-1' } + }) + await handler('runtimeEnvironments:recoverOrcad')(null, { selector: 'Managed' }) + expect(invalidateTransport).toHaveBeenCalledTimes(1) + }) + + it('stops with the Active Server guard and the shared removal cleanup', async () => { + mocks.stop.mockResolvedValueOnce({ outcome: 'unlinked' }) + await handler('runtimeEnvironments:stopOrcad')(null, { selector: ' Managed ' }) + const [, args, policy] = mocks.stop.mock.calls[0] ?? [] + expect(args).toEqual({ selector: 'Managed' }) + expect(policy.isActiveEnvironment('active-environment')).toBe(true) + expect(policy.isActiveEnvironment('e-1')).toBe(false) + policy.retireLocalState('e-1') + expect(mocks.retire).toHaveBeenCalledWith('e-1', invalidateTransport) + }) + + it('rejects a missing selector before touching SSH', async () => { + await expect(handler('runtimeEnvironments:cancelOrcadStop')(null, {})).rejects.toThrow( + 'Server is required' + ) + expect(mocks.cancel).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/orcad-runtime-maintenance-handlers.ts b/src/main/ipc/orcad-runtime-maintenance-handlers.ts new file mode 100644 index 00000000000..1fe7c49cdda --- /dev/null +++ b/src/main/ipc/orcad-runtime-maintenance-handlers.ts @@ -0,0 +1,83 @@ +import { ipcMain } from 'electron' +import type { + OrcadManagedCancelStopResult, + OrcadManagedDeployResult, + OrcadManagedRecoveryResult, + OrcadManagedRollbackResult, + OrcadManagedStopResult +} from '../../shared/orcad-managed-runtime' +import { + cancelManagedOrcadStop, + recoverManagedOrcadEnvironment, + rollbackManagedOrcadEnvironment, + stopManagedOrcadEnvironment, + updateManagedOrcadEnvironment +} from '../ssh/orcad-runtime-lifecycle' +import { retireRemovedRuntimeEnvironment } from './runtime-environment-removal-cleanup' +import { requiredString } from './orcad-runtime-lifecycle-handlers' + +export function registerOrcadRuntimeMaintenanceHandlers(options: { + getUserDataPath: () => string + getActiveEnvironmentId: () => string | null | undefined + invalidateTransport: (environmentId: string) => Promise | void +}): void { + ipcMain.handle( + 'runtimeEnvironments:updateOrcad', + async (_event, args: { selector: string; force?: boolean }): Promise => { + const result = await updateManagedOrcadEnvironment(options.getUserDataPath(), { + selector: requiredString(args?.selector, 'Server'), + force: args?.force === true + }) + // Why: a restarted orcad drops the old connection; reconnect on the new one. + if (result.outcome === 'updated') { + await options.invalidateTransport(result.environment.id) + } + return result + } + ) + ipcMain.handle( + 'runtimeEnvironments:rollbackOrcad', + async (_event, args: { selector: string }): Promise => { + const result = await rollbackManagedOrcadEnvironment(options.getUserDataPath(), { + selector: requiredString(args?.selector, 'Server') + }) + if (result.outcome === 'rolled-back') { + await options.invalidateTransport(result.environment.id) + } + return result + } + ) + ipcMain.handle( + 'runtimeEnvironments:recoverOrcad', + async (_event, args: { selector: string }): Promise => { + const result = await recoverManagedOrcadEnvironment(options.getUserDataPath(), { + selector: requiredString(args?.selector, 'Server') + }) + if (result.outcome === 'recovered' && result.activeVersion) { + await options.invalidateTransport(result.environment.id) + } + return result + } + ) + ipcMain.handle( + 'runtimeEnvironments:stopOrcad', + async (_event, args: { selector: string }): Promise => + stopManagedOrcadEnvironment( + options.getUserDataPath(), + { selector: requiredString(args?.selector, 'Server') }, + { + isActiveEnvironment: (environmentId) => + options.getActiveEnvironmentId() === environmentId, + retireLocalState: (environmentId) => + retireRemovedRuntimeEnvironment(environmentId, options.invalidateTransport) + } + ) + ) + ipcMain.handle( + 'runtimeEnvironments:cancelOrcadStop', + async (_event, args: { selector: string }): Promise => + cancelManagedOrcadStop(options.getUserDataPath(), { + selector: requiredString(args?.selector, 'Server') + }) + ) +} diff --git a/src/main/ipc/runtime-environment-connectivity-handlers.ts b/src/main/ipc/runtime-environment-connectivity-handlers.ts index 84d2754d556..6513c0756b8 100644 --- a/src/main/ipc/runtime-environment-connectivity-handlers.ts +++ b/src/main/ipc/runtime-environment-connectivity-handlers.ts @@ -14,10 +14,8 @@ import { RuntimeRpcCallQueueOverloadError } from '../../shared/runtime-rpc-call- import type { RuntimeRpcFailure, RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' import type { RuntimeStatus } from '../../shared/runtime-types' import type { Store } from '../persistence' -import { clearBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-runtime' -import { retireBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-retirement' +import { retireRemovedRuntimeEnvironment } from './runtime-environment-removal-cleanup' import { verifyAndAddRuntimeEnvironmentFromPairingCode } from './runtime-environment-pairing-verification' -import { clearRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence' import { closeRemoteRuntimeRequestConnection, getRuntimeEnvironmentStatusOwner, @@ -106,22 +104,8 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ throw new Error('Choose another Active Server in Advanced before removing this server.') } const removed = removeEnvironment(getUserDataPath(), args.selector) - clearRuntimeEnvironmentCapabilityEvidence(removed.id) - clearRuntimeEnvironmentManualDisconnect(removed.id) - const retiring = Promise.resolve(invalidateTransport(removed.id)) + void retireRemovedRuntimeEnvironment(removed.id, invalidateTransport) closeLegacySelectorTransport(args.selector, removed.id) - // Why: removal is an explicit lifecycle decision, so its client-hosted browser storage goes - // too -- but only once the client host releases its partitions, or every one refuses as live. - void retireBrowserRoutePartitionStorageForEnvironment({ - environmentId: removed.id, - whenClientHostClosed: retiring, - clearStorage: clearBrowserRoutePartitionStorageForEnvironment, - onError: (error) => { - console.warn('[runtime-environments] browser partition storage clear failed:', error) - } - }).catch((error) => { - console.warn('[runtime-environments] browser partition storage clear failed:', error) - }) return { removed: redactRuntimeEnvironment(removed) } } ) diff --git a/src/main/ipc/runtime-environment-handler-channels.ts b/src/main/ipc/runtime-environment-handler-channels.ts index de80ea0b178..3af5e61a64c 100644 --- a/src/main/ipc/runtime-environment-handler-channels.ts +++ b/src/main/ipc/runtime-environment-handler-channels.ts @@ -17,6 +17,11 @@ export const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [ 'runtimeEnvironments:unlinkSshAccess', 'runtimeEnvironments:deployOrcad', 'runtimeEnvironments:getOrcadStatus', + 'runtimeEnvironments:updateOrcad', + 'runtimeEnvironments:rollbackOrcad', + 'runtimeEnvironments:recoverOrcad', + 'runtimeEnvironments:stopOrcad', + 'runtimeEnvironments:cancelOrcadStop', 'runtimeEnvironments:createOrcadSshHost', 'runtimeEnvironments:resumeOrcadSshHost', 'runtimeEnvironments:listPendingOrcadSshProvisioning' diff --git a/src/main/ipc/runtime-environment-removal-cleanup.ts b/src/main/ipc/runtime-environment-removal-cleanup.ts new file mode 100644 index 00000000000..c4141c0e00e --- /dev/null +++ b/src/main/ipc/runtime-environment-removal-cleanup.ts @@ -0,0 +1,27 @@ +import { clearBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-runtime' +import { retireBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-retirement' +import { clearRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence' +import { clearRuntimeEnvironmentManualDisconnect } from './runtime-environment-manual-disconnect' + +/** Retires a removed server's client-side state; resolves once its transport is invalidated. */ +export function retireRemovedRuntimeEnvironment( + environmentId: string, + invalidateTransport: (environmentId: string) => Promise | void +): Promise { + clearRuntimeEnvironmentCapabilityEvidence(environmentId) + clearRuntimeEnvironmentManualDisconnect(environmentId) + const retiring = Promise.resolve(invalidateTransport(environmentId)) + // Why: removal is an explicit lifecycle decision, so its client-hosted browser storage goes + // too -- but only once the client host releases its partitions, or every one refuses as live. + void retireBrowserRoutePartitionStorageForEnvironment({ + environmentId, + whenClientHostClosed: retiring, + clearStorage: clearBrowserRoutePartitionStorageForEnvironment, + onError: (error) => { + console.warn('[runtime-environments] browser partition storage clear failed:', error) + } + }).catch((error) => { + console.warn('[runtime-environments] browser partition storage clear failed:', error) + }) + return retiring +} diff --git a/src/main/ipc/runtime-environments-pairing.test.ts b/src/main/ipc/runtime-environments-pairing.test.ts index 54a6deb9093..d798e12586c 100644 --- a/src/main/ipc/runtime-environments-pairing.test.ts +++ b/src/main/ipc/runtime-environments-pairing.test.ts @@ -161,6 +161,11 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:listPendingOrcadSshProvisioning', 'runtimeEnvironments:deployOrcad', 'runtimeEnvironments:getOrcadStatus', + 'runtimeEnvironments:updateOrcad', + 'runtimeEnvironments:rollbackOrcad', + 'runtimeEnvironments:recoverOrcad', + 'runtimeEnvironments:stopOrcad', + 'runtimeEnvironments:cancelOrcadStop', 'runtimeEnvironments:subscribe', 'runtimeEnvironments:unsubscribe' ]) @@ -191,6 +196,11 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:unlinkSshAccess', 'runtimeEnvironments:deployOrcad', 'runtimeEnvironments:getOrcadStatus', + 'runtimeEnvironments:updateOrcad', + 'runtimeEnvironments:rollbackOrcad', + 'runtimeEnvironments:recoverOrcad', + 'runtimeEnvironments:stopOrcad', + 'runtimeEnvironments:cancelOrcadStop', 'runtimeEnvironments:createOrcadSshHost', 'runtimeEnvironments:resumeOrcadSshHost', 'runtimeEnvironments:listPendingOrcadSshProvisioning', diff --git a/src/main/ipc/runtime-environments.ts b/src/main/ipc/runtime-environments.ts index 7e50f4b8f71..3117a81a067 100644 --- a/src/main/ipc/runtime-environments.ts +++ b/src/main/ipc/runtime-environments.ts @@ -19,6 +19,7 @@ import { } from './runtime-environment-subscriptions' import { RUNTIME_ENVIRONMENT_HANDLER_CHANNELS } from './runtime-environment-handler-channels' import { registerOrcadRuntimeLifecycleHandlers } from './orcad-runtime-lifecycle-handlers' +import { registerOrcadRuntimeMaintenanceHandlers } from './orcad-runtime-maintenance-handlers' import { registerRuntimeSshAccessHandlers } from './runtime-ssh-access-handlers' import { retirePairedRuntimeBrowserClientHostEnvironment } from '../browser/paired-runtime-browser-client-host-runtime' import { registerRuntimeEnvironmentBrowserClientHostHandler } from './runtime-environment-browser-client-host-handler' @@ -74,5 +75,10 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { invalidateTransport: invalidateRuntimeEnvironmentTransport }) registerOrcadRuntimeLifecycleHandlers({ getUserDataPath }) + registerOrcadRuntimeMaintenanceHandlers({ + getUserDataPath, + getActiveEnvironmentId: () => store.getSettings().activeRuntimeEnvironmentId, + invalidateTransport: invalidateRuntimeEnvironmentTransport + }) registerRuntimeEnvironmentSubscriptions(getUserDataPath) } diff --git a/src/main/orcad/orcad-terminal-census.test.ts b/src/main/orcad/orcad-terminal-census.test.ts new file mode 100644 index 00000000000..e8067f8a2b1 --- /dev/null +++ b/src/main/orcad/orcad-terminal-census.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import type { DaemonSessionInfo } from '../daemon/types' +import { collectOrcadTerminalCensus } from './orcad-terminal-census' + +function session(createdAt: number, protocolVersion = 7): DaemonSessionInfo { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the census reads only createdAt and protocolVersion. + return { sessionId: `s-${createdAt}`, createdAt, protocolVersion } as DaemonSessionInfo +} + +describe('orcad terminal census', () => { + it('counts live sessions, those since activation, and their single owning protocol', async () => { + await expect( + collectOrcadTerminalCensus(100, async () => [session(50), session(100), session(150)]) + ).resolves.toEqual({ liveSessions: 3, startedSinceActivation: 2, daemonProtocolVersion: 7 }) + }) + + it('reports zero, not unknown, for an answered empty daemon', async () => { + await expect(collectOrcadTerminalCensus(100, async () => [])).resolves.toEqual({ + liveSessions: 0, + startedSinceActivation: 0, + daemonProtocolVersion: null + }) + }) + + it('leaves the protocol unknown when sessions span daemon generations', async () => { + const census = await collectOrcadTerminalCensus(0, async () => [session(1, 7), session(2, 6)]) + expect(census.daemonProtocolVersion).toBeNull() + expect(census.liveSessions).toBe(2) + }) + + it('leaves the since-activation count unknown when a session has no creation time', async () => { + const census = await collectOrcadTerminalCensus(0, async () => [session(0)]) + expect(census).toMatchObject({ liveSessions: 1, startedSinceActivation: null }) + }) + + it.each([ + ['no inventory', async () => null], + [ + 'a failed inventory', + async () => { + throw new Error('daemon unreachable') + } + ] + ])('reads %s as unverifiable, never as zero', async (_label, list) => { + await expect(collectOrcadTerminalCensus(0, list)).resolves.toEqual({ + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null + }) + }) +}) diff --git a/src/main/orcad/orcad-terminal-census.ts b/src/main/orcad/orcad-terminal-census.ts new file mode 100644 index 00000000000..63fd96e72bd --- /dev/null +++ b/src/main/orcad/orcad-terminal-census.ts @@ -0,0 +1,39 @@ +/** The terminal census a managed orcad reports to the client planning an update or stop. */ +import { listLiveDaemonSessionsWithProtocol } from '../daemon/daemon-provider-state' +import type { DaemonSessionInfo } from '../daemon/types' +import type { OrcadTerminalCensus } from '../../shared/orcad-terminal-census' + +const UNVERIFIABLE: OrcadTerminalCensus = { + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null +} + +export async function collectOrcadTerminalCensus( + activatedAt: number, + listSessions: () => Promise = listLiveDaemonSessionsWithProtocol +): Promise { + let sessions: DaemonSessionInfo[] | null + try { + sessions = await listSessions() + } catch { + sessions = null + } + if (!sessions) { + return UNVERIFIABLE + } + const timestampsKnown = sessions.every( + (session) => Number.isFinite(session.createdAt) && session.createdAt > 0 + ) + const protocols = new Set(sessions.map((session) => session.protocolVersion)) + const [protocol] = protocols + return { + liveSessions: sessions.length, + startedSinceActivation: timestampsKnown + ? sessions.filter((session) => session.createdAt >= activatedAt).length + : null, + // Why one protocol only: sessions split across daemon generations have no single owner to + // check an incoming build against, so that reads as unverifiable. + daemonProtocolVersion: protocols.size === 1 && protocol !== undefined ? protocol : null + } +} diff --git a/src/main/runtime/rpc/methods/index.ts b/src/main/runtime/rpc/methods/index.ts index cbf6e0d8e0c..762d61cf3aa 100644 --- a/src/main/runtime/rpc/methods/index.ts +++ b/src/main/runtime/rpc/methods/index.ts @@ -1,4 +1,5 @@ import { STATUS_METHODS } from './status' +import { ORCAD_TERMINAL_CENSUS_METHODS } from './orcad-terminal-census' import { AI_VAULT_METHODS } from './ai-vault' import { AUTOMATION_METHODS } from './automations' import { REPO_METHODS } from './repo' @@ -54,6 +55,7 @@ import { AGENT_LAUNCH_METHODS } from './agent-launch' // auditing the security boundary or wiring new CLI commands. export const ALL_RPC_METHODS = [ ...STATUS_METHODS, + ...ORCAD_TERMINAL_CENSUS_METHODS, ...AGENT_HOOK_METHODS, ...AI_VAULT_METHODS, ...ARTIFACT_METHODS, diff --git a/src/main/runtime/rpc/methods/orcad-terminal-census.ts b/src/main/runtime/rpc/methods/orcad-terminal-census.ts new file mode 100644 index 00000000000..a35156c721b --- /dev/null +++ b/src/main/runtime/rpc/methods/orcad-terminal-census.ts @@ -0,0 +1,18 @@ +import { defineMethod } from '../core' +import { + ORCAD_TERMINAL_CENSUS_METHOD, + OrcadTerminalCensusParamsSchema +} from '../../../../shared/orcad-terminal-census' + +export const ORCAD_TERMINAL_CENSUS_METHODS = [ + defineMethod({ + name: ORCAD_TERMINAL_CENSUS_METHOD, + params: OrcadTerminalCensusParamsSchema, + handler: async (params) => { + // Why lazy: the census reaches the daemon modules, whose xterm polyfill defines a global + // `window`; importing them statically would load it into every process with the dispatcher. + const { collectOrcadTerminalCensus } = await import('../../../orcad/orcad-terminal-census') + return collectOrcadTerminalCensus(params.activatedAt) + } + }) +] diff --git a/src/main/ssh/orcad-managed-lifecycle-test-fixture.ts b/src/main/ssh/orcad-managed-lifecycle-test-fixture.ts new file mode 100644 index 00000000000..37d27cd3c9a --- /dev/null +++ b/src/main/ssh/orcad-managed-lifecycle-test-fixture.ts @@ -0,0 +1,103 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createManagedOrcadSshOwner } from '../../shared/managed-orcad-ssh-owner' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { addManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import type { SshTarget } from '../../shared/ssh-types' +import type { ServeReadiness } from '../server/serve-readiness' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import { SshTargetOrcadClaims } from './ssh-target-orcad-claims' +import { emptyDependentStateStore } from './ssh-target-orcad-dependents-fixture' + +export const MANAGED_VERSION = '0.2.0+abc' +export const MANAGED_PREVIOUS_VERSION = '0.1.0+def' +export const MANAGED_LOCAL_PORT = 46_768 + +/** A managed server registered in a temp profile, with its claimed SSH target. */ +export function createManagedLifecycleHarness() { + const userDataPath = mkdtempSync(join(tmpdir(), 'orcad-managed-lifecycle-')) + let target: SshTarget = { + id: 'ssh-1', + label: 'Builder', + host: 'builder', + port: 22, + username: 'dev', + generation: 4, + owner: createManagedOrcadSshOwner('environment-1') + } + const environment = addManagedOrcadEnvironment(userDataPath, { + id: 'environment-1', + name: 'Managed', + pairingCode: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: `ws://127.0.0.1:${MANAGED_LOCAL_PORT}/`, + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + orcadDeployment: { + sshTargetId: 'ssh-1', + sshTargetGeneration: 4, + localPort: MANAGED_LOCAL_PORT, + remotePort: 6_768 + } + }) + const flushes: number[] = [] + const claims = new SshTargetOrcadClaims({ + ...emptyDependentStateStore(), + allocateSshTargetGeneration: () => 5, + flushPendingOrThrowAsync: async () => { + flushes.push(flushes.length) + }, + getFolderWorkspaces: () => [], + getRepos: () => [], + getSshTarget: (id) => (id === target.id ? target : undefined), + getSshTargets: () => [target], + updateSshTarget: (_id, updates) => (target = { ...target, ...updates }) + }) + const targetStore = { + getTarget: (id: string) => (id === target.id ? target : undefined), + getOrcadRuntimeClaims: () => claims + } + const context = (record: Record = {}) => ({ + activationRecord: { + active: MANAGED_VERSION, + previous: MANAGED_PREVIOUS_VERSION, + activatedAt: '2026-01-01T00:00:00.000Z', + snapshot: null, + ...record + }, + serverTarget: 'linux-x64-glibc', + connection: {}, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/dev', + target, + userDataDir: '/home/dev/.orca' + }) + return { userDataPath, environment, targetStore, context, flushes, current: () => target } +} + +/** Readiness whose pairing offer, once tunneled, matches the harness server's saved one. */ +export function managedReadiness(deviceToken = 'device-token'): ServeReadiness { + const endpoint = 'ws://127.0.0.1:6768' + return { + runtimeId: 'runtime-1', + boundEndpoint: endpoint, + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { + available: true, + url: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint, + deviceToken, + publicKeyB64: 'public-key' + }), + endpoint, + deviceId: 'device-1', + webClientUrl: null, + scope: 'runtime', + qr: null + } + } +} diff --git a/src/main/ssh/orcad-managed-runtime-context.ts b/src/main/ssh/orcad-managed-runtime-context.ts index 55a29be7398..37e9c5d9daa 100644 --- a/src/main/ssh/orcad-managed-runtime-context.ts +++ b/src/main/ssh/orcad-managed-runtime-context.ts @@ -65,6 +65,20 @@ export async function resolveLinkedOrcadContext( return resolveOrcadRemoteContext(target, connection, signal) } +/** The slot options every remote lifecycle step takes for this managed server. */ +export function managedOrcadSlot(context: OrcadRemoteContext, port: number, signal?: AbortSignal) { + return { + conn: context.connection, + host: context.host, + remoteHome: context.remoteHome, + nodePath: MANAGED_ORCAD_LEGACY_NODE_PATH, + userDataDir: context.userDataDir, + bindHost: ORCAD_BIND_HOST, + port, + signal + } +} + export function managedOrcadInstallDir(context: OrcadRemoteContext, version: string): string { return computeRemoteInstallDir( ORCAD_INSTALL_MODEL, diff --git a/src/main/ssh/orcad-managed-update-deferrals.ts b/src/main/ssh/orcad-managed-update-deferrals.ts new file mode 100644 index 00000000000..1bf0285f2b2 --- /dev/null +++ b/src/main/ssh/orcad-managed-update-deferrals.ts @@ -0,0 +1,22 @@ +/** The last update each managed server deferred in this session, so status can report it. */ +import type { OrcadManagedDeferral } from '../../shared/orcad-managed-runtime' + +type RecordedDeferral = OrcadManagedDeferral & { deferredAt: string } + +const deferrals = new Map() + +export function recordManagedOrcadUpdateDeferral( + environmentId: string, + deferral: OrcadManagedDeferral, + now = new Date() +): void { + deferrals.set(environmentId, { ...deferral, deferredAt: now.toISOString() }) +} + +export function clearManagedOrcadUpdateDeferral(environmentId: string): void { + deferrals.delete(environmentId) +} + +export function readManagedOrcadUpdateDeferral(environmentId: string): RecordedDeferral | null { + return deferrals.get(environmentId) ?? null +} diff --git a/src/main/ssh/orcad-runtime-decommission.test.ts b/src/main/ssh/orcad-runtime-decommission.test.ts new file mode 100644 index 00000000000..b045fbcefdc --- /dev/null +++ b/src/main/ssh/orcad-runtime-decommission.test.ts @@ -0,0 +1,216 @@ +import { existsSync, readFileSync, rmSync } from 'node:fs' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getManagedOrcadOwnerEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { getRuntimeEnvironmentSidecarPath } from '../../shared/runtime-environment-sidecar' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { + createManagedLifecycleHarness, + MANAGED_VERSION +} from './orcad-managed-lifecycle-test-fixture' + +const mocks = vi.hoisted(() => { + const state: { store: unknown; transaction: unknown } = { store: null, transaction: null } + return { + state, + resolveContext: vi.fn(), + census: vi.fn(), + decommission: vi.fn(), + recover: vi.fn(), + cancel: vi.fn(), + keepServing: vi.fn(), + closeTunnel: vi.fn(), + ensureTunnel: vi.fn(), + retire: vi.fn() + } +}) + +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({ connect: async () => ({}) }), + getSshTargetRegistryStore: () => mocks.state.store +})) +vi.mock('./orcad-remote-context', () => ({ resolveOrcadRemoteContext: mocks.resolveContext })) +vi.mock('./orcad-terminal-census-client', () => ({ collectManagedTerminalCensus: mocks.census })) +vi.mock('./orcad-remote-stop', () => ({ decommissionRemoteOrcad: mocks.decommission })) +vi.mock('./orcad-activation-recovery', () => ({ recoverInterruptedOrcadActivation: mocks.recover })) +vi.mock('./orcad-activation-transaction-store', () => ({ + readOrcadActivationTransaction: async () => mocks.state.transaction +})) +vi.mock('./orcad-activation-lock', () => ({ + withStaleOrcadActivationRecoveryLock: async ( + _options: unknown, + run: (lock: { retain: () => void }) => Promise + ) => run({ retain: () => {} }) +})) +vi.mock('./orcad-managed-remote-stop', () => ({ cancelRemoteOrcadManagedStop: mocks.cancel })) +vi.mock('./orcad-decommission-recovery', () => ({ reconcileOrcadDecommission: mocks.keepServing })) +vi.mock('./orcad-managed-tunnel', () => ({ + closeOrcadManagedTunnel: mocks.closeTunnel, + ensureOrcadManagedTunnel: mocks.ensureTunnel +})) + +const { cancelManagedOrcadStop, stopManagedOrcadEnvironment } = + await import('./orcad-runtime-lifecycle') + +const idle = { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 7 } +let harness: ReturnType + +const stop = (isActive = false) => + stopManagedOrcadEnvironment( + harness.userDataPath, + { selector: 'Managed' }, + { isActiveEnvironment: () => isActive, retireLocalState: mocks.retire } + ) + +function expectStillLinked(): void { + expect(listEnvironments(harness.userDataPath)[0]?.orcadDeployment).toBeDefined() + expect(getManagedOrcadOwnerEnvironmentId(harness.current().owner)).toBe('environment-1') + expect(mocks.retire).not.toHaveBeenCalled() + expect(mocks.closeTunnel).not.toHaveBeenCalled() +} + +beforeEach(() => { + vi.resetAllMocks() + harness = createManagedLifecycleHarness() + mocks.state.store = harness.targetStore + mocks.state.transaction = null + mocks.resolveContext.mockImplementation(async () => harness.context()) + mocks.census.mockResolvedValue(idle) +}) + +afterEach(() => rmSync(harness.userDataPath, { recursive: true, force: true })) + +describe('stopManagedOrcadEnvironment', () => { + it('unlinks the server only after the host proves orcad exited', async () => { + mocks.decommission.mockResolvedValueOnce({ + outcome: 'decommissioned', + version: MANAGED_VERSION, + retirement: 'retired' + }) + await expect(stop()).resolves.toEqual({ + outcome: 'unlinked', + verdict: 'exited', + environmentId: 'environment-1', + sshTargetId: 'ssh-1', + stoppedVersion: MANAGED_VERSION, + retirement: 'retired' + }) + expect(mocks.decommission.mock.calls[0]?.[0]).toMatchObject({ census: idle, port: 6_768 }) + expect(listEnvironments(harness.userDataPath)).toEqual([]) + expect( + readFileSync(getRuntimeEnvironmentSidecarPath(harness.userDataPath), 'utf8') + ).not.toContain('orcadDeployment') + expect(harness.current().owner).toBeUndefined() + expect(harness.flushes).toHaveLength(1) + expect(mocks.retire).toHaveBeenCalledWith('environment-1') + expect(mocks.closeTunnel).toHaveBeenCalledWith('environment-1') + }) + + it.each([ + ['live', 'orcad_decommission_terminals_running'], + ['unverifiable', 'orcad_decommission_census_unavailable'], + ['unverifiable', 'orcad_decommission_stop_unsettled'] + ] as const)('keeps every link when the verdict is %s (%s)', async (verdict, code) => { + mocks.decommission.mockResolvedValueOnce({ outcome: 'refused', verdict, code, reason: 'no' }) + await expect(stop()).resolves.toMatchObject({ outcome: 'refused', verdict, code }) + expectStillLinked() + expect(existsSync(getRuntimeEnvironmentSidecarPath(harness.userDataPath))).toBe(true) + }) + + it('refuses the Active Server before contacting the host', async () => { + await expect(stop(true)).resolves.toMatchObject({ code: 'orcad_stop_active_environment' }) + expect(mocks.resolveContext).not.toHaveBeenCalled() + expectStillLinked() + }) + + it('finishes an interrupted stop from its journal and unlinks only on proven exit', async () => { + mocks.state.transaction = { operation: 'decommission', activeVersion: MANAGED_VERSION } + mocks.recover.mockResolvedValueOnce({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_recovery_decommission_unsettled', + reason: 'no answer' + }) + await expect(stop()).resolves.toMatchObject({ outcome: 'refused', verdict: 'unverifiable' }) + expectStillLinked() + mocks.recover.mockResolvedValueOnce({ + outcome: 'recovered', + resolution: 'committed', + activeVersion: null, + readiness: null + }) + await expect(stop()).resolves.toMatchObject({ outcome: 'unlinked', verdict: 'exited' }) + expect(mocks.decommission).not.toHaveBeenCalled() + }) + + it('refuses while another interrupted operation holds the journal', async () => { + mocks.state.transaction = { operation: 'activate' } + await expect(stop()).resolves.toMatchObject({ code: 'orcad_activation_recovery_required' }) + expectStillLinked() + }) + + it('unlinks a server whose record already shows nothing active', async () => { + mocks.resolveContext.mockImplementation(async () => harness.context({ active: null })) + await expect(stop()).resolves.toMatchObject({ outcome: 'unlinked', stoppedVersion: null }) + expect(mocks.decommission).not.toHaveBeenCalled() + }) +}) + +describe('cancelManagedOrcadStop', () => { + const request = { transactionId: 't-1', version: MANAGED_VERSION } + const cancel = () => cancelManagedOrcadStop(harness.userDataPath, { selector: 'Managed' }) + + it('has nothing to cancel without a decommission journal', async () => { + await expect(cancel()).resolves.toEqual({ outcome: 'none' }) + expect(mocks.cancel).not.toHaveBeenCalled() + }) + + it('withdraws a stop orcad never acted on and keeps the server serving', async () => { + mocks.state.transaction = { + operation: 'decommission', + phase: 'stop-dispatched', + activeVersion: MANAGED_VERSION, + request + } + mocks.cancel.mockResolvedValueOnce({ outcome: 'canceled' }) + mocks.keepServing.mockResolvedValueOnce({ + outcome: 'recovered', + resolution: 'restored-incumbent', + activeVersion: MANAGED_VERSION, + readiness: null + }) + await expect(cancel()).resolves.toEqual({ outcome: 'canceled', activeVersion: MANAGED_VERSION }) + expect(mocks.keepServing.mock.calls[0]?.[1]).toEqual({ + action: 'keep-serving', + version: MANAGED_VERSION + }) + expect(mocks.ensureTunnel).toHaveBeenCalledOnce() + expectStillLinked() + }) + + it('refuses once orcad already acted on the stop', async () => { + mocks.state.transaction = { + operation: 'decommission', + phase: 'stop-dispatched', + activeVersion: MANAGED_VERSION, + request + } + mocks.cancel.mockResolvedValueOnce({ outcome: 'dispatched' }) + await expect(cancel()).resolves.toMatchObject({ + outcome: 'refused', + verdict: 'live', + code: 'orcad_stop_already_dispatched' + }) + expect(mocks.keepServing).not.toHaveBeenCalled() + }) + + it('reports an already-exited stop for stop to finish', async () => { + mocks.state.transaction = { + operation: 'decommission', + phase: 'process-exited', + activeVersion: MANAGED_VERSION, + request + } + await expect(cancel()).resolves.toEqual({ outcome: 'already-stopped' }) + expectStillLinked() + }) +}) diff --git a/src/main/ssh/orcad-runtime-decommission.ts b/src/main/ssh/orcad-runtime-decommission.ts new file mode 100644 index 00000000000..ffb8d36954c --- /dev/null +++ b/src/main/ssh/orcad-runtime-decommission.ts @@ -0,0 +1,202 @@ +/** + * Stopping a managed orcad and unlinking it, on T6-4's journaled decommission. The server stays + * linked — its SSH claim, tunnel and deployment record — unless the host proves orcad exited. + */ +import type { + OrcadManagedCancelStopResult, + OrcadManagedStopResult +} from '../../shared/orcad-managed-runtime' +import { removeManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import type { KnownRuntimeEnvironment, OrcadDeploymentLink } from '../../shared/runtime-environments' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { withStaleOrcadActivationRecoveryLock } from './orcad-activation-lock' +import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { reconcileOrcadDecommission } from './orcad-decommission-recovery' +import { cancelRemoteOrcadManagedStop } from './orcad-managed-remote-stop' +import { + managedOrcadSlot, + requireManagedOrcadInfrastructure, + resolveLinkedOrcadContext +} from './orcad-managed-runtime-context' +import { closeOrcadManagedTunnel, ensureOrcadManagedTunnel } from './orcad-managed-tunnel' +import { clearManagedOrcadUpdateDeferral } from './orcad-managed-update-deferrals' +import { decommissionRemoteOrcad } from './orcad-remote-stop' +import { withManagedOrcadLifecycle } from './orcad-runtime-maintenance' +import { collectManagedTerminalCensus } from './orcad-terminal-census-client' +import { RemoteInstallLockBusyError } from './ssh-relay-install-lock' + +export type ManagedOrcadStopPolicy = { + isActiveEnvironment: (environmentId: string) => boolean + /** Invalidates the server's transport and retires its client-side state once it is unlinked. */ + retireLocalState: (environmentId: string) => Promise | void +} + +type Refusal = Extract +type Stopped = { version: string | null; retirement: 'retired' | 'live' | 'unverifiable' | null } + +function refuse(verdict: Refusal['verdict'], code: string, reason: string): Refusal { + return { outcome: 'refused', verdict, code, reason } +} + +export function stopManagedOrcadEnvironment( + userDataPath: string, + args: { selector: string; signal?: AbortSignal }, + policy: ManagedOrcadStopPolicy +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async (managed) => { + const { environment, deployment } = managed + if (policy.isActiveEnvironment(environment.id)) { + return refuse( + 'live', + 'orcad_stop_active_environment', + 'Choose another Active Server in Advanced before stopping this server.' + ) + } + const stopped = await stopRemote(userDataPath, environment, deployment, args.signal) + if ('outcome' in stopped) { + return stopped + } + await unlinkStoppedEnvironment(userDataPath, environment, deployment, policy) + return { + outcome: 'unlinked', + verdict: 'exited', + environmentId: environment.id, + sshTargetId: deployment.sshTargetId, + stoppedVersion: stopped.version, + retirement: stopped.retirement + } + }) +} + +/** Proven exit, or the refusal that keeps the server linked. */ +async function stopRemote( + userDataPath: string, + environment: KnownRuntimeEnvironment, + deployment: OrcadDeploymentLink, + signal?: AbortSignal +): Promise { + const context = await resolveLinkedOrcadContext(environment, deployment, signal) + const slot = managedOrcadSlot(context, deployment.remotePort, signal) + const transaction = await readOrcadActivationTransaction(slot) + if (transaction?.operation === 'decommission') { + // An earlier stop was interrupted: its journal decides, so finish exactly that one. + const recovered = await recoverInterruptedOrcadActivation(slot) + if (recovered.outcome === 'recovered' && recovered.activeVersion === null) { + return { version: transaction.activeVersion, retirement: null } + } + if (recovered.outcome === 'refused') { + return refuse(recovered.verdict, recovered.code, recovered.reason) + } + if (recovered.outcome === 'pending') { + return refuse('unverifiable', recovered.code, recovered.reason) + } + return refuse( + 'live', + 'orcad_stop_withdrawn', + 'The interrupted stop was withdrawn and the server keeps serving. Stop it again.' + ) + } + if (transaction) { + return refuse( + 'unverifiable', + 'orcad_activation_recovery_required', + 'An earlier update on this server was interrupted. Recover it before stopping.' + ) + } + const record = context.activationRecord + if (!record.active) { + // Only a proven exit (or a deploy that never activated) leaves the record without a version. + return { version: null, retirement: null } + } + const census = await collectManagedTerminalCensus(userDataPath, environment, record) + const result = await decommissionRemoteOrcad({ ...slot, record, census }) + if (result.outcome === 'refused') { + return refuse(result.verdict, result.code, result.reason) + } + return { version: result.version, retirement: result.retirement } +} + +async function unlinkStoppedEnvironment( + userDataPath: string, + environment: KnownRuntimeEnvironment, + deployment: OrcadDeploymentLink, + policy: ManagedOrcadStopPolicy +): Promise { + // Environment first: a crash before the claim is released leaves a hidden target to resume, + // never a linked server whose SSH target was already handed back. + removeManagedOrcadEnvironment(userDataPath, environment.id) + await policy.retireLocalState(environment.id) + await closeOrcadManagedTunnel(environment.id) + const { claims } = requireManagedOrcadInfrastructure() + claims.release(deployment.sshTargetId, environment.id) + clearManagedOrcadUpdateDeferral(environment.id) + await claims.flush() +} + +/** Withdraws a stop orcad has not acted on yet; the server then keeps serving. */ +export function cancelManagedOrcadStop( + userDataPath: string, + args: { selector: string; signal?: AbortSignal } +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async (managed) => { + const { environment, deployment } = managed + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) + let result: OrcadManagedCancelStopResult + try { + result = await withStaleOrcadActivationRecoveryLock(slot, async (lock) => { + const transaction = await readOrcadActivationTransaction(slot) + if (transaction?.operation !== 'decommission') { + if (transaction) { + lock.retain() + } + return { outcome: 'none' } + } + if (transaction.phase === 'process-exited') { + lock.retain() + return { outcome: 'already-stopped' } + } + if (transaction.request) { + const cancellation = await cancelRemoteOrcadManagedStop(slot, transaction.request) + if (cancellation.outcome !== 'canceled') { + lock.retain() + return { + outcome: 'refused', + verdict: 'live', + code: 'orcad_stop_already_dispatched', + reason: 'orcad already acted on the stop and is shutting down; it cannot be canceled.' + } + } + } + const kept = await reconcileOrcadDecommission(slot, { + action: 'keep-serving', + version: transaction.activeVersion + }) + if (kept.outcome !== 'recovered' || !kept.activeVersion) { + lock.retain() + return { + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_stop_cancel_unverifiable', + reason: 'The stop was withdrawn but the server could not be shown to be serving.' + } + } + return { outcome: 'canceled', activeVersion: kept.activeVersion } + }) + } catch (error) { + if (error instanceof RemoteInstallLockBusyError) { + return { + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_stop_still_running', + reason: 'A stop on this server may still be running. Retry after it settles.' + } + } + throw error + } + if (result.outcome === 'canceled') { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + } + return result + }) +} diff --git a/src/main/ssh/orcad-runtime-deployment.test.ts b/src/main/ssh/orcad-runtime-deployment.test.ts index 2faf1d7cb5d..f0f66ec5e5c 100644 --- a/src/main/ssh/orcad-runtime-deployment.test.ts +++ b/src/main/ssh/orcad-runtime-deployment.test.ts @@ -46,6 +46,13 @@ vi.mock('./orcad-artifact-materializer', () => ({ })) vi.mock('./orcad-local-build-hash', () => ({ computeLocalOrcadBuildHash: () => 'build-hash' })) vi.mock('./orcad-active-readiness', () => ({ probeActiveOrcadReadiness: mocks.probe })) +vi.mock('./orcad-terminal-census-client', () => ({ + collectManagedTerminalCensus: async () => ({ + liveSessions: 0, + startedSinceActivation: 0, + daemonProtocolVersion: 39 + }) +})) vi.mock('./orcad-managed-tunnel', () => ({ startOrcadManagedTunnel: mocks.startTunnel, ensureOrcadManagedTunnel: mocks.ensureTunnel, @@ -330,7 +337,9 @@ describe('getManagedOrcadRuntimeStatus', () => { phase: 'candidate-ready', version: '0.2.0+def', startedAt: 'then' - } + }, + terminals: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 39 }, + deferredUpdate: null }) expect(mocks.recover).toHaveBeenCalledTimes(1) }) diff --git a/src/main/ssh/orcad-runtime-deployment.ts b/src/main/ssh/orcad-runtime-deployment.ts index 575ad10da7e..fe4ee7b79a6 100644 --- a/src/main/ssh/orcad-runtime-deployment.ts +++ b/src/main/ssh/orcad-runtime-deployment.ts @@ -28,8 +28,7 @@ import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing' import { hasRegisteredDirectSshAuthority } from './ssh-target-registry' import { isForceableOrcadDeferral, - MANAGED_ORCAD_LEGACY_NODE_PATH, - ORCAD_BIND_HOST, + managedOrcadSlot, probeManagedOrcadReadiness, requireManagedOrcadInfrastructure } from './orcad-managed-runtime-context' @@ -70,16 +69,7 @@ export async function createManagedOrcadEnvironment( try { const connection = await connectionManager.connect(claimed) let context = await resolveOrcadRemoteContext(claimed, connection, args.signal) - const slot = { - conn: context.connection, - host: context.host, - remoteHome: context.remoteHome, - nodePath: MANAGED_ORCAD_LEGACY_NODE_PATH, - userDataDir: context.userDataDir, - bindHost: ORCAD_BIND_HOST, - port: ORCAD_MANAGED_REMOTE_PORT, - signal: args.signal - } + const slot = managedOrcadSlot(context, ORCAD_MANAGED_REMOTE_PORT, args.signal) const recovery = await recoverInterruptedOrcadActivation(slot) if (recovery.outcome === 'pending' || recovery.outcome === 'refused') { throw new Error(recovery.reason) diff --git a/src/main/ssh/orcad-runtime-lifecycle.ts b/src/main/ssh/orcad-runtime-lifecycle.ts index 247f6ca1732..0562ccce7c7 100644 --- a/src/main/ssh/orcad-runtime-lifecycle.ts +++ b/src/main/ssh/orcad-runtime-lifecycle.ts @@ -1,2 +1,8 @@ export { createManagedOrcadEnvironment } from './orcad-runtime-deployment' export { getManagedOrcadRuntimeStatus } from './orcad-runtime-status' +export { + recoverManagedOrcadEnvironment, + rollbackManagedOrcadEnvironment, + updateManagedOrcadEnvironment +} from './orcad-runtime-maintenance' +export { cancelManagedOrcadStop, stopManagedOrcadEnvironment } from './orcad-runtime-decommission' diff --git a/src/main/ssh/orcad-runtime-maintenance.test.ts b/src/main/ssh/orcad-runtime-maintenance.test.ts new file mode 100644 index 00000000000..c150dc7b59d --- /dev/null +++ b/src/main/ssh/orcad-runtime-maintenance.test.ts @@ -0,0 +1,197 @@ +import { rmSync } from 'node:fs' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { + createManagedLifecycleHarness, + MANAGED_PREVIOUS_VERSION, + MANAGED_VERSION, + managedReadiness +} from './orcad-managed-lifecycle-test-fixture' + +const mocks = vi.hoisted(() => { + const state: { store: unknown } = { store: null } + return { + state, + resolveContext: vi.fn(), + census: vi.fn(), + deploy: vi.fn(), + rollback: vi.fn(), + recover: vi.fn(), + probe: vi.fn(), + buildHash: vi.fn(), + ensureTunnel: vi.fn() + } +}) + +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({ connect: async () => ({}) }), + getSshTargetRegistryStore: () => mocks.state.store +})) +vi.mock('./orcad-remote-context', () => ({ resolveOrcadRemoteContext: mocks.resolveContext })) +vi.mock('./orcad-terminal-census-client', () => ({ collectManagedTerminalCensus: mocks.census })) +vi.mock('./orcad-remote-deploy', () => ({ deployOrcad: mocks.deploy })) +vi.mock('./orcad-remote-rollback', () => ({ rollbackOrcad: mocks.rollback })) +vi.mock('./orcad-activation-recovery', () => ({ recoverInterruptedOrcadActivation: mocks.recover })) +vi.mock('./orcad-active-readiness', () => ({ probeActiveOrcadReadiness: mocks.probe })) +vi.mock('./orcad-remote-build-hash', () => ({ readRemoteOrcadBuildHash: mocks.buildHash })) +vi.mock('./orcad-artifact-materializer', () => ({ + materializeOrcadArtifact: async () => '/local/orcad' +})) +vi.mock('./orcad-local-build-hash', () => ({ computeLocalOrcadBuildHash: () => 'local-hash' })) +vi.mock('./orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: mocks.ensureTunnel })) +vi.mock('./orcad-activation-transaction-store', () => ({ + readOrcadActivationTransaction: async () => null +})) + +const { + getManagedOrcadRuntimeStatus, + recoverManagedOrcadEnvironment, + rollbackManagedOrcadEnvironment, + updateManagedOrcadEnvironment +} = await import('./orcad-runtime-lifecycle') + +const idle = { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 7 } +let harness: ReturnType + +beforeEach(() => { + vi.resetAllMocks() + harness = createManagedLifecycleHarness() + mocks.state.store = harness.targetStore + mocks.resolveContext.mockImplementation(async () => harness.context()) + mocks.census.mockResolvedValue(idle) + mocks.probe.mockResolvedValue(managedReadiness()) + mocks.buildHash.mockResolvedValue('previous-hash') + mocks.recover.mockResolvedValue({ outcome: 'none' }) +}) + +afterEach(() => rmSync(harness.userDataPath, { recursive: true, force: true })) + +describe('updateManagedOrcadEnvironment', () => { + it('defers over live or unverifiable terminals and reports the deferral in status', async () => { + mocks.census.mockResolvedValue({ ...idle, liveSessions: null }) + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '0.3.0+new', + code: 'orcad_update_terminal_census_unavailable', + reason: 'The terminal daemon did not answer a session count.' + }) + const result = await updateManagedOrcadEnvironment(harness.userDataPath, { + selector: 'Managed' + }) + expect(result).toMatchObject({ outcome: 'deferred', forceable: false }) + expect(mocks.deploy.mock.calls[0]?.[0]).toMatchObject({ + census: { liveSessions: null }, + port: 6_768, + nodePath: '' + }) + const status = await getManagedOrcadRuntimeStatus(harness.userDataPath, 'Managed') + expect(status.deferredUpdate).toMatchObject({ + candidateVersion: '0.3.0+new', + code: 'orcad_update_terminal_census_unavailable' + }) + expect(status.terminals.liveSessions).toBeNull() + }) + + it('clears the deferral once an update goes through and keeps an unchanged pairing as is', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '0.3.0+new', + code: 'orcad_update_terminals_running', + reason: 'busy' + }) + await updateManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-and-activated', + fullVersion: '0.3.0+new' + }) + const result = await updateManagedOrcadEnvironment(harness.userDataPath, { + selector: 'Managed', + force: true + }) + expect(result).toMatchObject({ outcome: 'updated', activeVersion: '0.3.0+new' }) + expect(mocks.probe).toHaveBeenCalledWith(expect.anything(), { + buildHash: 'local-hash', + fullVersion: '0.3.0+new' + }) + const [environment] = listEnvironments(harness.userDataPath) + expect(environment?.pairingRevision).toBe(harness.environment.pairingRevision) + expect(environment?.orcadDeployment).toEqual(harness.environment.orcadDeployment) + const status = await getManagedOrcadRuntimeStatus(harness.userDataPath, 'Managed') + expect(status.deferredUpdate).toBeNull() + }) + + it('re-pairs through the same tunnel, keeping the deployment link, when the offer changed', async () => { + mocks.deploy.mockResolvedValueOnce({ outcome: 'installed-and-activated', fullVersion: 'v3' }) + mocks.probe.mockResolvedValueOnce(managedReadiness('rotated-token')) + await updateManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + const [environment] = listEnvironments(harness.userDataPath) + expect(environment?.endpoints[0]?.deviceToken).toBe('rotated-token') + expect(environment?.orcadDeployment).toEqual(harness.environment.orcadDeployment) + }) +}) + +describe('rollbackManagedOrcadEnvironment', () => { + it.each([ + [{ ...idle, liveSessions: 2 }, 'orcad_rollback_terminals_running'], + [{ ...idle, liveSessions: null }, 'orcad_rollback_census_unavailable'] + ])('refuses while terminals run or cannot be counted: %j', async (census, code) => { + mocks.census.mockResolvedValue(census) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', code }) + expect(mocks.rollback).not.toHaveBeenCalled() + }) + + it('refuses with no previous version without contacting the slot', async () => { + mocks.resolveContext.mockImplementation(async () => harness.context({ previous: null })) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', code: 'orcad_rollback_no_target' }) + expect(mocks.census).not.toHaveBeenCalled() + }) + + it('rolls back against the installed bytes of the previous slot', async () => { + mocks.rollback.mockResolvedValueOnce({ + outcome: 'rolled-back', + target: MANAGED_PREVIOUS_VERSION, + discarded: [MANAGED_VERSION], + verdict: { decision: 'activate', coverage: 'pty-spawn', warnings: [] } + }) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'rolled-back', activeVersion: MANAGED_PREVIOUS_VERSION }) + expect(mocks.rollback.mock.calls[0]?.[0]).toMatchObject({ targetBuildHash: 'previous-hash' }) + expect(mocks.probe).toHaveBeenCalledWith(expect.anything(), { + buildHash: 'previous-hash', + fullVersion: MANAGED_PREVIOUS_VERSION + }) + }) +}) + +describe('recoverManagedOrcadEnvironment', () => { + it('passes a refusal through and leaves the server linked', async () => { + mocks.recover.mockResolvedValueOnce({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_recovery_unverifiable', + reason: 'host fenced' + }) + await expect( + recoverManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', verdict: 'unverifiable' }) + expect(listEnvironments(harness.userDataPath)).toHaveLength(1) + }) + + it('re-ensures the tunnel once a serving slot is restored', async () => { + mocks.recover.mockResolvedValueOnce({ + outcome: 'recovered', + resolution: 'restored-incumbent', + activeVersion: MANAGED_VERSION, + readiness: managedReadiness() + }) + await expect( + recoverManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'recovered', activeVersion: MANAGED_VERSION }) + expect(mocks.ensureTunnel).toHaveBeenCalledWith(harness.userDataPath, 'environment-1') + }) +}) diff --git a/src/main/ssh/orcad-runtime-maintenance.ts b/src/main/ssh/orcad-runtime-maintenance.ts new file mode 100644 index 00000000000..5ed3a8e16f5 --- /dev/null +++ b/src/main/ssh/orcad-runtime-maintenance.ts @@ -0,0 +1,203 @@ +/** + * Updating, rolling back and recovering a managed orcad, on T6-2's deploy, rollback and recovery. + * Every step reads the terminal census through the server's tunnel first; an unanswered census + * is unverifiable, never zero, so an update over live or uncounted terminals defers (D7). + */ +import { refreshManagedOrcadPairing } from '../../shared/runtime-environment-managed-orcad-store' +import { assertRuntimeEnvironmentNotReconciling } from '../../shared/runtime-environment-reconciliation-record' +import { + redactRuntimeEnvironment, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { + OrcadManagedDeployResult, + OrcadManagedRecoveryResult, + OrcadManagedRollbackResult +} from '../../shared/orcad-managed-runtime' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' +import type { ServeReadiness } from '../server/serve-readiness' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { probeActiveOrcadReadiness } from './orcad-active-readiness' +import { materializeOrcadArtifact } from './orcad-artifact-materializer' +import { CURRENT_ORCAD_DAEMON_PROTOCOL } from './orcad-daemon-protocol-crossing' +import { ensureOrcadManagedTunnel } from './orcad-managed-tunnel' +import { + clearManagedOrcadUpdateDeferral, + recordManagedOrcadUpdateDeferral +} from './orcad-managed-update-deferrals' +import { + isForceableOrcadDeferral, + managedOrcadInstallDir, + managedOrcadSlot, + probeManagedOrcadReadiness, + requireManagedOrcadEnvironment, + resolveLinkedOrcadContext +} from './orcad-managed-runtime-context' +import { readRemoteOrcadBuildHash } from './orcad-remote-build-hash' +import { deployOrcad } from './orcad-remote-deploy' +import { rollbackOrcad } from './orcad-remote-rollback' +import { collectManagedTerminalCensus } from './orcad-terminal-census-client' +import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing' + +type LifecycleArgs = { selector: string; signal?: AbortSignal } + +export function withManagedOrcadLifecycle( + userDataPath: string, + selector: string, + run: (managed: ReturnType) => Promise +): Promise { + const { environment, deployment } = requireManagedOrcadEnvironment(userDataPath, selector) + return runTargetLifecycle(deployment.sshTargetId, async () => { + // Re-read under the queue: a reconciliation or stop may have won the race for it. + const current = requireManagedOrcadEnvironment(userDataPath, environment.id) + assertRuntimeEnvironmentNotReconciling(current.environment) + return run(current) + }) +} + +function refreshPairing( + userDataPath: string, + environment: KnownRuntimeEnvironment, + readiness: ServeReadiness, + localPort: number +): KnownRuntimeEnvironment { + return refreshManagedOrcadPairing( + userDataPath, + environment.id, + tunneledOrcadPairingCode(readiness, localPort) + ) +} + +export function updateManagedOrcadEnvironment( + userDataPath: string, + args: LifecycleArgs & { force?: boolean } +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const census = await collectManagedTerminalCensus( + userDataPath, + environment, + context.activationRecord + ) + const localOrcadDir = await materializeOrcadArtifact(context.serverTarget, { + signal: args.signal + }) + const result = await deployOrcad({ + ...managedOrcadSlot(context, deployment.remotePort, args.signal), + localOrcadDir, + target: context.serverTarget, + census, + force: args.force + }) + if (result.outcome === 'installed-not-activated') { + const deferral = { + outcome: 'deferred' as const, + candidateVersion: result.fullVersion, + code: result.code, + reason: result.reason, + forceable: isForceableOrcadDeferral(result.code) + } + recordManagedOrcadUpdateDeferral(environment.id, deferral) + return deferral + } + clearManagedOrcadUpdateDeferral(environment.id) + const readiness = await probeManagedOrcadReadiness( + context, + localOrcadDir, + result.fullVersion, + args.signal + ) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: result.outcome === 'already-active' ? 'already-current' : 'updated', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.fullVersion + } + }) +} + +export function rollbackManagedOrcadEnvironment( + userDataPath: string, + args: LifecycleArgs +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const record = context.activationRecord + const target = record.previous + if (!target) { + return { + outcome: 'refused', + code: 'orcad_rollback_no_target', + reason: 'This server has no previous version to roll back to.' + } + } + const census = await collectManagedTerminalCensus(userDataPath, environment, record) + // Why idle only: this client cannot read the older build's daemon protocol, so it cannot show + // that build would reach terminals that are still running. + if (census.liveSessions !== 0) { + return { + outcome: 'refused', + code: + census.liveSessions === null + ? 'orcad_rollback_census_unavailable' + : 'orcad_rollback_terminals_running', + reason: + census.liveSessions === null + ? 'The server did not answer how many terminals it runs. Retry when it answers.' + : 'Close the terminals running on this server before rolling it back.' + } + } + const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) + const targetDir = managedOrcadInstallDir(context, target) + const targetBuildHash = await readRemoteOrcadBuildHash(slot, targetDir) + const result = await rollbackOrcad({ + ...slot, + record, + census, + targetBuildHash, + targetDaemonProtocol: CURRENT_ORCAD_DAEMON_PROTOCOL + }) + if (result.outcome !== 'rolled-back') { + return result + } + const readiness = await probeActiveOrcadReadiness( + { ...slot, remoteInstallDir: targetDir }, + { buildHash: targetBuildHash, fullVersion: result.target } + ) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: 'rolled-back', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.target, + discarded: result.discarded + } + }) +} + +/** Finishes or undoes an interrupted activation, rollback or decommission on the host. */ +export function recoverManagedOrcadEnvironment( + userDataPath: string, + args: LifecycleArgs +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const result = await recoverInterruptedOrcadActivation( + managedOrcadSlot(context, deployment.remotePort, args.signal) + ) + if (result.outcome !== 'recovered') { + return result + } + const updated = result.readiness + ? refreshPairing(userDataPath, environment, result.readiness, deployment.localPort) + : environment + if (result.activeVersion) { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + } + return { + outcome: 'recovered', + resolution: result.resolution, + activeVersion: result.activeVersion, + environment: redactRuntimeEnvironment(updated) + } + }) +} diff --git a/src/main/ssh/orcad-runtime-status.ts b/src/main/ssh/orcad-runtime-status.ts index 639bd7a4199..101eb6d81e5 100644 --- a/src/main/ssh/orcad-runtime-status.ts +++ b/src/main/ssh/orcad-runtime-status.ts @@ -6,6 +6,8 @@ import { requireManagedOrcadEnvironment, resolveLinkedOrcadContext } from './orcad-managed-runtime-context' +import { readManagedOrcadUpdateDeferral } from './orcad-managed-update-deferrals' +import { collectManagedTerminalCensus } from './orcad-terminal-census-client' /** Read-only: what the host's activation record and journal say, without repairing either. */ export async function getManagedOrcadRuntimeStatus( @@ -30,7 +32,9 @@ export async function getManagedOrcadRuntimeStatus( previousVersion: record.previous, activatedAt: record.activatedAt, rollbackAvailable: Boolean(record.previous && record.snapshot), - recovery: transaction ? managedRecoveryStatus(transaction) : null + recovery: transaction ? managedRecoveryStatus(transaction) : null, + terminals: await collectManagedTerminalCensus(userDataPath, environment, record), + deferredUpdate: readManagedOrcadUpdateDeferral(environment.id) } }) } diff --git a/src/main/ssh/orcad-terminal-census-client.test.ts b/src/main/ssh/orcad-terminal-census-client.test.ts new file mode 100644 index 00000000000..feb3bb15cd9 --- /dev/null +++ b/src/main/ssh/orcad-terminal-census-client.test.ts @@ -0,0 +1,101 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY } from '../../shared/protocol-version' +import { + createEnvironmentFromPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { emptyOrcadActivationRecord } from './orcad-activation-record' + +const mocks = vi.hoisted(() => ({ send: vi.fn(), ensure: vi.fn() })) +vi.mock('../../shared/remote-runtime-client', () => ({ + sendRemoteRuntimeRequestWithStatusPreflight: mocks.send +})) +vi.mock('./orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: mocks.ensure })) + +const { collectManagedTerminalCensus, collectRemoteOrcadTerminalCensus } = + await import('./orcad-terminal-census-client') + +const environment: KnownRuntimeEnvironment = createEnvironmentFromPairingOffer({ + id: 'environment-1', + name: 'Managed', + now: 1, + offer: { v: 2, endpoint: 'ws://127.0.0.1:46768', deviceToken: 't', publicKeyB64: 'k' }, + connectionDependency: 'ssh-tunnel' +}) +const active = { + ...emptyOrcadActivationRecord(), + active: '1.0.0', + activatedAt: '2026-01-01T00:00:00.000Z' +} +const unverifiable = { + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null +} +const census = { liveSessions: 2, startedSinceActivation: 1, daemonProtocolVersion: 7 } + +function answerWith(capabilities: string[], response: unknown) { + mocks.send.mockImplementation(async (_pairing, _method, _params, _timeout, validate) => { + validate({ ok: true, result: { capabilities } }) + return response + }) +} + +describe('managed orcad terminal census client', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.ensure.mockResolvedValue(undefined) + }) + + it('reads an idle census without contacting a host that has nothing active', async () => { + await expect( + collectRemoteOrcadTerminalCensus(environment, { ...active, active: null }) + ).resolves.toEqual({ liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: null }) + expect(mocks.send).not.toHaveBeenCalled() + }) + + it('asks an advertising host with the activation time', async () => { + answerWith([ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY], { ok: true, result: census }) + await expect(collectRemoteOrcadTerminalCensus(environment, active)).resolves.toEqual(census) + expect(mocks.send.mock.calls[0]?.slice(1, 3)).toEqual([ + 'orcad.terminalCensus', + { activatedAt: Date.parse(active.activatedAt) } + ]) + }) + + it.each([ + ['an older host without the capability', () => answerWith([], { ok: true, result: census })], + [ + 'method not found', + () => + answerWith([ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY], { + ok: false, + error: { code: 'method_not_found', message: 'no' } + }) + ], + [ + 'a malformed answer', + () => answerWith([ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY], { ok: true, result: {} }) + ], + ['loss of contact', () => mocks.send.mockRejectedValue(new Error('socket closed'))] + ])('reads %s as unverifiable, never as zero', async (_label, arrange) => { + arrange() + await expect(collectRemoteOrcadTerminalCensus(environment, active)).resolves.toEqual( + unverifiable + ) + }) + + it('reads an unparseable activation time as unverifiable', async () => { + await expect( + collectRemoteOrcadTerminalCensus(environment, { ...active, activatedAt: 'later' }) + ).resolves.toEqual(unverifiable) + expect(mocks.send).not.toHaveBeenCalled() + }) + + it('reads a tunnel that cannot open as unverifiable', async () => { + mocks.ensure.mockRejectedValue(new Error('SSH unavailable')) + await expect(collectManagedTerminalCensus('/profile', environment, active)).resolves.toEqual( + unverifiable + ) + }) +}) diff --git a/src/main/ssh/orcad-terminal-census-client.ts b/src/main/ssh/orcad-terminal-census-client.ts new file mode 100644 index 00000000000..244589e9827 --- /dev/null +++ b/src/main/ssh/orcad-terminal-census-client.ts @@ -0,0 +1,84 @@ +/** + * Asks a managed orcad, through its tunnel, how many terminals its daemon runs. Every failure, + * including an older host without the method, reads as an unverifiable census, never as zero. + */ +import { + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, + ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY +} from '../../shared/protocol-version' +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { + ORCAD_TERMINAL_CENSUS_METHOD, + OrcadTerminalCensusSchema, + type OrcadTerminalCensus +} from '../../shared/orcad-terminal-census' +import { sendRemoteRuntimeRequestWithStatusPreflight } from '../../shared/remote-runtime-client' +import type { OrcadActivationRecord } from './orcad-activation-record' +import { ensureOrcadManagedTunnel } from './orcad-managed-tunnel' + +export const UNVERIFIABLE_ORCAD_TERMINAL_CENSUS: OrcadTerminalCensus = { + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null +} + +export async function collectRemoteOrcadTerminalCensus( + environment: KnownRuntimeEnvironment, + record: OrcadActivationRecord, + timeoutMs = 15_000 +): Promise { + if (!record.active) { + return collectIdle() + } + const activatedAt = record.activatedAt ? Date.parse(record.activatedAt) : Number.NaN + if (!Number.isFinite(activatedAt) || activatedAt < 0) { + return UNVERIFIABLE_ORCAD_TERMINAL_CENSUS + } + try { + const response = await sendRemoteRuntimeRequestWithStatusPreflight( + getPreferredPairingOffer(environment), + ORCAD_TERMINAL_CENSUS_METHOD, + { activatedAt }, + timeoutMs, + (status) => { + if ( + !status.ok || + !status.result.capabilities?.includes(ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY) + ) { + throw new Error('The managed Orca server does not report a terminal census.') + } + }, + undefined, + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES + ) + return response.ok + ? OrcadTerminalCensusSchema.parse(response.result) + : UNVERIFIABLE_ORCAD_TERMINAL_CENSUS + } catch { + return UNVERIFIABLE_ORCAD_TERMINAL_CENSUS + } +} + +/** The census through the server's ensured tunnel; a tunnel that cannot open is unverifiable. */ +export async function collectManagedTerminalCensus( + userDataPath: string, + environment: KnownRuntimeEnvironment, + record: OrcadActivationRecord +): Promise { + if (!record.active) { + return collectIdle() + } + try { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + } catch { + return UNVERIFIABLE_ORCAD_TERMINAL_CENSUS + } + return collectRemoteOrcadTerminalCensus(environment, record) +} + +function collectIdle(): OrcadTerminalCensus { + return { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: null } +} diff --git a/src/shared/orcad-managed-runtime.ts b/src/shared/orcad-managed-runtime.ts index 8f070d74838..85a13fb2dde 100644 --- a/src/shared/orcad-managed-runtime.ts +++ b/src/shared/orcad-managed-runtime.ts @@ -1,21 +1,64 @@ import type { PublicKnownRuntimeEnvironment } from './runtime-environments' +import type { OrcadTerminalCensus } from './orcad-terminal-census' export const ORCAD_MANAGED_REMOTE_PORT = 6_768 +export type OrcadManagedDeferral = { + outcome: 'deferred' + candidateVersion: string + code: string + reason: string + /** False when force cannot make the rejected lifecycle transition safe. */ + forceable?: boolean +} + export type OrcadManagedDeployResult = | { - outcome: 'created' | 'already-current' + outcome: 'created' | 'updated' | 'already-current' environment: PublicKnownRuntimeEnvironment activeVersion: string } + | OrcadManagedDeferral + +export type OrcadManagedRollbackResult = | { - outcome: 'deferred' - candidateVersion: string - code: string - reason: string - /** False when force cannot make the rejected lifecycle transition safe. */ - forceable?: boolean + outcome: 'rolled-back' + environment: PublicKnownRuntimeEnvironment + activeVersion: string + discarded: string[] } + | { outcome: 'refused' | 'failed'; code: string; reason: string } + +export type OrcadManagedRecoveryResult = + | { outcome: 'none' } + | { outcome: 'pending'; code: string; reason: string } + | { + outcome: 'recovered' + resolution: 'committed' | 'restored-incumbent' + activeVersion: string | null + environment: PublicKnownRuntimeEnvironment + } + | { outcome: 'refused'; verdict: 'live' | 'unverifiable'; code: string; reason: string } + +/** Only a proven `exited` unlinks the server locally; anything less keeps it linked. */ +export type OrcadManagedStopResult = + | { + outcome: 'unlinked' + verdict: 'exited' + environmentId: string + sshTargetId: string + stoppedVersion: string | null + retirement: 'retired' | 'live' | 'unverifiable' | null + } + | { outcome: 'refused'; verdict: 'live' | 'unverifiable'; code: string; reason: string } + +export type OrcadManagedCancelStopResult = + | { outcome: 'none' } + /** The stop was withdrawn before orcad acted on it; the server keeps serving. */ + | { outcome: 'canceled'; activeVersion: string } + /** orcad had already exited; finish with stop to unlink the server. */ + | { outcome: 'already-stopped' } + | { outcome: 'refused'; verdict: 'live' | 'unverifiable'; code: string; reason: string } export type OrcadManagedRuntimeStatus = { environmentId: string @@ -49,4 +92,8 @@ export type OrcadManagedRuntimeStatus = { startedAt: string } | null + /** Terminals the daemon runs; `null` counts are unverifiable and block updates and stops. */ + terminals: OrcadTerminalCensus + /** The last update this client deferred for this server, cleared once one goes through. */ + deferredUpdate: (OrcadManagedDeferral & { deferredAt: string }) | null } diff --git a/src/shared/orcad-terminal-census.ts b/src/shared/orcad-terminal-census.ts index c6a9bf23bee..11cdfb3449d 100644 --- a/src/shared/orcad-terminal-census.ts +++ b/src/shared/orcad-terminal-census.ts @@ -17,3 +17,11 @@ export const OrcadTerminalCensusSchema = z.object({ }) export type OrcadTerminalCensus = z.infer + +/** RPC a managed orcad answers with its census; clients call it only when advertised. */ +export const ORCAD_TERMINAL_CENSUS_METHOD = 'orcad.terminalCensus' + +export const OrcadTerminalCensusParamsSchema = z.object({ + /** The active version's activation time, epoch ms; sessions created at or after it count. */ + activatedAt: z.number().finite().nonnegative() +}) diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index 4d0c0ca6976..0e32913a337 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -345,7 +345,12 @@ export const ANTIGRAVITY_CONFIGURED_MODEL_RUNTIME_CAPABILITY = export const AGENT_SESSION_CREATE_TAB_ID_RUNTIME_CAPABILITY = 'agentSession.create.tab-id.v1' as const +// Why: older hosts answer orcad.terminalCensus with method-not-found, so a client asks only when +// this is advertised and otherwise treats the census as unverifiable, never as zero. +export const ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY = 'orcad.terminal-census.v1' as const + export const RUNTIME_CAPABILITIES = [ + ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY, ...AGENT_SESSION_STOP_RUNTIME_CAPABILITIES, AGENT_SESSION_CREATE_TAB_ID_RUNTIME_CAPABILITY, ANTIGRAVITY_CONFIGURED_MODEL_RUNTIME_CAPABILITY, diff --git a/src/shared/rpc-contract/rpc-params-catalog.generated.ts b/src/shared/rpc-contract/rpc-params-catalog.generated.ts index 87eeae77024..f0eeb856f14 100644 --- a/src/shared/rpc-contract/rpc-params-catalog.generated.ts +++ b/src/shared/rpc-contract/rpc-params-catalog.generated.ts @@ -23,6 +23,7 @@ import { PairingProvisionRelayParamsSchema } from '../mobile-relay-credential-contract' import { MobileWebBundleChunkParamsSchema } from '../mobile-web-bundle/bundle-rpc-contract' +import { OrcadTerminalCensusParamsSchema } from '../orcad-terminal-census' import { pluginConsentRequestSchema } from '../plugins/plugin-consent-request' import { AccountsUnsubscribeParams, @@ -986,6 +987,7 @@ export const RPC_PARAMS_BY_METHOD = { 'notifications.testPush': null, 'notifications.unregisterPush': null, 'notifications.unsubscribe': NotificationUnsubscribeParams, + 'orcad.terminalCensus': OrcadTerminalCensusParamsSchema, 'orchestration.ask': AskParams, 'orchestration.check': CheckParams, 'orchestration.dispatch': DispatchParams, diff --git a/src/shared/runtime-environment-managed-orcad-store.test.ts b/src/shared/runtime-environment-managed-orcad-store.test.ts index 4c5b08139ac..2458ad176fa 100644 --- a/src/shared/runtime-environment-managed-orcad-store.test.ts +++ b/src/shared/runtime-environment-managed-orcad-store.test.ts @@ -9,7 +9,11 @@ import { removeEnvironment, updateEnvironmentFromPairingCode } from './runtime-environment-store' -import { addManagedOrcadEnvironment } from './runtime-environment-managed-orcad-store' +import { + addManagedOrcadEnvironment, + refreshManagedOrcadPairing, + removeManagedOrcadEnvironment +} from './runtime-environment-managed-orcad-store' import { getRuntimeEnvironmentSidecarPath } from './runtime-environment-sidecar' import { getRuntimeSshAccess } from './runtime-environments' import { shippedBuildRewrite } from './runtime-environment-shipped-store-fixture' @@ -97,4 +101,33 @@ describe('managed orcad environment store', () => { ).toThrow('managed by Orca over SSH') expect(listEnvironments(userDataPath)).toHaveLength(1) }) + + it('removes the server and its deployment record together once it is unlinked', () => { + add() + removeManagedOrcadEnvironment(userDataPath, 'environment-1') + expect(listEnvironments(userDataPath)).toEqual([]) + expect(readFileSync(getRuntimeEnvironmentSidecarPath(userDataPath), 'utf8')).not.toContain( + 'environment-1' + ) + }) + + it('leaves an unchanged pairing alone and re-binds the link when the offer rotates', () => { + const environment = add() + expect(refreshManagedOrcadPairing(userDataPath, 'environment-1', pairingCode())).toEqual( + environment + ) + const rotated = encodePairingOffer({ + v: 2, + endpoint: 'ws://127.0.0.1:46768', + deviceToken: 'rotated-token', + publicKeyB64: Buffer.alloc(32, 1).toString('base64') + }) + const refreshed = refreshManagedOrcadPairing(userDataPath, 'environment-1', rotated, 500) + expect(refreshed.endpoints[0]?.deviceToken).toBe('rotated-token') + expect(refreshed.pairingRevision).toBe(500) + expect(refreshed.orcadDeployment).toEqual(deployment) + expect(() => + refreshManagedOrcadPairing(userDataPath, 'environment-1', pairingCode('ws://127.0.0.1:1')) + ).toThrow('does not point at its SSH tunnel') + }) }) diff --git a/src/shared/runtime-environment-managed-orcad-store.ts b/src/shared/runtime-environment-managed-orcad-store.ts index 91db33218f0..7ca7e1dd9b5 100644 --- a/src/shared/runtime-environment-managed-orcad-store.ts +++ b/src/shared/runtime-environment-managed-orcad-store.ts @@ -1,7 +1,8 @@ -import { parsePairingCode } from './pairing' +import { parsePairingCode, type PairingOffer } from './pairing' import { createEnvironmentFromPairingOffer, getPreferredLoopbackRuntimePort, + getPreferredPairingOffer, PersistedRuntimeEnvironmentSchema, type KnownRuntimeEnvironment, type OrcadDeploymentLink @@ -12,7 +13,11 @@ import { RuntimeEnvironmentStoreError, writeEnvironmentStore } from './runtime-environment-store-file' -import { writeRuntimeEnvironmentSidecarEntry } from './runtime-environment-sidecar' +import { + readCurrentRuntimeEnvironmentSidecarEntry, + writeRuntimeEnvironmentSidecarEntry +} from './runtime-environment-sidecar' +import { resolveEnvironmentFromStore } from './runtime-environment-store' /** Registers a server Orca deployed over SSH, paired through its own loopback tunnel. */ export function addManagedOrcadEnvironment( @@ -78,3 +83,79 @@ export function addManagedOrcadEnvironment( } return registered } + +/** + * Unlinks a managed server once its orcad is proven to have exited: drops the persisted entry + * and the sidecar record that held its deployment link. + */ +export function removeManagedOrcadEnvironment(userDataPath: string, environmentId: string): void { + const store = readPersistedEnvironmentStore(userDataPath) + const persisted = resolveEnvironmentFromStore(store, environmentId) + const remaining = store.environments.filter((entry) => entry.id !== persisted.id) + writeEnvironmentStore(userDataPath, { version: 1, environments: remaining }) + writeRuntimeEnvironmentSidecarEntry(userDataPath, remaining, persisted, null) +} + +/** Keeps a managed server's pairing current after its orcad changed versions. */ +export function refreshManagedOrcadPairing( + userDataPath: string, + environmentId: string, + pairingCode: string, + now = Date.now() +): KnownRuntimeEnvironment { + const offer = parsePairingCode(pairingCode) + if (!offer) { + throw new RuntimeEnvironmentStoreError('invalid_argument', 'Invalid managed pairing code.') + } + const current = resolveEnvironmentFromStore(readEnvironmentStore(userDataPath), environmentId) + const deployment = current.orcadDeployment + if (!deployment) { + throw new RuntimeEnvironmentStoreError('invalid_argument', 'This server is not managed.') + } + // Why skip: orcad keeps its pairing state across versions, so an unchanged offer needs no + // rewrite, and every rewrite re-binds the sidecar entry. + if (samePairing(getPreferredPairingOffer(current), offer)) { + return current + } + const store = readPersistedEnvironmentStore(userDataPath) + const existing = resolveEnvironmentFromStore(store, environmentId) + const entry = readCurrentRuntimeEnvironmentSidecarEntry(userDataPath, existing) + const next = PersistedRuntimeEnvironmentSchema.parse({ + ...createEnvironmentFromPairingOffer({ + id: existing.id, + name: existing.name, + now: existing.createdAt, + offer, + runtimeId: existing.runtimeId, + connectionDependency: 'ssh-tunnel' + }), + updatedAt: now, + pairingRevision: Math.max(now, (existing.pairingRevision ?? existing.createdAt) + 1), + lastUsedAt: existing.lastUsedAt + }) + if (getPreferredLoopbackRuntimePort(next) !== deployment.localPort) { + throw new RuntimeEnvironmentStoreError( + 'invalid_argument', + 'The managed pairing does not point at its SSH tunnel.' + ) + } + const environments = store.environments.map((candidate) => + candidate.id === existing.id ? next : candidate + ) + const { binding: _binding, ...state } = entry ?? {} + writeRuntimeEnvironmentSidecarEntry(userDataPath, environments, next, { + ...state, + orcadDeployment: deployment + }) + writeEnvironmentStore(userDataPath, { version: 1, environments }) + return resolveEnvironmentFromStore(readEnvironmentStore(userDataPath), environmentId) +} + +function samePairing(left: PairingOffer, right: PairingOffer): boolean { + return ( + left.endpoint === right.endpoint && + left.deviceToken === right.deviceToken && + left.publicKeyB64 === right.publicKeyB64 && + left.pairedDeviceId === right.pairedDeviceId + ) +}