diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8770700b818..0bf5b83812e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -343,7 +343,8 @@ jobs: . != "tests/e2e/terminal-ibus-hangul-native.spec.ts" and . != "tests/e2e/orcad-serve-mode-switch.spec.ts" and . != "tests/e2e/ssh-orcad-auto-convert.spec.ts" and - . != "tests/e2e/windows-missing-appdata-startup.spec.ts" + . != "tests/e2e/windows-missing-appdata-startup.spec.ts" and + . != "tests/e2e/ssh-orcad-idle-exit.spec.ts" )' <<<"$TEST_FILES_JSON" > "$RUNNER_TEMP/general-e2e-specs" fi mapfile -t TEST_FILES < "$RUNNER_TEMP/general-e2e-specs" @@ -594,12 +595,13 @@ jobs: retention-days: 7 if-no-files-found: ignore - # #24979 on a real host: a relay-era Docker host converts to managed orcad on connect. Needs the + # #24979 on a real host: a relay-era Docker host converts to managed orcad on connect, and a managed + # orcad idles out and restarts on the next connect. Needs the # orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds. orcad-auto-convert-docker: name: ssh host auto-converts to managed orcad (Docker) needs: [build, prepare-native-cache] - if: inputs.test_files == '' || inputs.ssh_source_changed == 'true' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-auto-convert.spec.ts') + if: inputs.test_files == '' || inputs.ssh_source_changed == 'true' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-auto-convert.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-idle-exit.spec.ts') runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -635,7 +637,7 @@ jobs: ORCA_RELAY_PATH: ${{ github.workspace }}/out/relay run: | export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template" - xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-auto-convert.spec.ts --project=electron-headless --workers=1 + xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-auto-convert.spec.ts tests/e2e/ssh-orcad-idle-exit.spec.ts --project=electron-headless --workers=1 - uses: actions/upload-artifact@v7 if: failure() with: diff --git a/config/scripts/ci-e2e-job-selection.mjs b/config/scripts/ci-e2e-job-selection.mjs index b31e68008eb..62050b9dbcb 100644 --- a/config/scripts/ci-e2e-job-selection.mjs +++ b/config/scripts/ci-e2e-job-selection.mjs @@ -43,6 +43,8 @@ export const ORCAD_SERVE_MODE_SWITCH_E2E_SPEC = 'tests/e2e/orcad-serve-mode-swit export const ORCAD_AUTO_CONVERT_E2E_SPEC = 'tests/e2e/ssh-orcad-auto-convert.spec.ts' // Windows-only; its own job runs it on a Windows runner. export const WINDOWS_MISSING_APPDATA_E2E_SPEC = 'tests/e2e/windows-missing-appdata-startup.spec.ts' +// Runs in the auto-convert job, which builds the template it needs. +export const ORCAD_IDLE_EXIT_E2E_SPEC = 'tests/e2e/ssh-orcad-idle-exit.spec.ts' export const DEDICATED_E2E_SPECS = [ ...DOCKER_SSH_E2E_SPECS, NODE_NETWORK_E2E_SPEC, @@ -50,7 +52,8 @@ export const DEDICATED_E2E_SPECS = [ NATIVE_IME_E2E_SPEC, ORCAD_SERVE_MODE_SWITCH_E2E_SPEC, ORCAD_AUTO_CONVERT_E2E_SPEC, - WINDOWS_MISSING_APPDATA_E2E_SPEC + WINDOWS_MISSING_APPDATA_E2E_SPEC, + ORCAD_IDLE_EXIT_E2E_SPEC ] const dedicatedSpecs = new Set(DEDICATED_E2E_SPECS) const dockerSpecs = new Set(DOCKER_SSH_E2E_SPECS) diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index bab26f13fd0..2284b0ca99c 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -38,6 +38,15 @@ export const PR_E2E_SOURCE_ROUTES = [ file ) }, + { + id: 'ssh.orcad-idle-exit', + specs: ['tests/e2e/ssh-orcad-idle-exit.spec.ts'], + matches: (file) => + isProductSource(file) && + /^src\/(?:main\/(?:orcad\/orcad-(?:idle-|managed-idle-)|ssh\/orcad-(?:managed-wake|managed-tunnel|recovery-slot|remote-launch))|shared\/orcad-idle-exit)/.test( + file + ) + }, { id: 'ssh.localhost-agent-hooks', specs: ['tests/e2e/ssh-localhost.spec.ts'], diff --git a/config/scripts/ssh-docker-ci-sharding.test.mjs b/config/scripts/ssh-docker-ci-sharding.test.mjs index 4afddbe6fed..39ca9e05d60 100644 --- a/config/scripts/ssh-docker-ci-sharding.test.mjs +++ b/config/scripts/ssh-docker-ci-sharding.test.mjs @@ -37,6 +37,7 @@ it('gives every removed SSH spec a dedicated owner even for test-only edits', () 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts', 'tests/e2e/ssh-localhost.spec.ts', 'tests/e2e/ssh-orcad-auto-convert.spec.ts', + 'tests/e2e/ssh-orcad-idle-exit.spec.ts', 'tests/e2e/terminal-ibus-hangul-native.spec.ts', 'tests/e2e/windows-missing-appdata-startup.spec.ts' ]) diff --git a/docs/reference/orcad-operations.md b/docs/reference/orcad-operations.md index 32133bce66d..e0a6dbe5296 100644 --- a/docs/reference/orcad-operations.md +++ b/docs/reference/orcad-operations.md @@ -301,6 +301,38 @@ What differs on a Windows SSH host, and what deliberately does not: versioned directories that nothing deletes while a process runs from them: Windows refuses to delete a running image, and GC treats an in-use slot as live. +## Idle exit (client-managed orcad only) + +An orcad that a desktop client launched over SSH stops itself, like the relay, once its host has +been unused for 15 minutes. The client's launch sets `ORCA_ORCAD_MANAGED_ACTIVATION_ROOT`; an +orcad started by hand, by a supervisor, or as a paired server never carries it and never idles +out. + +"Unused" means every one of these held on every check for the whole period: + +- no client socket open and no RPC request running; +- no terminal in the PTY provider, and the daemon answered with zero live sessions (a daemon + that does not answer keeps orcad up); +- no agent reporting `working`; +- no staged migration into this server; +- no activation fence on the host (an update, rollback, decommission or recovery in flight). + +The stop is the ordinary graceful shutdown, which disconnects from the daemon and never shuts it +down, so it cannot kill a terminal. It then asks the daemon to retire only if the daemon itself +proves it holds no session. Before stopping, orcad writes `/orcad-idle-stop.json`; +the next start reports it once as `health.previousIdleStop` and removes it, so a later crash is +never read as an idle stop. A managed start with no record reports `previousIdleStop: null`. + +The client starts a stopped server again, whatever stopped it (an idle stop, a kill, a host +reboot): on every connect, on every fresh tunnel (including after the client wakes from sleep), +and before a call through an environment the client restored at launch. A server that does not +answer is checked on the host; only a proven exit starts the activated slot, under the activation +fence, and the status line shows "Starting managed server…". A daemon that survived is adopted +with its terminals; after a reboot both start fresh. A process that is live or cannot be proven +gone is left alone, and a start that fails keeps the host managed with the reason and orcad.log's +tail, never as a verdict about its terminals. `ORCA_E2E_ORCAD_IDLE_TIMEOUT_MS` shortens the idle +period for tests; the client forwards it to the servers it launches. + ## Health The readiness payload carries a `health` object: diff --git a/src/main/ipc/runtime-environment-managed-tunnel.ts b/src/main/ipc/runtime-environment-managed-tunnel.ts index d8deb9066dc..fdd1f3f1024 100644 --- a/src/main/ipc/runtime-environment-managed-tunnel.ts +++ b/src/main/ipc/runtime-environment-managed-tunnel.ts @@ -1,7 +1,13 @@ import { resolveEnvironment } from '../../shared/runtime-environment-store' -import type { SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' +import type { + SshManagedServerStatus, + SshManagedServerUpdateNote, + SshTarget +} from '../../shared/ssh-types' import { managedServerUpdateDeps } from '../ssh/managed-server-update-deps' import { ensureOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' +import { verifyOrcadManagedServing } from '../ssh/orcad-managed-serving-verify' +import { setManagedOrcadStartListener } from '../ssh/orcad-managed-serving' import { updateManagedOrcadOnRestore } from '../ssh/orcad-managed-update-on-restore' import { setSshHostServerStatus } from '../ssh/ssh-host-server-status' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' @@ -14,6 +20,8 @@ export async function resolveManagedRuntimeEnvironment( ): Promise> { const environment = resolveEnvironment(userDataPath, selector) await ensureOrcadManagedTunnel(userDataPath, environment.id) + // Why: a server that stopped (idle, killed, host rebooted) starts before the call that needs it. + await verifyOrcadManagedServing(userDataPath, environment.id) // Why here: an auto-restored host may never see an SSH connect, so it would never update. void updateManagedOrcadOnRestore(environment.id, () => ({ ...managedServerUpdateDeps(userDataPath), @@ -32,12 +40,30 @@ function publishRestoreUpdate( phase: 'updating' | 'settled', note?: SshManagedServerUpdateNote ): void { - setSshHostServerStatus( - target.id, + publishHostServerStatus( + target, phase === 'updating' ? { kind: 'setting-up', phase: 'updating' } : { kind: 'managed', environmentId, ...(note ? { update: note } : {}) } ) +} + +/** Shows a managed server's start on the host's status line, wherever the start began. */ +export function installManagedOrcadStartStatus(): void { + setManagedOrcadStartListener({ + starting: (target) => + publishHostServerStatus(target, { kind: 'setting-up', phase: 'starting' }), + settled: (target, environmentId, serving) => + publishHostServerStatus(target, { + kind: 'managed', + environmentId, + ...(serving.state === 'unverifiable' ? { serving } : {}) + }) + }) +} + +function publishHostServerStatus(target: SshTarget, status: SshManagedServerStatus): void { + setSshHostServerStatus(target.id, status) // Only a host with a connection state has a status line to refresh. const state = connectionManager?.getState(target.id) if (state) { diff --git a/src/main/ipc/ssh-connect-flow.ts b/src/main/ipc/ssh-connect-flow.ts index 1e36532c850..d2f16a07a8f 100644 --- a/src/main/ipc/ssh-connect-flow.ts +++ b/src/main/ipc/ssh-connect-flow.ts @@ -179,7 +179,12 @@ async function doConnect( throw createCancelledConnectAttemptError() } if (server?.route === 'managed') { - return publishManagedServerConnect(targetId, server.environmentId, server.update) + return publishManagedServerConnect( + targetId, + server.environmentId, + server.update, + server.serving + ) } if (server) { recordRelayDecision(target, server) diff --git a/src/main/ipc/ssh-host-server-connect.ts b/src/main/ipc/ssh-host-server-connect.ts index 45b7ea00051..ff7735f7d79 100644 --- a/src/main/ipc/ssh-host-server-connect.ts +++ b/src/main/ipc/ssh-host-server-connect.ts @@ -2,6 +2,7 @@ import { getAppEnvironment } from '../../shared/app-environment' import type { SshConnectionState, + SshManagedServerServingNote, SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' @@ -38,9 +39,15 @@ export async function decideHostServer( export function publishManagedServerConnect( targetId: string, environmentId: string, - update?: SshManagedServerUpdateNote + update?: SshManagedServerUpdateNote, + serving?: SshManagedServerServingNote ): SshConnectionState { - const managedServer = { kind: 'managed' as const, environmentId, ...(update ? { update } : {}) } + const managedServer = { + kind: 'managed' as const, + environmentId, + ...(update ? { update } : {}), + ...(serving ? { serving } : {}) + } setSshHostServerStatus(targetId, managedServer) const state: SshConnectionState = { ...(connectionManager!.getState(targetId) ?? { targetId, reconnectAttempt: 0 }), diff --git a/src/main/ipc/ssh-host-server-on-connect-wiring.ts b/src/main/ipc/ssh-host-server-on-connect-wiring.ts index fa78cd2e02f..bbd72437f31 100644 --- a/src/main/ipc/ssh-host-server-on-connect-wiring.ts +++ b/src/main/ipc/ssh-host-server-on-connect-wiring.ts @@ -14,6 +14,7 @@ import { assessOrcadMigrationTerminals } from '../ssh/orcad-migration-terminal-g import { hasOrcadTemplate } from '../ssh/orcad-artifact-materializer' import { managedServerUpdateDeps } from '../ssh/managed-server-update-deps' import { ensureOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' +import { verifyOrcadManagedServing } from '../ssh/orcad-managed-serving-verify' import { convertSshTargetToManagedOrcad } from '../ssh/orcad-runtime-conversion' import { orcadMigrationDestinationFor } from '../ssh/orcad-runtime-conversion-wiring' import { createManagedOrcadEnvironment } from '../ssh/orcad-runtime-deployment' @@ -49,6 +50,7 @@ export function hostServerOnConnectDeps(userDataPath: string): HostServerOnConne ensureTunnel: async (environmentId) => { await ensureOrcadManagedTunnel(userDataPath, environmentId) }, + ensureServing: (environmentId) => verifyOrcadManagedServing(userDataPath, environmentId), retireRetainedSource: async (target) => { if (isOrcadSourceRetirementEnabled()) { await retireRetainedOrcadSourceChain(userDataPath, store, target, runTargetLifecycle) diff --git a/src/main/ipc/ssh.ts b/src/main/ipc/ssh.ts index 9c34afb4fd9..7f97f2d3577 100644 --- a/src/main/ipc/ssh.ts +++ b/src/main/ipc/ssh.ts @@ -41,6 +41,7 @@ import { } from '../ssh/ssh-connection-generation' import { resetSshProviderAuthorities } from '../ssh/ssh-provider-authority' import { activeSessions } from './ssh-active-relay-sessions' +import { installManagedOrcadStartStatus } from './runtime-environment-managed-tunnel' import { registerAdvertisedUrlRefresh, unregisterAdvertisedUrlRefresh @@ -190,6 +191,7 @@ export function registerSshHandlers( setPersistedStore(store) reconcileManagedOrcadSshTargets(getAppEnvironment().getPath('userData'), store) registerAdvertisedUrlRefresh(getCurrentMainWindow) + installManagedOrcadStartStatus() registerCredentialHandler() diff --git a/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts b/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts index a80a6c9b9ec..8e3e2d3be9a 100644 --- a/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts +++ b/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts @@ -179,7 +179,8 @@ async function launch(slot: Slot, userDataDir: string): Promise fullVersion: slot.version, userDataDir, bindHost: '127.0.0.1', - port: 0 + port: 0, + activationRoot: join(userDataDir, '.orcad-activation-transaction') }) ) ).trim() diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index ea311efb7d8..c72fb0eab2b 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -26,6 +26,7 @@ import type { OrcadRuntimeCleanup } from './orcad-runtime-lifetime' import { installOrcadStopRequestListeners } from './orcad-stop-request-listener' import { prepareOrcadManagedStop } from './orcad-managed-stop-admission' import type { OrcadManagedStopContext } from '../../shared/orcad-stop-request' +import { beginOrcadIdleExit, bindOrcadIdleShutdown } from './orcad-managed-idle-exit-host' import { changedAiVaultSearchSettings, type AiVaultSearchSettings @@ -191,6 +192,7 @@ async function startOrcadRuntime( const { resolvePushGatewayOrigin } = await import('../runtime/push/push-gateway-origin') const runtimeUserDataPath = getAppEnvironment().getPath('userData') + const idleExitStartup = beginOrcadIdleExit(runtimeUserDataPath) const { store: profileStore, authority: profileStateAuthority } = await createOrcadProfileStateStartup(runtimeUserDataPath) const observedPaneIdentities = new AgentStatusObservedPaneIdentities() @@ -386,7 +388,11 @@ async function startOrcadRuntime( // Why in the readiness payload: this is the one message a supervisor and a deploy // transaction both read, and a green orcad with a dead daemon is exactly the // looks-healthy-but-useless state they must not activate. - health: await collectOrcadHealth(getAppEnvironment().getVersion(), profileStateAuthority) + health: await collectOrcadHealth( + getAppEnvironment().getVersion(), + profileStateAuthority, + idleExitStartup.previousIdleStop + ) } await new ServeReadinessPublisher().publish( @@ -396,6 +402,12 @@ async function startOrcadRuntime( : { mode: options.json ? 'json' : 'human' } ) + await idleExitStartup.start({ + rpc, + agentStates: () => agentHookServer.getStatusSnapshot(), + hasStagedMigration: () => profileStore.hasStagedOrcadMigrationCatalog(), + registerCleanup + }) return { readiness } } @@ -427,4 +439,5 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise { return { buildHash: computeOrcadBuildHash(), @@ -173,6 +180,7 @@ export async function collectOrcadHealth( pid: process.pid, terminalDaemon: await collectTerminalDaemonHealth(), ...(profileStateAuthority ? { profileStateAuthority } : {}), - stopRequests: ORCAD_STOP_REQUESTS_CAPABILITY + stopRequests: ORCAD_STOP_REQUESTS_CAPABILITY, + ...(previousIdleStop !== undefined ? { previousIdleStop } : {}) } } diff --git a/src/main/orcad/orcad-idle-exit-monitor.test.ts b/src/main/orcad/orcad-idle-exit-monitor.test.ts new file mode 100644 index 00000000000..fbe31074a52 --- /dev/null +++ b/src/main/orcad/orcad-idle-exit-monitor.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it, vi } from 'vitest' +import { + OrcadIdleExitMonitor, + resolveOrcadIdlePollMs, + type OrcadIdleProbe, + type OrcadIdleVerdict +} from './orcad-idle-exit-monitor' + +function harness(options: { timeoutMs?: number; lastClientActivityAt?: number } = {}) { + let now = 1_000 + let lastActivity = options.lastClientActivityAt ?? 0 + const verdicts: Record = { clients: 'idle', terminals: 'idle' } + const probes: OrcadIdleProbe[] = Object.keys(verdicts).map((name) => ({ + name, + read: () => { + const verdict = verdicts[name] + if (verdict instanceof Error) { + throw verdict + } + return verdict + } + })) + const onIdle = vi.fn() + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: options.timeoutMs ?? 100, + probes, + lastClientActivityAt: () => lastActivity, + onIdle, + now: () => now, + log: () => {} + }) + return { + monitor, + onIdle, + verdicts, + advance: (ms: number) => (now += ms), + touch: () => (lastActivity = now) + } +} + +describe('OrcadIdleExitMonitor', () => { + it('fires once every probe has stayed idle for the whole quiet period', async () => { + const h = harness() + expect(await h.monitor.check()).toBe(false) + h.advance(99) + expect(await h.monitor.check()).toBe(false) + h.advance(1) + expect(await h.monitor.check()).toBe(true) + expect(h.onIdle).toHaveBeenCalledWith({ quietSince: 1_000, stoppedAt: 1_100, timeoutMs: 100 }) + h.advance(1_000) + expect(await h.monitor.check()).toBe(false) + expect(h.onIdle).toHaveBeenCalledTimes(1) + }) + + it('restarts the quiet period whenever any probe is busy', async () => { + const h = harness() + await h.monitor.check() + h.advance(90) + h.verdicts.terminals = 'busy' + expect(await h.monitor.check()).toBe(false) + h.verdicts.terminals = 'idle' + h.advance(10) + expect(await h.monitor.check()).toBe(false) + h.advance(99) + expect(await h.monitor.check()).toBe(false) + h.advance(1) + expect(await h.monitor.check()).toBe(true) + }) + + it.each([ + ['unverifiable', 'unverifiable' as const], + ['throwing', new Error('daemon did not answer')] + ])('treats a %s probe as busy, never as idle', async (_label, verdict) => { + const h = harness() + h.verdicts.terminals = verdict + for (let i = 0; i < 5; i += 1) { + expect(await h.monitor.check()).toBe(false) + h.advance(100) + } + expect(h.onIdle).not.toHaveBeenCalled() + }) + + it('counts a request that came and went between checks as activity', async () => { + const h = harness() + await h.monitor.check() + h.advance(80) + h.touch() + h.advance(20) + expect(await h.monitor.check()).toBe(false) + h.advance(80) + expect(await h.monitor.check()).toBe(true) + }) + + it('does not fire after it was stopped', async () => { + const h = harness() + await h.monitor.check() + h.monitor.stop() + h.advance(1_000) + expect(await h.monitor.check()).toBe(false) + expect(h.onIdle).not.toHaveBeenCalled() + }) + + it('polls often enough for a short test timeout and at most once a minute', () => { + expect(resolveOrcadIdlePollMs(15 * 60_000)).toBe(60_000) + expect(resolveOrcadIdlePollMs(2_000)).toBe(400) + expect(resolveOrcadIdlePollMs(10)).toBe(250) + }) + + it('stops itself on a timer once idle', async () => { + vi.useFakeTimers() + try { + const onIdle = vi.fn() + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: 1_000, + probes: [{ name: 'clients', read: () => 'idle' }], + lastClientActivityAt: () => 0, + onIdle, + log: () => {} + }) + monitor.start() + await vi.advanceTimersByTimeAsync(1_600) + expect(onIdle).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(5_000) + expect(onIdle).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/orcad/orcad-idle-exit-monitor.ts b/src/main/orcad/orcad-idle-exit-monitor.ts new file mode 100644 index 00000000000..58a053adffc --- /dev/null +++ b/src/main/orcad/orcad-idle-exit-monitor.ts @@ -0,0 +1,116 @@ +/** + * Decides when a managed orcad has been unused long enough to stop. + * + * Every probe must answer `idle` on the same check, continuously for the whole quiet period. + * A probe that throws or cannot answer counts as busy: silence is never evidence of idleness. + */ + +export type OrcadIdleVerdict = 'idle' | 'busy' | 'unverifiable' + +export type OrcadIdleProbe = { + name: string + read: () => OrcadIdleVerdict | Promise +} + +export type OrcadIdleExitEvidence = { quietSince: number; stoppedAt: number; timeoutMs: number } + +export type OrcadIdleExitMonitorOptions = { + timeoutMs: number + probes: readonly OrcadIdleProbe[] + /** Any client request restarts the quiet period, even one that came and went between checks. */ + lastClientActivityAt: () => number + onIdle: (evidence: OrcadIdleExitEvidence) => void + now?: () => number + pollMs?: number + log?: (line: string) => void +} + +export function resolveOrcadIdlePollMs(timeoutMs: number): number { + return Math.min(60_000, Math.max(250, Math.floor(timeoutMs / 5))) +} + +export class OrcadIdleExitMonitor { + private timer: ReturnType | null = null + private quietSince: number | null = null + private blocker: string | null = null + private stopped = false + private readonly now: () => number + private readonly pollMs: number + private readonly log: (line: string) => void + + constructor(private readonly options: OrcadIdleExitMonitorOptions) { + this.now = options.now ?? Date.now + this.pollMs = options.pollMs ?? resolveOrcadIdlePollMs(options.timeoutMs) + this.log = options.log ?? ((line) => console.error(line)) + } + + start(): void { + this.schedule() + } + + stop(): void { + this.stopped = true + if (this.timer) { + clearTimeout(this.timer) + this.timer = null + } + } + + /** One check; resolves true once the quiet period elapsed and `onIdle` fired. */ + async check(): Promise { + const blocker = await this.findBlocker() + if (this.stopped) { + return false + } + const now = this.now() + if (blocker) { + if (this.quietSince !== null || this.blocker !== blocker) { + this.log(`[orcad] idle exit waiting: ${blocker}`) + } + this.quietSince = null + this.blocker = blocker + return false + } + if (this.quietSince === null) { + this.quietSince = now + this.blocker = null + this.log(`[orcad] idle; stopping after ${this.options.timeoutMs}ms unless a client returns`) + } + const quietSince = Math.max(this.quietSince, this.options.lastClientActivityAt()) + if (now - quietSince < this.options.timeoutMs) { + return false + } + this.stop() + this.options.onIdle({ quietSince, stoppedAt: now, timeoutMs: this.options.timeoutMs }) + return true + } + + private async findBlocker(): Promise { + for (const probe of this.options.probes) { + let verdict: OrcadIdleVerdict + try { + verdict = await probe.read() + } catch { + verdict = 'unverifiable' + } + if (verdict !== 'idle') { + return `${probe.name} ${verdict}` + } + } + return null + } + + private schedule(): void { + if (this.stopped) { + return + } + this.timer = setTimeout(() => { + this.timer = null + void this.check() + .catch((error: unknown) => this.log(`[orcad] idle check failed: ${String(error)}`)) + .finally(() => this.schedule()) + }, this.pollMs) + // Never the reason the process stays alive. + this.timer.unref?.() + } +} diff --git a/src/main/orcad/orcad-idle-stop-record.test.ts b/src/main/orcad/orcad-idle-stop-record.test.ts new file mode 100644 index 00000000000..2e5c5659f7e --- /dev/null +++ b/src/main/orcad/orcad-idle-stop-record.test.ts @@ -0,0 +1,57 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + consumeOrcadIdleStopRecord, + orcadIdleStopRecordPath, + writeOrcadIdleStopRecord +} from './orcad-idle-stop-record' + +let root: string + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orcad-idle-stop-')) +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) + vi.restoreAllMocks() +}) + +describe('orcad idle-stop record', () => { + it('lets the next start tell an idle stop apart from a crash, exactly once', () => { + writeOrcadIdleStopRecord( + root, + { + quietSince: Date.parse('2026-10-03T10:00:00Z'), + stoppedAt: Date.parse('2026-10-03T10:15:00Z'), + timeoutMs: 900_000 + }, + '1.2.3' + ) + + expect(consumeOrcadIdleStopRecord(root)).toMatchObject({ + kind: 'orcad_idle_stop', + pid: process.pid, + version: '1.2.3', + quietSince: '2026-10-03T10:00:00.000Z', + stoppedAt: '2026-10-03T10:15:00.000Z', + idleTimeoutMs: 900_000 + }) + // A crash after this start must not inherit the earlier idle stop. + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + }) + + it('reads a start with no record as "not an idle stop"', () => { + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + }) + + it('drops a corrupt record instead of trusting it', () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + writeFileSync(orcadIdleStopRecordPath(root), '{"kind":"orcad_idle_stop"') + + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + expect(existsSync(orcadIdleStopRecordPath(root))).toBe(false) + }) +}) diff --git a/src/main/orcad/orcad-idle-stop-record.ts b/src/main/orcad/orcad-idle-stop-record.ts new file mode 100644 index 00000000000..3656d09415d --- /dev/null +++ b/src/main/orcad/orcad-idle-stop-record.ts @@ -0,0 +1,60 @@ +/** + * The record that tells a clean idle stop apart from a crash. Written just before an idle + * stop; the next start reports it once and removes it, so a later crash never inherits it. + */ +import { rmSync } from 'node:fs' +import { join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { writeDurableSecureJsonFile } from '../../shared/secure-file' +import { + ORCAD_IDLE_STOP_RECORD_FILENAME, + OrcadIdleStopRecordSchema, + type OrcadIdleStopRecord +} from '../../shared/orcad-idle-exit' +import type { OrcadIdleExitEvidence } from './orcad-idle-exit-monitor' + +const RECORD_MAX_BYTES = 16 * 1024 + +export function orcadIdleStopRecordPath(userDataPath: string): string { + return join(userDataPath, ORCAD_IDLE_STOP_RECORD_FILENAME) +} + +export function writeOrcadIdleStopRecord( + userDataPath: string, + evidence: OrcadIdleExitEvidence, + version: string +): void { + const record: OrcadIdleStopRecord = { + schemaVersion: 1, + kind: 'orcad_idle_stop', + pid: process.pid, + version, + quietSince: new Date(evidence.quietSince).toISOString(), + stoppedAt: new Date(evidence.stoppedAt).toISOString(), + idleTimeoutMs: evidence.timeoutMs + } + if (!writeDurableSecureJsonFile(orcadIdleStopRecordPath(userDataPath), record)) { + throw new Error('orcad_idle_stop_record_permissions_unconfirmed') + } +} + +/** The previous run's idle stop, or null when it ended any other way (or never ran). */ +export function consumeOrcadIdleStopRecord(userDataPath: string): OrcadIdleStopRecord | null { + const path = orcadIdleStopRecordPath(userDataPath) + let record: OrcadIdleStopRecord | null = null + try { + const raw = readNodeFileSyncWithinLimit(path, RECORD_MAX_BYTES).buffer.toString('utf8') + record = OrcadIdleStopRecordSchema.parse(JSON.parse(raw)) + } catch (error) { + if (isMissing(error)) { + return null + } + console.error('[orcad] ignoring an unreadable idle-stop record:', error) + } + rmSync(path, { force: true }) + return record +} + +function isMissing(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT' +} diff --git a/src/main/orcad/orcad-managed-idle-exit-host.ts b/src/main/orcad/orcad-managed-idle-exit-host.ts new file mode 100644 index 00000000000..022e2fda01b --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit-host.ts @@ -0,0 +1,101 @@ +/** Binds managed idle exit to this orcad's RPC server, PTY provider and terminal daemon. */ +import type { RuntimeRpcClientActivity } from '../runtime/runtime-rpc/runtime-rpc-shutdown' +import type { OrcadIdleExitEvidence } from './orcad-idle-exit-monitor' +import { + activationFenceExists, + installOrcadManagedIdleExit, + resolveOrcadManagedIdleExit, + type OrcadManagedIdleExitConfig +} from './orcad-managed-idle-exit' +import { consumeOrcadIdleStopRecord, writeOrcadIdleStopRecord } from './orcad-idle-stop-record' +import type { OrcadIdleStopRecord } from '../../shared/orcad-idle-exit' + +let requestIdleShutdown: ((reason: string) => void) | null = null + +/** main binds its shutdown once signal handling exists; the quiet period outlasts that gap. */ +export function bindOrcadIdleShutdown(request: (reason: string) => void): void { + requestIdleShutdown = request +} + +type OrcadIdleExitRuntimePorts = { + rpc: { readClientActivity(): RuntimeRpcClientActivity } + agentStates: () => readonly { state: string }[] + hasStagedMigration: () => boolean + /** Shutdown stops the monitor first, so a signal stop is never recorded as an idle one. */ + registerCleanup: (cleanup: () => void) => void +} + +/** + * Runs under the instance lock at startup: reads the previous run's idle-stop record before + * anything this run does could be mistaken for it. Inert for an orcad no client launched. + */ +export function beginOrcadIdleExit(userDataPath: string): { + previousIdleStop: OrcadIdleStopRecord | null | undefined + start: (ports: OrcadIdleExitRuntimePorts) => Promise +} { + const config = resolveOrcadManagedIdleExit(process.env) + if (!config) { + return { previousIdleStop: undefined, start: async () => {} } + } + const previousIdleStop = consumeOrcadIdleStopRecord(userDataPath) + if (previousIdleStop) { + console.error(`[orcad] the previous run stopped idle at ${previousIdleStop.stoppedAt}`) + } + const version = process.env.ORCA_VERSION ?? '0.0.0-orcad' + return { + previousIdleStop, + start: (ports) => startOrcadManagedIdleExit({ ...ports, config, userDataPath, version }) + } +} + +async function startOrcadManagedIdleExit( + input: OrcadIdleExitRuntimePorts & { + config: OrcadManagedIdleExitConfig + userDataPath: string + version: string + } +): Promise { + const { getLocalPtyProvider } = await import('../ipc/pty') + const { getDaemonEndpointFacts } = await import('../daemon/daemon-init') + const { countLiveOrcadDaemonSessions, retireOrcadDaemonIfIdle } = + await import('./orcad-daemon-retirement') + const dispose = installOrcadManagedIdleExit({ + config: input.config, + ports: { + readClientActivity: () => input.rpc.readClientActivity(), + listTerminals: () => getLocalPtyProvider().listProcesses(), + countDaemonSessions: countLiveOrcadDaemonSessions, + hasDaemon: () => getDaemonEndpointFacts() !== null, + agentStates: input.agentStates, + hasStagedMigration: input.hasStagedMigration, + activationFenceExists + }, + stop: (evidence) => { + void stopForIdle(input, evidence, retireOrcadDaemonIfIdle).finally(() => + requestIdleShutdown?.('idle') + ) + } + }) + input.registerCleanup(dispose) +} + +async function stopForIdle( + input: { userDataPath: string; version: string }, + evidence: OrcadIdleExitEvidence, + retireDaemon: () => Promise<{ retirement: string; reason: string | null }> +): Promise { + console.error(`[orcad] stopping: no client, terminal or job for ${evidence.timeoutMs}ms`) + try { + writeOrcadIdleStopRecord(input.userDataPath, evidence, input.version) + } catch (error) { + console.error('[orcad] could not record the idle stop:', error) + } + // The daemon leaves only if it proves itself empty; a busy one stays up with its terminals. + const outcome = await retireDaemon().catch((error: unknown) => ({ + retirement: 'unverifiable', + reason: String(error) + })) + console.error( + `[orcad] terminal daemon on idle stop: ${outcome.retirement}${outcome.reason ? ` (${outcome.reason})` : ''}` + ) +} diff --git a/src/main/orcad/orcad-managed-idle-exit.test.ts b/src/main/orcad/orcad-managed-idle-exit.test.ts new file mode 100644 index 00000000000..f3f8ed10e74 --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit.test.ts @@ -0,0 +1,134 @@ +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + activationFenceExists, + createOrcadIdleProbes, + resolveOrcadManagedIdleExit, + type OrcadManagedIdleExitPorts +} from './orcad-managed-idle-exit' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_IDLE_EXIT_TIMEOUT_MS, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV +} from '../../shared/orcad-idle-exit' + +const config = { timeoutMs: 1_000, activationRoot: '/home/u/.orca-remote/.fence' } + +function idlePorts(): OrcadManagedIdleExitPorts { + return { + readClientActivity: () => ({ openConnections: 0, requestsInFlight: 0, lastRequestAt: 0 }), + listTerminals: async () => [], + countDaemonSessions: async () => 0, + hasDaemon: () => true, + agentStates: () => [{ state: 'done' }], + hasStagedMigration: () => false, + activationFenceExists: async () => false + } +} + +async function verdicts(ports: OrcadManagedIdleExitPorts): Promise> { + const entries = await Promise.all( + createOrcadIdleProbes(config, ports).map(async (probe) => [probe.name, await probe.read()]) + ) + return Object.fromEntries(entries) +} + +describe('resolveOrcadManagedIdleExit', () => { + it('stays off for a server the user started or paired', () => { + expect(resolveOrcadManagedIdleExit({})).toBeNull() + expect(resolveOrcadManagedIdleExit({ [ORCAD_E2E_IDLE_TIMEOUT_ENV]: '50' })).toBeNull() + }) + + it('matches the relay quiet period for a managed launch', () => { + expect(resolveOrcadManagedIdleExit({ [ORCAD_MANAGED_ACTIVATION_ROOT_ENV]: '/f' })).toEqual({ + timeoutMs: ORCAD_IDLE_EXIT_TIMEOUT_MS, + activationRoot: '/f' + }) + expect(ORCAD_IDLE_EXIT_TIMEOUT_MS).toBe(15 * 60_000) + }) + + it.each([ + ['3000', 3_000], + ['0', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['-1', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['1.5', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['abc', ORCAD_IDLE_EXIT_TIMEOUT_MS], + [String(2 * 60 * 60_000), ORCAD_IDLE_EXIT_TIMEOUT_MS] + ])('accepts only a bounded test timeout (%s)', (raw, expected) => { + expect( + resolveOrcadManagedIdleExit({ + [ORCAD_MANAGED_ACTIVATION_ROOT_ENV]: '/f', + [ORCAD_E2E_IDLE_TIMEOUT_ENV]: raw + })?.timeoutMs + ).toBe(expected) + }) +}) + +describe('createOrcadIdleProbes', () => { + it('reads an unused host as idle on every probe', async () => { + expect(await verdicts(idlePorts())).toEqual({ + clients: 'idle', + terminals: 'idle', + agents: 'idle', + migration: 'idle', + activation: 'idle' + }) + }) + + it.each<[string, Partial, string]>([ + [ + 'an open client socket', + { + readClientActivity: () => ({ openConnections: 1, requestsInFlight: 0, lastRequestAt: 0 }) + }, + 'clients' + ], + [ + 'a request still running', + { + readClientActivity: () => ({ openConnections: 0, requestsInFlight: 1, lastRequestAt: 0 }) + }, + 'clients' + ], + ['an in-process terminal', { listTerminals: async () => [{ id: 'pty-1' }] }, 'terminals'], + ['a live daemon session', { countDaemonSessions: async () => 2 }, 'terminals'], + ['a working agent', { agentStates: () => [{ state: 'working' }] }, 'agents'], + ['a staged migration', { hasStagedMigration: () => true }, 'migration'], + ['a held activation fence', { activationFenceExists: async () => true }, 'activation'] + ])('reads %s as busy', async (_label, override, probe) => { + expect((await verdicts({ ...idlePorts(), ...override }))[probe]).toBe('busy') + }) + + it('treats a daemon that did not answer as unverifiable, not as no terminals', async () => { + const result = await verdicts({ ...idlePorts(), countDaemonSessions: async () => null }) + expect(result.terminals).toBe('unverifiable') + }) + + it('needs only the provider census when this orcad runs without a daemon', async () => { + const result = await verdicts({ + ...idlePorts(), + hasDaemon: () => false, + countDaemonSessions: async () => null + }) + expect(result.terminals).toBe('idle') + }) +}) + +describe('activationFenceExists', () => { + let root: string | null = null + afterEach(() => { + if (root) { + rmSync(root, { recursive: true, force: true }) + } + }) + + it('follows the fence directory a client holds during an update', async () => { + root = mkdtempSync(join(tmpdir(), 'orcad-fence-')) + const fence = join(root, '.orcad-activation-transaction') + expect(await activationFenceExists(fence)).toBe(false) + mkdirSync(fence) + expect(await activationFenceExists(fence)).toBe(true) + }) +}) diff --git a/src/main/orcad/orcad-managed-idle-exit.ts b/src/main/orcad/orcad-managed-idle-exit.ts new file mode 100644 index 00000000000..857e55e3f91 --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit.ts @@ -0,0 +1,116 @@ +/** + * What a managed orcad counts as "in use" before an idle stop, and the stop itself. + * + * The stop is the ordinary graceful shutdown, which disconnects from the terminal daemon and + * never shuts it down, so no terminal can be killed by it. Terminals are still a blocker: a + * host with live terminals keeps its server so a returning client finds it serving. + */ +import { stat } from 'node:fs/promises' +import type { RuntimeRpcClientActivity } from '../runtime/runtime-rpc/runtime-rpc-shutdown' +import { + ORCAD_MANAGED_ACTIVATION_ROOT_ENV, + ORCAD_IDLE_EXIT_TIMEOUT_MS, + readOrcadE2EIdleTimeoutMs +} from '../../shared/orcad-idle-exit' +import { + OrcadIdleExitMonitor, + type OrcadIdleExitEvidence, + type OrcadIdleProbe, + type OrcadIdleVerdict +} from './orcad-idle-exit-monitor' + +export type OrcadManagedIdleExitConfig = { timeoutMs: number; activationRoot: string } + +/** Null for any orcad a client did not launch: a user-started or paired server never idles out. */ +export function resolveOrcadManagedIdleExit( + env: NodeJS.ProcessEnv +): OrcadManagedIdleExitConfig | null { + const activationRoot = env[ORCAD_MANAGED_ACTIVATION_ROOT_ENV] + if (!activationRoot) { + return null + } + return { + timeoutMs: readOrcadE2EIdleTimeoutMs(env) ?? ORCAD_IDLE_EXIT_TIMEOUT_MS, + activationRoot + } +} + +export type OrcadManagedIdleExitPorts = { + readClientActivity: () => RuntimeRpcClientActivity + /** Every terminal the PTY provider knows, daemon-owned or in-process. */ + listTerminals: () => Promise + /** Live daemon sessions across generations; null when a daemon did not answer. */ + countDaemonSessions: () => Promise + hasDaemon: () => boolean + agentStates: () => readonly { state: string }[] + hasStagedMigration: () => boolean + /** Exists while a client holds the host's activation fence (update, rollback, decommission). */ + activationFenceExists: (root: string) => Promise +} + +export function createOrcadIdleProbes( + config: OrcadManagedIdleExitConfig, + ports: OrcadManagedIdleExitPorts +): OrcadIdleProbe[] { + const verdict = (busy: boolean): OrcadIdleVerdict => (busy ? 'busy' : 'idle') + return [ + { + name: 'clients', + read: () => { + const activity = ports.readClientActivity() + return verdict(activity.openConnections > 0 || activity.requestsInFlight > 0) + } + }, + { + name: 'terminals', + read: async () => { + if ((await ports.listTerminals()).length > 0) { + return 'busy' + } + // Why read the daemon too: it outlives this process and may hold sessions no provider lists. + if (!ports.hasDaemon()) { + return 'idle' + } + const live = await ports.countDaemonSessions() + return live === null ? 'unverifiable' : verdict(live > 0) + } + }, + { + name: 'agents', + read: () => verdict(ports.agentStates().some((entry) => entry.state === 'working')) + }, + { name: 'migration', read: () => verdict(ports.hasStagedMigration()) }, + { + name: 'activation', + read: async () => verdict(await ports.activationFenceExists(config.activationRoot)) + } + ] +} + +export async function activationFenceExists(root: string): Promise { + try { + await stat(root) + return true + } catch (error) { + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT') { + return false + } + throw error + } +} + +export function installOrcadManagedIdleExit(input: { + config: OrcadManagedIdleExitConfig + ports: OrcadManagedIdleExitPorts + /** Records the clean stop, then runs the same graceful shutdown as SIGTERM. */ + stop: (evidence: OrcadIdleExitEvidence) => void +}): () => void { + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: input.config.timeoutMs, + probes: createOrcadIdleProbes(input.config, input.ports), + lastClientActivityAt: () => input.ports.readClientActivity().lastRequestAt, + onIdle: input.stop + }) + monitor.start() + return () => monitor.stop() +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts index d6fdf15122d..cadb4a6121f 100644 --- a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts +++ b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts @@ -191,6 +191,13 @@ export class OrcadCatalogImportPersistence { context.runtime.terminalScrollbackSnapshotStorage ) } + + /** A migration into this server that is staged but neither committed nor aborted. */ + hasStagedOrcadMigrationCatalog(): boolean { + return ( + (this[orcadCatalogImportContext].runtime.state.orcadMigrationStagedCatalogs?.length ?? 0) > 0 + ) + } } function commitPreparedCatalog( diff --git a/src/main/runtime/runtime-rpc-client-activity.test.ts b/src/main/runtime/runtime-rpc-client-activity.test.ts new file mode 100644 index 00000000000..12b6c3a5206 --- /dev/null +++ b/src/main/runtime/runtime-rpc-client-activity.test.ts @@ -0,0 +1,34 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' +import { OrcaRuntimeRpcServer } from './runtime-rpc' +import { readRuntimeMetadata } from './runtime-metadata' +import { sendRequest } from './runtime-rpc-test-harness' + +describe('OrcaRuntimeRpcServer client activity', () => { + it('records each request so an idle host restarts its quiet period', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-activity-')) + const server = new OrcaRuntimeRpcServer({ runtime: new OrcaRuntimeService(), userDataPath }) + await server.start() + try { + const before = server.readClientActivity() + expect(before).toMatchObject({ openConnections: 0, requestsInFlight: 0 }) + + const metadata = readRuntimeMetadata(userDataPath) + await new Promise((resolve) => setTimeout(resolve, 5)) + await sendRequest(metadata!.transports[0]!.endpoint, { + id: 'req_status', + authToken: metadata!.authToken, + method: 'status.get' + }) + + const after = server.readClientActivity() + expect(after.requestsInFlight).toBe(0) + expect(after.lastRequestAt).toBeGreaterThan(before.lastRequestAt) + } finally { + await server.stop() + } + }) +}) diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts index 4c595b3bbab..3a1df4ce8ba 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts @@ -45,7 +45,7 @@ export class RuntimeRpcLifecycle extends RuntimeRpcWebSocketDispatch { // Why: the `.catch` guarantees reply() always fires so a throw can't strand the client or leak the AbortController. socketTransport.onMessage((msg, reply, context) => { - void this.handleMessage(msg, context) + void this.trackClientRequest(() => this.handleMessage(msg, context)) .then((response) => { reply(JSON.stringify(response)) }) @@ -222,17 +222,22 @@ export class RuntimeRpcLifecycle extends RuntimeRpcWebSocketDispatch { deviceRegistry, e2eeKeypair, onText: (socket, plaintext, reply, sendBinary) => { - void this.handleWebSocketMessage( - plaintext, - reply, - sendBinary, - undefined, - socket.ws, - socket.device.deviceToken, - socket + void this.trackClientRequest(() => + this.handleWebSocketMessage( + plaintext, + reply, + sendBinary, + undefined, + socket.ws, + socket.device.deviceToken, + socket + ) ) }, - onBinary: (socket, bytes) => this.handleWebSocketBinaryMessage(bytes, socket.ws), + onBinary: (socket, bytes) => { + this.lastClientRequestAt = Date.now() + this.handleWebSocketBinaryMessage(bytes, socket.ws) + }, onReady: () => { // Why: first authenticated mobile/remote client (direct WS and // cloud relay both attach here) starts path-candidate tracking. diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts b/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts index f79834fbcc2..c16bcda5510 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts @@ -1,11 +1,26 @@ import { RuntimeRpcMobilePairing } from './runtime-rpc-mobile-pairing' +export type RuntimeRpcClientActivity = { + openConnections: number + requestsInFlight: number + lastRequestAt: number +} + export class RuntimeRpcShutdown extends RuntimeRpcMobilePairing { /** Why: test-only seam — runs one ownership check instead of waiting out the poll interval. */ checkRuntimeMetadataOwnership(): Promise { return this.metadataOwnershipWatch?.check() ?? Promise.resolve() } + /** What a host's idle exit reads to know whether any client is still using this server. */ + readClientActivity(): RuntimeRpcClientActivity { + return { + openConnections: this.mobileSocketWiring?.connectionCount ?? 0, + requestsInFlight: this.clientRequestsInFlight, + lastRequestAt: this.lastClientRequestAt + } + } + async stop(): Promise { // Why: STA-2370 — refuse new widens, then let any in-flight pairing widen settle into the live // transport arrays before snapshotting them, so a racing rebind can't strand a wide 0.0.0.0 listener diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-state.ts b/src/main/runtime/runtime-rpc/runtime-rpc-state.ts index 20ddbf70e38..bd9a5ad2106 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-state.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-state.ts @@ -92,6 +92,8 @@ export class RuntimeRpcState { protected activeAskLongPolls = 0 protected activeBrowserHostLongPolls = 0 protected readonly activeBrowserHostLongPollsByDevice = new Map() + protected clientRequestsInFlight = 0 + protected lastClientRequestAt = Date.now() constructor({ runtime, @@ -137,4 +139,14 @@ export class RuntimeRpcState { this.pushUnregisterOutbox = new PushUnregisterOutbox(userDataPath) this.runtime.configureNotificationDismissalStore(userDataPath) } + + /** Counts a client message for idle exit, from receipt until its dispatch settles. */ + protected trackClientRequest(work: () => Promise): Promise { + this.clientRequestsInFlight += 1 + this.lastClientRequestAt = Date.now() + return work().finally(() => { + this.clientRequestsInFlight -= 1 + this.lastClientRequestAt = Date.now() + }) + } } diff --git a/src/main/ssh/orcad-activation-crash-recovery.test.ts b/src/main/ssh/orcad-activation-crash-recovery.test.ts index 8a61515f1a7..758291c3838 100644 --- a/src/main/ssh/orcad-activation-crash-recovery.test.ts +++ b/src/main/ssh/orcad-activation-crash-recovery.test.ts @@ -231,3 +231,20 @@ describe('recovery refusals keep the fence', () => { expect(host.fence).toBe(false) }) }) + +describe('every launch is a managed one', () => { + it.each(scenarios)( + '%s starts orcad with idle exit and its activation fence', + async (_n, scenario, run) => { + host = scenario() + await run() + const launches = host.commands.filter((command) => command.includes('nohup')) + expect(launches.length).toBeGreaterThan(0) + for (const launch of launches) { + expect(launch).toContain( + "ORCA_ORCAD_MANAGED_ACTIVATION_ROOT='/home/u/.orca-remote/.orcad-activation-transaction'" + ) + } + } + ) +}) diff --git a/src/main/ssh/orcad-installed-activation.ts b/src/main/ssh/orcad-installed-activation.ts index 97c6272b0f9..eb33ccb41f0 100644 --- a/src/main/ssh/orcad-installed-activation.ts +++ b/src/main/ssh/orcad-installed-activation.ts @@ -27,7 +27,10 @@ import { orcadStopFreedTheHost } from './orcad-remote-process-control' import { joinRemotePath } from './ssh-remote-platform' import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' import { preflightInstalledOrcad } from './orcad-remote-preflight' -import type { OrcadActivationLockControl } from './orcad-activation-lock' +import { + orcadActivationTransactionRoot, + type OrcadActivationLockControl +} from './orcad-activation-lock' import type { OrcadActivateTransaction } from './orcad-activation-transaction' import { createOrcadActivationTransaction, @@ -222,7 +225,8 @@ export async function activateInstalledOrcad( fullVersion, userDataDir: options.userDataDir, bindHost: options.bindHost, - port: options.port + port: options.port, + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) }) } catch (error) { if (isUnconfirmedSshCommandTermination(error)) { diff --git a/src/main/ssh/orcad-managed-serving-verify.ts b/src/main/ssh/orcad-managed-serving-verify.ts new file mode 100644 index 00000000000..65a4b3dedaf --- /dev/null +++ b/src/main/ssh/orcad-managed-serving-verify.ts @@ -0,0 +1,13 @@ +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import type { OrcadManagedServing } from './orcad-managed-serving' +import { verifyManagedTunnelServing } from './orcad-managed-tunnel' + +/** After the tunnel: the server answers, or was proven stopped and started; never throws. */ +export function verifyOrcadManagedServing( + userDataPath: string, + selector: string +): Promise { + return Promise.resolve() + .then(() => verifyManagedTunnelServing(resolveEnvironment(userDataPath, selector))) + .catch((error: unknown) => ({ state: 'unverifiable', detail: String(error) })) +} diff --git a/src/main/ssh/orcad-managed-serving.test.ts b/src/main/ssh/orcad-managed-serving.test.ts new file mode 100644 index 00000000000..171157a1575 --- /dev/null +++ b/src/main/ssh/orcad-managed-serving.test.ts @@ -0,0 +1,127 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./orcad-remote-context', () => ({ + resolveOrcadRemoteContext: vi.fn(async () => ({ + connection: {}, + host: {}, + remoteHome: '/home/u', + userDataDir: '/home/u/.orca' + })) +})) +vi.mock('./orcad-managed-wake', () => ({ wakeStoppedManagedOrcad: vi.fn() })) + +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { + ensureManagedOrcadServing, + resetManagedOrcadServingForTests, + setManagedOrcadStartListener, + type OrcadManagedServingInput +} from './orcad-managed-serving' + +const listener = { starting: vi.fn(), settled: vi.fn() } +let generation = 1 +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the transport generation is read; the remote context is mocked. +const connection = { getTransportGeneration: () => generation } as never + +function input(probe: () => Promise): OrcadManagedServingInput { + return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the id is read. + environment: { id: 'env-1' } as never, + target: { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' }, + connection, + remotePort: 6768, + probe: vi.fn(probe) + } +} + +beforeEach(() => { + vi.clearAllMocks() + resetManagedOrcadServingForTests() + setManagedOrcadStartListener(listener) + vi.spyOn(console, 'info').mockImplementation(() => {}) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) + +describe('ensureManagedOrcadServing', () => { + it('costs one round trip when the server answers', async () => { + expect(await ensureManagedOrcadServing(input(async () => true))).toEqual({ state: 'serving' }) + expect(wakeStoppedManagedOrcad).not.toHaveBeenCalled() + expect(listener.starting).not.toHaveBeenCalled() + }) + + it('starts a stopped server from its slot and shows the start on the status line', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockImplementation(async (_slot, onStarting) => { + onStarting?.() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only health is read. + return { outcome: 'started', readiness: { health: { previousIdleStop: null } } as never } + }) + + expect(await ensureManagedOrcadServing(input(async () => false))).toEqual({ + state: 'started', + boundPort: null + }) + expect(vi.mocked(wakeStoppedManagedOrcad).mock.calls[0]?.[0]).toMatchObject({ port: 6768 }) + expect(listener.starting).toHaveBeenCalledOnce() + expect(listener.settled).toHaveBeenCalledWith(expect.anything(), 'env-1', { + state: 'started', + boundPort: null + }) + }) + + it('stays unverifiable with the reason when the start fails, never a terminal verdict', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockImplementation(async (_slot, onStarting) => { + onStarting?.() + throw new Error('orcad did not become ready.\nLast lines of orcad.log:\nboom') + }) + + const serving = await ensureManagedOrcadServing(input(async () => false)) + expect(serving).toEqual({ + state: 'unverifiable', + detail: 'orcad did not become ready.\nLast lines of orcad.log:\nboom' + }) + expect(listener.settled).toHaveBeenCalledWith(expect.anything(), 'env-1', serving) + }) + + it('leaves a live process that did not answer alone', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'serving' }) + expect(await ensureManagedOrcadServing(input(async () => false))).toEqual({ state: 'serving' }) + expect(listener.starting).not.toHaveBeenCalled() + }) + + it.each(['fenced', 'unverifiable', 'not-activated'] as const)( + 'does not start a server whose host says %s', + async (outcome) => { + vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome }) + const serving = await ensureManagedOrcadServing(input(async () => false)) + expect(serving.state).toBe('unverifiable') + expect(listener.starting).not.toHaveBeenCalled() + } + ) + + it('shares one check between a fresh tunnel and the connect right after it', async () => { + let now = 0 + const probe = vi.fn(async () => true) + const first = input(probe) + await Promise.all([ + ensureManagedOrcadServing(first, () => now), + ensureManagedOrcadServing(first, () => now) + ]) + now = 4_000 + await ensureManagedOrcadServing(first, () => now) + expect(probe).toHaveBeenCalledOnce() + now = 6_000 + await ensureManagedOrcadServing(first, () => now) + expect(probe).toHaveBeenCalledTimes(2) + }) + + it('never answers a new SSH transport from an earlier verdict, as after a reboot', async () => { + const probe = vi.fn(async () => true) + await ensureManagedOrcadServing(input(probe), () => 0) + generation += 1 + await ensureManagedOrcadServing(input(probe), () => 1) + expect(probe).toHaveBeenCalledTimes(2) + // A rebind to the port a restarted server bound is a different server address. + await ensureManagedOrcadServing({ ...input(probe), remotePort: 40_001 }, () => 2) + expect(probe).toHaveBeenCalledTimes(3) + }) +}) diff --git a/src/main/ssh/orcad-managed-serving.ts b/src/main/ssh/orcad-managed-serving.ts new file mode 100644 index 00000000000..64e59132031 --- /dev/null +++ b/src/main/ssh/orcad-managed-serving.ts @@ -0,0 +1,157 @@ +/** + * Making sure a managed orcad is serving before a client relies on it: a server that answers + * costs one round trip; one that does not is checked on the host and, only if proven stopped + * (an idle stop, a kill, a host reboot), started from its activated slot. A daemon that survived + * is adopted by the new orcad with its terminals; after a reboot both start fresh. + * + * Never throws. A server that cannot be started stays `unverifiable` with the reason, and is + * never read as evidence that its terminals exited. + */ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import { orcadBoundPort } from './orcad-managed-bound-port' +import { managedOrcadSlot } from './orcad-managed-runtime-context' +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import type { SshConnection } from './ssh-connection' + +export type OrcadManagedServing = + | { state: 'serving' } + /** `boundPort` is the port the restarted server bound, which a forward must follow. */ + | { state: 'started'; boundPort: number | null } + | { state: 'unverifiable'; detail: string } + +export type OrcadManagedServingInput = { + environment: KnownRuntimeEnvironment + target: SshTarget + connection: SshConnection + remotePort: number + probe: (environment: KnownRuntimeEnvironment, timeoutMs: number) => Promise +} + +const PROBE_TIMEOUT_MS = 5_000 +// Why: a connect checks right after its fresh tunnel did; one verdict serves both, on that +// transport and port only, so a kill, reboot or rebind is never answered from cache. +const VERDICT_REUSE_MS = 5_000 + +const inFlight = new Map>() +type RecentVerdict = { + at: number + connection: SshConnection + generation: number + remotePort: number + serving: OrcadManagedServing +} +const recent = new Map() +export type ManagedOrcadStartListener = { + /** Shows "Starting managed server…" for the host. */ + starting: (target: SshTarget) => void + /** The start finished; an `unverifiable` result carries why, with orcad.log's tail. */ + settled: (target: SshTarget, environmentId: string, serving: OrcadManagedServing) => void +} + +let startListener: ManagedOrcadStartListener | null = null + +/** The SSH status wiring registers where a start is shown. */ +export function setManagedOrcadStartListener(listener: ManagedOrcadStartListener | null): void { + startListener = listener +} + +export function ensureManagedOrcadServing( + input: OrcadManagedServingInput, + now: () => number = Date.now +): Promise { + const id = input.environment.id + const generation = input.connection.getTransportGeneration() + const cached = recent.get(id) + if ( + cached && + cached.connection === input.connection && + cached.generation === generation && + cached.remotePort === input.remotePort && + now() - cached.at < VERDICT_REUSE_MS + ) { + return Promise.resolve(cached.serving) + } + const pending = inFlight.get(id) + if (pending) { + return pending + } + const operation = checkAndStart(input).then((serving) => { + recent.set(id, { + at: now(), + connection: input.connection, + generation, + remotePort: input.remotePort, + serving + }) + return serving + }) + const settled = operation.finally(() => inFlight.delete(id)) + inFlight.set(id, settled) + return settled +} + +/** Test-only: forget cached verdicts. */ +export function resetManagedOrcadServingForTests(): void { + inFlight.clear() + recent.clear() +} + +async function checkAndStart(input: OrcadManagedServingInput): Promise { + if (await input.probe(input.environment, PROBE_TIMEOUT_MS)) { + return { state: 'serving' } + } + let starting = false + const serving = await wakeIfStopped(input, () => { + starting = true + startListener?.starting(input.target) + }) + if (starting) { + startListener?.settled(input.target, input.environment.id, serving) + } + return serving +} + +async function wakeIfStopped( + input: OrcadManagedServingInput, + onStarting: () => void +): Promise { + const label = input.target.label + try { + const context = await resolveOrcadRemoteContext(input.target, input.connection) + const wake = await wakeStoppedManagedOrcad( + managedOrcadSlot(context, input.remotePort), + onStarting + ) + if (wake.outcome === 'started') { + const idle = wake.readiness.health?.previousIdleStop + const cause = idle + ? `it had stopped after idling at ${idle.stoppedAt}` + : 'it had stopped without an idle-stop record (crash, signal or host restart)' + console.info(`[ssh] Started the managed Orca server on ${label}; ${cause}.`) + return { state: 'started', boundPort: orcadBoundPort(wake.readiness) } + } + // A live process that did not answer may still be starting; it is not restarted. + if (wake.outcome === 'serving') { + return { state: 'serving' } + } + const detail = wakeRefusal(wake.outcome) + console.warn(`[ssh] The managed Orca server on ${label} is not answering: ${detail}`) + return { state: 'unverifiable', detail } + } catch (error) { + console.warn(`[ssh] Could not start the managed Orca server on ${label}:`, error) + return { state: 'unverifiable', detail: error instanceof Error ? error.message : String(error) } + } +} + +function wakeRefusal(outcome: 'not-activated' | 'unverifiable' | 'fenced'): string { + switch (outcome) { + case 'fenced': + return 'An update, rollback or recovery holds this host; it was not started.' + case 'not-activated': + return 'This host has no activated managed server to start.' + case 'unverifiable': + return 'Whether the server process is still running could not be proven, so it was not started.' + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-manager.ts b/src/main/ssh/orcad-managed-tunnel-manager.ts new file mode 100644 index 00000000000..ce549278cde --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-manager.ts @@ -0,0 +1,300 @@ +/** + * One SSH forward per managed environment, owned across reconnects and host resume, and the + * serving check that starts a stopped server behind it. + */ +import { + getRuntimeSshAccess, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { SshConnection } from './ssh-connection' +import type { SshConnectionManager } from './ssh-connection-manager' +import { SshPortForwardManager } from './ssh-port-forward' +import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport' +import { + OrcadManagedTunnelResumeRecovery, + type ActiveOrcadTunnel, + type OrcadManagedServingCheck, + type OrcadManagedTunnelProbe, + type OrcadManagedTunnelResumeOptions +} from './orcad-managed-tunnel-resume' +import type { OrcadManagedServing } from './orcad-managed-serving' +import { checkManagedTunnelServing, type OrcadTunnelServing } from './orcad-managed-tunnel-serving' +import type { getSshTargetRegistryStore } from './ssh-target-registry' +import { + environmentForwardChecks, + forwardToVerifiedOrcad, + PERSISTED_PORT_TARGETING, + type OrcadManagedTunnelTargeting, + type OrcadTunnelStartChecks +} from './orcad-managed-tunnel-target' + +export type OrcadManagedTunnelDependencies = { + getConnectionManager: () => SshConnectionManager | null + getTargetStore: () => ReturnType + forwardManager?: SshPortForwardManager + probeTunnel?: OrcadManagedTunnelProbe + targeting?: OrcadManagedTunnelTargeting + /** Runs after a fresh forward is up; starts a server that stopped (e.g. after idling). */ + ensureServing?: OrcadManagedServingCheck +} + +export class OrcadManagedTunnelManager { + private readonly active = new Map() + private readonly inFlight = new Map>() + private readonly ownershipGenerations = new Map() + private readonly forwards: SshPortForwardManager + private readonly resumeRecovery: OrcadManagedTunnelResumeRecovery + private readonly targeting: OrcadManagedTunnelTargeting + private managerGeneration = 0 + + constructor(private readonly dependencies: OrcadManagedTunnelDependencies) { + this.forwards = + dependencies.forwardManager ?? + new SshPortForwardManager({}, [new OrcadManagedTunnelTransportProvider()]) + this.targeting = dependencies.targeting ?? PERSISTED_PORT_TARGETING + this.resumeRecovery = new OrcadManagedTunnelResumeRecovery({ + active: this.active, + forwards: this.forwards, + getConnectionManager: dependencies.getConnectionManager, + getManagerGeneration: () => this.managerGeneration, + getTargetStore: dependencies.getTargetStore, + inFlight: this.inFlight, + ownershipGenerations: this.ownershipGenerations, + probeTunnel: dependencies.probeTunnel, + targeting: this.targeting, + checkServing: (environment) => this.checkServing(environment) + }) + this.forwards.setCallbacks({ + onForwardClosed: (entry) => { + for (const [environmentId, active] of this.active) { + if (active.forwardId === entry.id) { + this.active.delete(environmentId) + } + } + } + }) + } + + ensure( + environment: KnownRuntimeEnvironment, + resolveCurrent: () => KnownRuntimeEnvironment | null = () => environment + ): Promise { + if (!getRuntimeSshAccess(environment)) { + return Promise.resolve() + } + const pending = this.inFlight.get(environment.id) + if (pending) { + return pending + } + const operation = this.ensureManagedTunnel(environment, resolveCurrent).finally(() => { + if (this.inFlight.get(environment.id) === operation) { + this.inFlight.delete(environment.id) + } + }) + this.inFlight.set(environment.id, operation) + return operation + } + + async start( + environmentId: string, + target: SshTarget, + connection: SshConnection, + remotePort: number, + checks: OrcadTunnelStartChecks = {} + ): Promise { + if (!target.generation) { + throw new Error('Managed Orca SSH target has no registration generation.') + } + const managerGeneration = this.managerGeneration + const ownershipGeneration = (this.ownershipGenerations.get(environmentId) ?? 0) + 1 + const transportGeneration = connection.getTransportGeneration() + const stillCurrent = (): boolean => + this.managerGeneration === managerGeneration && + this.ownershipGenerations.get(environmentId) === ownershipGeneration && + connection.getTransportGeneration() === transportGeneration + await this.close(environmentId) + if (!stillCurrent()) { + throw new Error('Orca SSH tunnel setup was superseded.') + } + const forward = await forwardToVerifiedOrcad({ + targetId: target.id, + connection, + forwards: this.forwards, + localPort: 0, + label: 'Managed Orca server', + remotePort, + rereadRemotePort: checks.rereadRemotePort, + verify: checks.verify, + stillCurrent + }) + if (!forward) { + throw new Error('Orca SSH tunnel setup was superseded.') + } + this.active.set(environmentId, { + connection, + forwardId: forward.id, + localPort: forward.localPort, + remotePort: forward.remotePort, + preferredPort: checks.preferredPort ?? remotePort, + sshTargetGeneration: target.generation, + targetId: target.id, + transportGeneration + }) + return forward.localPort + } + + async close(environmentId: string): Promise { + this.ownershipGenerations.set( + environmentId, + (this.ownershipGenerations.get(environmentId) ?? 0) + 1 + ) + const active = this.active.get(environmentId) + if (!active) { + return + } + this.active.delete(environmentId) + await this.forwards.removeForwardAndWait(active.forwardId) + } + + dispose(): void { + this.managerGeneration += 1 + this.active.clear() + this.inFlight.clear() + this.ownershipGenerations.clear() + this.resumeRecovery.dispose() + this.forwards.dispose() + } + + /** The server behind the tunnel answers, or is started; a moved port rebuilds the forward. */ + async verifyServing(environment: KnownRuntimeEnvironment): Promise { + if (!this.active.has(environment.id)) { + await this.ensure(environment) + } + const serving = await this.checkServing(environment) + if (serving.rebind) { + await this.ensure(environment) + } + return serving + } + + /** Never rebuilds, so it is safe inside a build: a moved port only drops the forward. */ + checkServing(environment: KnownRuntimeEnvironment): Promise { + return checkManagedTunnelServing({ + environment, + active: this.active, + getTarget: (id) => this.dependencies.getTargetStore()?.getTarget(id), + forwards: this.forwards, + ensureServing: this.dependencies.ensureServing + }) + } + + recoverAfterHostResume(options: OrcadManagedTunnelResumeOptions): Promise { + return this.resumeRecovery.recover(options) + } + + private async ensureManagedTunnel( + environment: KnownRuntimeEnvironment, + resolveCurrent: () => KnownRuntimeEnvironment | null, + rebound = false + ): Promise { + const deployment = getRuntimeSshAccess(environment) + if (!deployment || environment.connectionDependency !== 'ssh-tunnel') { + throw new Error('Managed orcad environment is missing its SSH tunnel dependency.') + } + const targetStore = this.dependencies.getTargetStore() + const connectionManager = this.dependencies.getConnectionManager() + if (!targetStore || !connectionManager) { + throw new Error('SSH is unavailable on this client; the managed Orca server is unverifiable.') + } + const target = targetStore.getTarget(deployment.sshTargetId) + if (!target || target.generation !== deployment.sshTargetGeneration) { + throw new Error( + 'The SSH registration for this managed Orca server was removed or re-created.' + ) + } + if (getManagedOrcadFenceEnvironmentId(target) !== environment.id) { + throw new Error('The SSH target is no longer owned by this managed Orca server.') + } + + const managerGeneration = this.managerGeneration + const ownershipGeneration = this.ownershipGenerations.get(environment.id) ?? 0 + const stillOwned = (): boolean => { + const currentTarget = targetStore.getTarget(target.id) + const currentEnvironment = resolveCurrent() + const currentAccess = currentEnvironment ? getRuntimeSshAccess(currentEnvironment) : undefined + return ( + this.managerGeneration === managerGeneration && + (this.ownershipGenerations.get(environment.id) ?? 0) === ownershipGeneration && + currentTarget?.generation === target.generation && + getManagedOrcadFenceEnvironmentId(currentTarget) === environment.id && + currentEnvironment?.id === environment.id && + currentEnvironment.runtimeId === environment.runtimeId && + (currentEnvironment.pairingRevision ?? currentEnvironment.createdAt) === + (environment.pairingRevision ?? environment.createdAt) && + currentEnvironment.connectionDependency === 'ssh-tunnel' && + currentAccess?.sshTargetId === deployment.sshTargetId && + currentAccess.sshTargetGeneration === deployment.sshTargetGeneration && + currentAccess.localPort === deployment.localPort && + currentAccess.remotePort === deployment.remotePort + ) + } + const connection = await connectionManager.connect(target) + if (!stillOwned()) { + return + } + const transportGeneration = connection.getTransportGeneration() + const active = this.active.get(environment.id) + if ( + active?.connection === connection && + active.transportGeneration === transportGeneration && + active.targetId === target.id && + active.sshTargetGeneration === target.generation && + active.localPort === deployment.localPort && + active.preferredPort === deployment.remotePort + ) { + return + } + const checks = await environmentForwardChecks(this.targeting, { + environment, + target, + connection + }) + if (active && stillOwned()) { + await this.forwards.removeForwardAndWait(active.forwardId) + if (this.active.get(environment.id) === active) { + this.active.delete(environment.id) + } + } + if (!stillOwned()) { + return + } + const forward = await forwardToVerifiedOrcad({ + targetId: target.id, + connection, + forwards: this.forwards, + localPort: deployment.localPort, + label: `Managed Orca server: ${environment.name}`, + ...checks, + stillCurrent: () => + stillOwned() && connection.getTransportGeneration() === transportGeneration + }) + if (!forward) { + return + } + this.active.set(environment.id, { + connection, + forwardId: forward.id, + localPort: forward.localPort, + remotePort: forward.remotePort, + preferredPort: deployment.remotePort, + sshTargetGeneration: target.generation, + targetId: target.id, + transportGeneration + }) + if ((await this.checkServing(environment)).rebind && !rebound) { + await this.ensureManagedTunnel(environment, resolveCurrent, true) + } + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-resume.ts b/src/main/ssh/orcad-managed-tunnel-resume.ts index b42e0830ac7..203d722b835 100644 --- a/src/main/ssh/orcad-managed-tunnel-resume.ts +++ b/src/main/ssh/orcad-managed-tunnel-resume.ts @@ -7,6 +7,8 @@ import { type RuntimeSshTunnelLink } from '../../shared/runtime-environments' import type { SshConnection } from './ssh-connection' +import type { SshTarget } from '../../shared/ssh-types' +import type { OrcadManagedServing } from './orcad-managed-serving' import type { SshConnectionManager } from './ssh-connection-manager' import type { SshPortForwardManager } from './ssh-port-forward' import type { getSshTargetRegistryStore } from './ssh-target-registry' @@ -33,6 +35,13 @@ export type OrcadManagedTunnelProbe = ( timeoutMs: number ) => Promise +export type OrcadManagedServingCheck = (input: { + environment: KnownRuntimeEnvironment + target: SshTarget + connection: SshConnection + remotePort: number +}) => Promise + export type OrcadManagedTunnelResumeOptions = { attempts: number resolveEnvironment: (environmentId: string) => KnownRuntimeEnvironment | null @@ -49,6 +58,8 @@ type ResumeRecoveryDependencies = { ownershipGenerations: Map probeTunnel?: OrcadManagedTunnelProbe targeting: OrcadManagedTunnelTargeting + /** Never rebuilds: a server restarted on a new port drops this forward for the next ensure. */ + checkServing?: (environment: KnownRuntimeEnvironment) => Promise } type ResolvedManagedTunnelEnvironment = { @@ -222,6 +233,8 @@ export class OrcadManagedTunnelResumeRecovery { targetId: active.targetId, transportGeneration }) + // A server that idled out while this client slept is started here, not reported as lost. + await this.dependencies.checkServing?.(current.environment) } private resolveEnvironment( @@ -265,7 +278,7 @@ export class OrcadManagedTunnelResumeRecovery { } } -async function probeManagedOrcadTunnel( +export async function probeManagedOrcadTunnel( environment: KnownRuntimeEnvironment, timeoutMs: number ): Promise { diff --git a/src/main/ssh/orcad-managed-tunnel-serving.ts b/src/main/ssh/orcad-managed-tunnel-serving.ts new file mode 100644 index 00000000000..bd545c75360 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-serving.ts @@ -0,0 +1,45 @@ +/** + * The serving check behind an established managed tunnel. A restarted orcad may bind a port + * other than the one the tunnel forwards to; the forward is then dropped, and the next build + * forwards to the port the new server actually bound. + */ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { OrcadManagedServing } from './orcad-managed-serving' +import type { ActiveOrcadTunnel, OrcadManagedServingCheck } from './orcad-managed-tunnel-resume' +import type { SshPortForwardManager } from './ssh-port-forward' +import type { SshTarget } from '../../shared/ssh-types' + +export type OrcadTunnelServing = OrcadManagedServing & { rebind?: true } + +export async function checkManagedTunnelServing(input: { + environment: KnownRuntimeEnvironment + active: Map + getTarget: (targetId: string) => SshTarget | null | undefined + forwards: Pick + ensureServing: OrcadManagedServingCheck | undefined +}): Promise { + const { environment, active: tunnels } = input + const active = tunnels.get(environment.id) + const target = active && input.getTarget(active.targetId) + if (!active || !target || !input.ensureServing) { + return { state: 'unverifiable', detail: 'The managed server tunnel is not up.' } + } + const serving = await input.ensureServing({ + environment, + target, + connection: active.connection, + remotePort: active.remotePort + }) + if ( + serving.state !== 'started' || + serving.boundPort === null || + serving.boundPort === active.remotePort + ) { + return serving + } + if (tunnels.get(environment.id) === active) { + tunnels.delete(environment.id) + } + await input.forwards.removeForwardAndWait(active.forwardId) + return { ...serving, rebind: true } +} diff --git a/src/main/ssh/orcad-managed-tunnel.test.ts b/src/main/ssh/orcad-managed-tunnel.test.ts index 338516074d3..0a285126c7a 100644 --- a/src/main/ssh/orcad-managed-tunnel.test.ts +++ b/src/main/ssh/orcad-managed-tunnel.test.ts @@ -84,6 +84,7 @@ function setup(overrides: Partial = {}, targeting?: OrcadManagedTunne }) ) const removeForwardAndWait = vi.fn().mockResolvedValue(null) + const ensureServing = vi.fn().mockResolvedValue({ state: 'serving' }) // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the members the tunnel manager calls. const forwardManager = { setCallbacks: vi.fn(), @@ -104,12 +105,14 @@ function setup(overrides: Partial = {}, targeting?: OrcadManagedTunne getTargetStore: () => ({ getTarget: vi.fn(() => target) }) as unknown as SshConnectionStore, forwardManager, probeTunnel, - targeting + targeting, + ensureServing }) return { addForward, connect, connection, + ensureServing, getConnection, getState, manager, @@ -356,6 +359,38 @@ describe.each(['orcadDeployment', 'sshAccess'] as const)( expect(state.probeTunnel).not.toHaveBeenCalled() }) + it('checks that the server is running only when it sets up a fresh forward', async () => { + const state = setup() + + await state.manager.ensure(environment()) + await state.manager.ensure(environment()) + expect(state.ensureServing).toHaveBeenCalledOnce() + expect(state.ensureServing).toHaveBeenCalledWith({ + environment: expect.objectContaining({ id: 'environment-1' }), + target: state.target, + connection: state.connection, + remotePort: 6_768 + }) + + state.setTransportGeneration(4) + await state.manager.ensure(environment()) + expect(state.ensureServing).toHaveBeenCalledTimes(2) + }) + + it('starts a server that stopped while the client slept once the tunnel is rebuilt', async () => { + const state = setup() + await state.manager.ensure(environment()) + state.probeTunnel.mockResolvedValue(false) + + await state.manager.recoverAfterHostResume(resumeOptions()) + + expect(state.reconnect).toHaveBeenCalledOnce() + expect(state.ensureServing).toHaveBeenCalledTimes(2) + expect(state.ensureServing).toHaveBeenLastCalledWith( + expect.objectContaining({ target: state.target, remotePort: 6_768 }) + ) + }) + it('keeps a healthy managed tunnel intact after host resume', async () => { const state = setup() await state.manager.ensure(environment()) @@ -534,6 +569,50 @@ describe('OrcadManagedTunnelManager bound port', () => { expect(state.verifyIdentity).toHaveBeenCalledOnce() }) + it('follows a restarted server to the port it bound, within the same ensure', async () => { + const state = boundPortSetup([6_768, 58_520]) + state.ensureServing + .mockResolvedValueOnce({ state: 'started', boundPort: 58_520 }) + .mockResolvedValue({ state: 'serving' }) + await state.manager.ensure(createEnvironment('orcadDeployment')) + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + expect(state.addForward).toHaveBeenLastCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 58_520, + 'Managed Orca server: Managed server' + ) + expect(state.ensureServing).toHaveBeenLastCalledWith( + expect.objectContaining({ remotePort: 58_520 }) + ) + }) + + it('rebuilds after an explicit check finds the server restarted on another port', async () => { + const state = boundPortSetup([6_768, 58_520]) + const environment = createEnvironment('orcadDeployment') + await state.manager.ensure(environment) + state.ensureServing + .mockResolvedValueOnce({ state: 'started', boundPort: 58_520 }) + .mockResolvedValue({ state: 'serving' }) + + await expect(state.manager.verifyServing(environment)).resolves.toMatchObject({ + state: 'started', + rebind: true + }) + expect(state.addForward).toHaveBeenCalledTimes(2) + expect(state.addForward).toHaveBeenLastCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 58_520, + 'Managed Orca server: Managed server' + ) + }) + it('reuses a verified tunnel without reading the port again', async () => { const state = boundPortSetup([58_520]) const environment = createEnvironment('orcadDeployment') diff --git a/src/main/ssh/orcad-managed-tunnel.ts b/src/main/ssh/orcad-managed-tunnel.ts index 660651e83f9..f0b6509e38c 100644 --- a/src/main/ssh/orcad-managed-tunnel.ts +++ b/src/main/ssh/orcad-managed-tunnel.ts @@ -1,273 +1,23 @@ -import { - getRuntimeSshAccess, - type KnownRuntimeEnvironment -} from '../../shared/runtime-environments' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' import { resolveEnvironment } from '../../shared/runtime-environment-store' import type { SshTarget } from '../../shared/ssh-types' -import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' import type { SshConnection } from './ssh-connection' -import type { SshConnectionManager } from './ssh-connection-manager' -import { SshPortForwardManager } from './ssh-port-forward' -import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport' -import { - OrcadManagedTunnelResumeRecovery, - type ActiveOrcadTunnel, - type OrcadManagedTunnelProbe, - type OrcadManagedTunnelResumeOptions -} from './orcad-managed-tunnel-resume' +import { probeManagedOrcadTunnel } from './orcad-managed-tunnel-resume' +import { ensureManagedOrcadServing } from './orcad-managed-serving' +import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry' import { - environmentForwardChecks, - forwardToVerifiedOrcad, MANAGED_ORCAD_TUNNEL_TARGETING, - PERSISTED_PORT_TARGETING, - type OrcadManagedTunnelTargeting, type OrcadTunnelStartChecks } from './orcad-managed-tunnel-target' -type OrcadManagedTunnelDependencies = { - getConnectionManager: () => SshConnectionManager | null - getTargetStore: () => ReturnType - forwardManager?: SshPortForwardManager - probeTunnel?: OrcadManagedTunnelProbe - targeting?: OrcadManagedTunnelTargeting -} - -export class OrcadManagedTunnelManager { - private readonly active = new Map() - private readonly inFlight = new Map>() - private readonly ownershipGenerations = new Map() - private readonly forwards: SshPortForwardManager - private readonly resumeRecovery: OrcadManagedTunnelResumeRecovery - private readonly targeting: OrcadManagedTunnelTargeting - private managerGeneration = 0 - - constructor(private readonly dependencies: OrcadManagedTunnelDependencies) { - this.forwards = - dependencies.forwardManager ?? - new SshPortForwardManager({}, [new OrcadManagedTunnelTransportProvider()]) - this.targeting = dependencies.targeting ?? PERSISTED_PORT_TARGETING - this.resumeRecovery = new OrcadManagedTunnelResumeRecovery({ - active: this.active, - forwards: this.forwards, - getConnectionManager: dependencies.getConnectionManager, - getManagerGeneration: () => this.managerGeneration, - getTargetStore: dependencies.getTargetStore, - inFlight: this.inFlight, - ownershipGenerations: this.ownershipGenerations, - probeTunnel: dependencies.probeTunnel, - targeting: this.targeting - }) - this.forwards.setCallbacks({ - onForwardClosed: (entry) => { - for (const [environmentId, active] of this.active) { - if (active.forwardId === entry.id) { - this.active.delete(environmentId) - } - } - } - }) - } - - ensure( - environment: KnownRuntimeEnvironment, - resolveCurrent: () => KnownRuntimeEnvironment | null = () => environment - ): Promise { - if (!getRuntimeSshAccess(environment)) { - return Promise.resolve() - } - const pending = this.inFlight.get(environment.id) - if (pending) { - return pending - } - const operation = this.ensureManagedTunnel(environment, resolveCurrent).finally(() => { - if (this.inFlight.get(environment.id) === operation) { - this.inFlight.delete(environment.id) - } - }) - this.inFlight.set(environment.id, operation) - return operation - } - - async start( - environmentId: string, - target: SshTarget, - connection: SshConnection, - remotePort: number, - checks: OrcadTunnelStartChecks = {} - ): Promise { - if (!target.generation) { - throw new Error('Managed Orca SSH target has no registration generation.') - } - const managerGeneration = this.managerGeneration - const ownershipGeneration = (this.ownershipGenerations.get(environmentId) ?? 0) + 1 - const transportGeneration = connection.getTransportGeneration() - const stillCurrent = (): boolean => - this.managerGeneration === managerGeneration && - this.ownershipGenerations.get(environmentId) === ownershipGeneration && - connection.getTransportGeneration() === transportGeneration - await this.close(environmentId) - if (!stillCurrent()) { - throw new Error('Orca SSH tunnel setup was superseded.') - } - const forward = await forwardToVerifiedOrcad({ - targetId: target.id, - connection, - forwards: this.forwards, - localPort: 0, - label: 'Managed Orca server', - remotePort, - rereadRemotePort: checks.rereadRemotePort, - verify: checks.verify, - stillCurrent - }) - if (!forward) { - throw new Error('Orca SSH tunnel setup was superseded.') - } - this.active.set(environmentId, { - connection, - forwardId: forward.id, - localPort: forward.localPort, - remotePort: forward.remotePort, - preferredPort: checks.preferredPort ?? remotePort, - sshTargetGeneration: target.generation, - targetId: target.id, - transportGeneration - }) - return forward.localPort - } - - async close(environmentId: string): Promise { - this.ownershipGenerations.set( - environmentId, - (this.ownershipGenerations.get(environmentId) ?? 0) + 1 - ) - const active = this.active.get(environmentId) - if (!active) { - return - } - this.active.delete(environmentId) - await this.forwards.removeForwardAndWait(active.forwardId) - } - - dispose(): void { - this.managerGeneration += 1 - this.active.clear() - this.inFlight.clear() - this.ownershipGenerations.clear() - this.resumeRecovery.dispose() - this.forwards.dispose() - } - - recoverAfterHostResume(options: OrcadManagedTunnelResumeOptions): Promise { - return this.resumeRecovery.recover(options) - } - - private async ensureManagedTunnel( - environment: KnownRuntimeEnvironment, - resolveCurrent: () => KnownRuntimeEnvironment | null - ): Promise { - const deployment = getRuntimeSshAccess(environment) - if (!deployment || environment.connectionDependency !== 'ssh-tunnel') { - throw new Error('Managed orcad environment is missing its SSH tunnel dependency.') - } - const targetStore = this.dependencies.getTargetStore() - const connectionManager = this.dependencies.getConnectionManager() - if (!targetStore || !connectionManager) { - throw new Error('SSH is unavailable on this client; the managed Orca server is unverifiable.') - } - const target = targetStore.getTarget(deployment.sshTargetId) - if (!target || target.generation !== deployment.sshTargetGeneration) { - throw new Error( - 'The SSH registration for this managed Orca server was removed or re-created.' - ) - } - if (getManagedOrcadFenceEnvironmentId(target) !== environment.id) { - throw new Error('The SSH target is no longer owned by this managed Orca server.') - } - - const managerGeneration = this.managerGeneration - const ownershipGeneration = this.ownershipGenerations.get(environment.id) ?? 0 - const stillOwned = (): boolean => { - const currentTarget = targetStore.getTarget(target.id) - const currentEnvironment = resolveCurrent() - const currentAccess = currentEnvironment ? getRuntimeSshAccess(currentEnvironment) : undefined - return ( - this.managerGeneration === managerGeneration && - (this.ownershipGenerations.get(environment.id) ?? 0) === ownershipGeneration && - currentTarget?.generation === target.generation && - getManagedOrcadFenceEnvironmentId(currentTarget) === environment.id && - currentEnvironment?.id === environment.id && - currentEnvironment.runtimeId === environment.runtimeId && - (currentEnvironment.pairingRevision ?? currentEnvironment.createdAt) === - (environment.pairingRevision ?? environment.createdAt) && - currentEnvironment.connectionDependency === 'ssh-tunnel' && - currentAccess?.sshTargetId === deployment.sshTargetId && - currentAccess.sshTargetGeneration === deployment.sshTargetGeneration && - currentAccess.localPort === deployment.localPort && - currentAccess.remotePort === deployment.remotePort - ) - } - const connection = await connectionManager.connect(target) - if (!stillOwned()) { - return - } - const transportGeneration = connection.getTransportGeneration() - const active = this.active.get(environment.id) - if ( - active?.connection === connection && - active.transportGeneration === transportGeneration && - active.targetId === target.id && - active.sshTargetGeneration === target.generation && - active.localPort === deployment.localPort && - active.preferredPort === deployment.remotePort - ) { - return - } - const checks = await environmentForwardChecks(this.targeting, { - environment, - target, - connection - }) - if (active && stillOwned()) { - await this.forwards.removeForwardAndWait(active.forwardId) - if (this.active.get(environment.id) === active) { - this.active.delete(environment.id) - } - } - if (!stillOwned()) { - return - } - const forward = await forwardToVerifiedOrcad({ - targetId: target.id, - connection, - forwards: this.forwards, - localPort: deployment.localPort, - label: `Managed Orca server: ${environment.name}`, - ...checks, - stillCurrent: () => - stillOwned() && connection.getTransportGeneration() === transportGeneration - }) - if (!forward) { - return - } - this.active.set(environment.id, { - connection, - forwardId: forward.id, - localPort: forward.localPort, - remotePort: forward.remotePort, - preferredPort: deployment.remotePort, - sshTargetGeneration: target.generation, - targetId: target.id, - transportGeneration - }) - } -} +export { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' const managedTunnels = new OrcadManagedTunnelManager({ getConnectionManager: getSshConnectionManager, getTargetStore: getSshTargetRegistryStore, - targeting: MANAGED_ORCAD_TUNNEL_TARGETING + targeting: MANAGED_ORCAD_TUNNEL_TARGETING, + ensureServing: (input) => ensureManagedOrcadServing({ ...input, probe: probeManagedOrcadTunnel }) }) export async function ensureOrcadManagedTunnel( @@ -288,6 +38,10 @@ function resolveEnvironmentOrNull(userDataPath: string, id: string) { } } +export function verifyManagedTunnelServing(environment: KnownRuntimeEnvironment) { + return managedTunnels.verifyServing(environment) +} + export function disposeOrcadManagedTunnels(): void { managedTunnels.dispose() } diff --git a/src/main/ssh/orcad-managed-wake.test.ts b/src/main/ssh/orcad-managed-wake.test.ts new file mode 100644 index 00000000000..4b99c79e1ca --- /dev/null +++ b/src/main/ssh/orcad-managed-wake.test.ts @@ -0,0 +1,118 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' +import type * as InstallLock from './ssh-relay-install-lock' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn() +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' +import { FakeOrcadHost, OLD } from './orcad-activation-host-test-harness' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV +} from '../../shared/orcad-idle-exit' + +let host = new FakeOrcadHost() + +const slot = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked, so the connection is never used. + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + readinessTimeoutMs: 50, + sleep: async () => {} +} + +function launches(): string[] { + return host.commands.filter((command) => command.includes('nohup')) +} + +/** The slot's process exited on its own, as an idle stop leaves it. */ +function stoppedHost(): FakeOrcadHost { + const stopped = FakeOrcadHost.deployedOld() + stopped.alive.clear() + return stopped +} + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + vi.mocked(acquireInstallLock).mockImplementation(async () => host.acquireFence()) +}) + +afterEach(() => { + vi.unstubAllEnvs() +}) + +describe('wakeStoppedManagedOrcad', () => { + it('starts a stopped active slot as a managed launch and releases the fence', async () => { + host = stoppedHost() + + const wake = await wakeStoppedManagedOrcad(slot) + + expect(wake.outcome).toBe('started') + expect([...host.alive]).toEqual([OLD]) + expect(host.fence).toBe(false) + expect(launches()).toHaveLength(1) + expect(launches()[0]).toContain( + `${ORCAD_MANAGED_ACTIVATION_ROOT_ENV}='/home/u/.orca-remote/.orcad-activation-transaction'` + ) + expect(launches()[0]).not.toContain(ORCAD_E2E_IDLE_TIMEOUT_ENV) + }) + + it('forwards the test-only idle timeout to the server it starts', async () => { + vi.stubEnv(ORCAD_E2E_IDLE_TIMEOUT_ENV, '3000') + host = stoppedHost() + + await wakeStoppedManagedOrcad(slot) + + expect(launches()[0]).toContain(`${ORCAD_E2E_IDLE_TIMEOUT_ENV}='3000'`) + }) + + it('leaves a running server alone', async () => { + host = FakeOrcadHost.deployedOld() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' }) + expect(launches()).toEqual([]) + expect(acquireInstallLock).not.toHaveBeenCalled() + }) + + it('never starts a second process when the slot state is unprovable', async () => { + host = stoppedHost() + host.pidFiles.clear() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'unverifiable' }) + expect(launches()).toEqual([]) + }) + + it('defers to an update or recovery that holds the activation fence', async () => { + host = stoppedHost() + host.fence = true + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'fenced' }) + expect(launches()).toEqual([]) + expect(host.fence).toBe(true) + }) + + it('has nothing to start on a host with no activated server', async () => { + host = new FakeOrcadHost() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'not-activated' }) + expect(launches()).toEqual([]) + }) +}) diff --git a/src/main/ssh/orcad-managed-wake.ts b/src/main/ssh/orcad-managed-wake.ts new file mode 100644 index 00000000000..aa89056b4c9 --- /dev/null +++ b/src/main/ssh/orcad-managed-wake.ts @@ -0,0 +1,62 @@ +/** + * Starting a managed orcad that is installed and activated but not running, most often one + * that stopped itself after idling. A stopped server is just "not running": it is neither a + * failure nor evidence about terminals, which the daemon owns and which outlive orcad. + */ +import type { ServeReadiness } from '../server/serve-readiness' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { orcadActivationFenceExists, withOrcadActivationLock } from './orcad-activation-lock' +import { orcadLivenessProbeCommand, parseOrcadLiveness } from './orcad-remote-launch' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import { + ensureOrcadSlotServing, + orcadSlotDir, + resolveOrcadSlotIdentity, + type OrcadSlotOptions +} from './orcad-recovery-slot' + +export type OrcadManagedWake = + | { outcome: 'serving' | 'not-activated' | 'unverifiable' } + /** An update, rollback or recovery holds the host; it owns which slot serves. */ + | { outcome: 'fenced' } + | { outcome: 'started'; readiness: ServeReadiness } + +/** Launches the active slot only on proven exit; a live or unprovable process is left alone. */ +export async function wakeStoppedManagedOrcad( + options: OrcadSlotOptions, + onStarting: () => void = () => {} +): Promise { + const before = await readOrcadActivationRecord(options) + if (!before.active) { + return { outcome: 'not-activated' } + } + const liveness = await slotLiveness(options, before.active) + if (liveness !== 'DEAD') { + return { outcome: liveness === 'LIVE' ? 'serving' : 'unverifiable' } + } + if (await orcadActivationFenceExists(options)) { + return { outcome: 'fenced' } + } + return withOrcadActivationLock(options, async () => { + // Re-read under the fence: another client may have activated or started a slot meanwhile. + const active = (await readOrcadActivationRecord(options)).active + if (!active) { + return { outcome: 'not-activated' } + } + const identity = await resolveOrcadSlotIdentity(options, active) + onStarting() + return { outcome: 'started', readiness: await ensureOrcadSlotServing(options, identity) } + }) +} + +async function slotLiveness( + options: OrcadSlotOptions, + version: string +): Promise<'LIVE' | 'DEAD' | 'UNKNOWN'> { + return parseOrcadLiveness( + await execOrcadRemote( + options, + orcadLivenessProbeCommand(options.host, orcadSlotDir(options, version)) + ) + ) +} diff --git a/src/main/ssh/orcad-recovery-slot.ts b/src/main/ssh/orcad-recovery-slot.ts index 8e4cec5a768..1fb374a8eab 100644 --- a/src/main/ssh/orcad-recovery-slot.ts +++ b/src/main/ssh/orcad-recovery-slot.ts @@ -21,6 +21,7 @@ import { type OrcadStopOutcome } from './orcad-remote-process-control' import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { orcadActivationTransactionRoot } from './orcad-activation-lock' import { initialOrcadActivationAdmissionCommand, parseInitialOrcadActivationAdmission @@ -70,7 +71,9 @@ export function launchOrcadSlot( fullVersion: identity.version, userDataDir: options.userDataDir, bindHost: options.bindHost, - port: options.port + port: options.port, + // Every SSH launch is client-managed, so every one may idle out and be woken on connect. + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) }, expectation(identity) ) diff --git a/src/main/ssh/orcad-remote-launch-windows.ts b/src/main/ssh/orcad-remote-launch-windows.ts index 1a8c733063e..5aa0fedb192 100644 --- a/src/main/ssh/orcad-remote-launch-windows.ts +++ b/src/main/ssh/orcad-remote-launch-windows.ts @@ -33,7 +33,7 @@ import { ORCAD_READINESS_FILENAME, ORCAD_WINDOWS_PROCESS_FILENAME } from './orcad-remote-host-support' -import type { OrcadLaunchSpec } from './orcad-remote-launch' +import { orcadManagedLaunchEnv, type OrcadLaunchSpec } from './orcad-remote-launch' export class OrcadWindowsLaunchRefusedError extends Error { readonly code = 'orcad_windows_launch_refused' @@ -87,6 +87,10 @@ export function windowsOrcadLaunchCommand( `ORCA_VERSION=${spec.fullVersion}`, WINDOWS_BREAKAWAY_ENV_FLAG, `ORCA_USER_DATA=${spec.userDataDir}`, + ...orcadManagedLaunchEnv(spec).flatMap(([name, value]) => [ + WINDOWS_BREAKAWAY_ENV_FLAG, + `${name}=${value}` + ]), ORCAD_WINDOWS_BREAKAWAY_CONTRACT.argsFlag, '--json', '--bind', diff --git a/src/main/ssh/orcad-remote-launch.test.ts b/src/main/ssh/orcad-remote-launch.test.ts index 0be9ddc76c1..3730deb0ee1 100644 --- a/src/main/ssh/orcad-remote-launch.test.ts +++ b/src/main/ssh/orcad-remote-launch.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { ORCAD_MANAGED_ACTIVATION_ROOT_ENV } from '../../shared/orcad-idle-exit' import { ORCAD_READINESS_FILENAME, @@ -25,7 +26,8 @@ const SPEC = { fullVersion: '0.2.0+bb01', userDataDir: '/home/u/.orca', bindHost: '127.0.0.1', - port: 7777 + port: 7777, + activationRoot: '/home/u/.orca-remote/.orcad-activation-transaction' } const READY_LINE = JSON.stringify({ @@ -58,6 +60,14 @@ describe('orcadLaunchCommand', () => { expect(command).toContain(`ORCA_USER_DATA='${SPEC.userDataDir}'`) }) + it('names the activation fence on every launch, which enables idle exit', () => { + const command = orcadLaunchCommand(posix, SPEC) + expect(command).toContain(`${ORCAD_MANAGED_ACTIVATION_ROOT_ENV}='${SPEC.activationRoot}'`) + expect(command.indexOf(ORCAD_MANAGED_ACTIVATION_ROOT_ENV)).toBeLessThan( + command.indexOf('nohup') + ) + }) + it('declares the Windows refusal instead of emitting a command that cannot work', () => { expect(() => orcadLaunchCommand(windows, SPEC)).toThrow(OrcadRemoteLaunchUnsupportedError) }) diff --git a/src/main/ssh/orcad-remote-launch.ts b/src/main/ssh/orcad-remote-launch.ts index c133ddb4a88..28754f0c755 100644 --- a/src/main/ssh/orcad-remote-launch.ts +++ b/src/main/ssh/orcad-remote-launch.ts @@ -26,6 +26,11 @@ import type { ServeReadiness } from '../server/serve-readiness' import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime' import { ORCAD_STOP_REQUEST_FILENAME } from '../../shared/orcad-stop-request' import { windowsOrcadLivenessProbeCommand } from './orcad-remote-liveness-windows' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV, + readOrcadE2EIdleTimeoutMs +} from '../../shared/orcad-idle-exit' export { ORCAD_LOG_FILENAME, @@ -44,6 +49,22 @@ export type OrcadLaunchSpec = { /** Loopback by default; the client reaches it through an SSH local port-forward. */ bindHost: string port: number + /** The host's activation fence. Every SSH launch is client-managed, so every one may idle out. */ + activationRoot: string +} + +/** Env a managed launch adds; an older orcad ignores both. */ +export function orcadManagedLaunchEnv( + spec: OrcadLaunchSpec, + env: NodeJS.ProcessEnv = process.env +): [string, string][] { + const e2eTimeout = readOrcadE2EIdleTimeoutMs(env) + return [ + [ORCAD_MANAGED_ACTIVATION_ROOT_ENV, spec.activationRoot], + ...(e2eTimeout === null + ? [] + : [[ORCAD_E2E_IDLE_TIMEOUT_ENV, String(e2eTimeout)] satisfies [string, string]]) + ] } /** @@ -73,6 +94,7 @@ export function orcadLaunchCommand(host: RemoteHostPlatform, spec: OrcadLaunchSp 'umask 077 &&', `ORCA_VERSION=${shellEscape(spec.fullVersion)}`, `ORCA_USER_DATA=${shellEscape(spec.userDataDir)}`, + ...orcadManagedLaunchEnv(spec).map(([name, value]) => `${name}=${shellEscape(value)}`), // Keep $! equal to the runtime PID rather than a waiting shell's PID. `exec nohup "$orcad_runtime" ${entry}`, `--json --bind ${shellEscape(spec.bindHost)} --port ${String(spec.port)}`, diff --git a/src/main/ssh/orcad-remote-primitives.test.ts b/src/main/ssh/orcad-remote-primitives.test.ts index 8cc38a43eb0..290496deddb 100644 --- a/src/main/ssh/orcad-remote-primitives.test.ts +++ b/src/main/ssh/orcad-remote-primitives.test.ts @@ -212,7 +212,8 @@ describe('installed build identity and readiness', () => { fullVersion: '0.2.0+bb01', userDataDir: '/home/u/.orca', bindHost: '127.0.0.1', - port: 7777 + port: 7777, + activationRoot: '/home/u/.orca-remote/.orcad-activation-transaction' }, expectation ) diff --git a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts index 1ca251a2e1b..87022993655 100644 --- a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts +++ b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts @@ -134,7 +134,8 @@ async function launchTestRuntime( fullVersion: '0.2.0+bb01', userDataDir: dataDir, bindHost: '127.0.0.1', - port: 0 + port: 0, + activationRoot: join(dataDir, '.orcad-activation-transaction') }) if (legacyWrapper) { // The trailing command retains the old macOS waiting-shell behavior on every POSIX shell. diff --git a/src/main/ssh/orcad-remote-windows-commands.test.ts b/src/main/ssh/orcad-remote-windows-commands.test.ts index 7ea3489576d..d12f05cd66f 100644 --- a/src/main/ssh/orcad-remote-windows-commands.test.ts +++ b/src/main/ssh/orcad-remote-windows-commands.test.ts @@ -54,7 +54,8 @@ const spec: OrcadLaunchSpec = { fullVersion: '0.2.0+bb01', userDataDir: 'C:/Users/u/.orca', bindHost: '127.0.0.1', - port: 7777 + port: 7777, + activationRoot: `${base}/.orcad-activation-transaction` } function windowsConn(): { conn: SshConnection; writes: [string, string][] } { @@ -107,10 +108,22 @@ describe('Windows orcad commands run node.exe directly', () => { `${NODE} ${SCRIPT} slot-runtime ${slot} clear-stop-request` ) expect(windowsOrcadLaunchCommand(host, spec, SLOT_NODE)).toMatchInlineSnapshot( - `"C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.js --windows-breakaway-launch --stdout-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-readiness --stderr-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.log --process-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-process.json --env ORCA_VERSION=0.2.0+bb01 --env ORCA_USER_DATA=C:/Users/u/.orca --orcad-args --json --bind "127.0.0.1" --port "7777""` + `"C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.js --windows-breakaway-launch --stdout-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-readiness --stderr-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.log --process-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-process.json --env ORCA_VERSION=0.2.0+bb01 --env ORCA_USER_DATA=C:/Users/u/.orca --env ORCA_ORCAD_MANAGED_ACTIVATION_ROOT=C:/Users/u/.orca-remote/.orcad-activation-transaction --orcad-args --json --bind "127.0.0.1" --port "7777""` ) }) + it('passes the managed idle-exit fence through the breakaway environment', () => { + const command = windowsOrcadLaunchCommand( + host, + { ...spec, activationRoot: 'C:/Users/u/.orca-remote/.orcad-activation-transaction' }, + SLOT_NODE + ) + expect(command).toContain( + '--env ORCA_ORCAD_MANAGED_ACTIVATION_ROOT=C:/Users/u/.orca-remote/.orcad-activation-transaction --orcad-args' + ) + expect(command).not.toMatch(/[%$`']/u) + }) + it('never polls across SSH: runtime, launch, then one host-side wait', async () => { mockExec .mockResolvedValueOnce(encoded('__ORCAD_RUNTIME__', SLOT_NODE)) diff --git a/src/main/ssh/orcad-rollback-transition.ts b/src/main/ssh/orcad-rollback-transition.ts index a3d653affe5..fe9f27b78ce 100644 --- a/src/main/ssh/orcad-rollback-transition.ts +++ b/src/main/ssh/orcad-rollback-transition.ts @@ -21,7 +21,10 @@ import { } from './orcad-state-snapshot' import { orcadStopFreedTheHost } from './orcad-remote-process-control' import { joinRemotePath } from './ssh-remote-platform' -import type { OrcadActivationLockControl } from './orcad-activation-lock' +import { + orcadActivationTransactionRoot, + type OrcadActivationLockControl +} from './orcad-activation-lock' import type { OrcadRollbackTransaction } from './orcad-activation-transaction' import { createOrcadRollbackTransaction, @@ -209,7 +212,8 @@ export async function rollbackOrcadLocked( fullVersion: safety.target, userDataDir: options.userDataDir, bindHost: options.bindHost, - port: options.port + port: options.port, + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) }) } catch (error) { if (isUnconfirmedSshCommandTermination(error)) { diff --git a/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts b/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts index 7fa8a704a9d..82210ea7cf8 100644 --- a/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts +++ b/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts @@ -19,6 +19,7 @@ function deps(overrides: Partial = {}): HostServerOnCon return { managedEnvironmentId: () => null, ensureTunnel: vi.fn(async () => undefined), + ensureServing: vi.fn(async () => ({ state: 'serving' as const })), retireRetainedSource: vi.fn(async () => undefined), hasUnfinishedConversion: () => false, abandonConversion: vi.fn(async () => undefined), diff --git a/src/main/ssh/ssh-host-server-on-connect.test.ts b/src/main/ssh/ssh-host-server-on-connect.test.ts index 5d16e591192..fcc5836a190 100644 --- a/src/main/ssh/ssh-host-server-on-connect.test.ts +++ b/src/main/ssh/ssh-host-server-on-connect.test.ts @@ -17,6 +17,7 @@ function deps(overrides: Partial = {}): HostServerOnCon return { managedEnvironmentId: () => null, ensureTunnel: vi.fn(async () => undefined), + ensureServing: vi.fn(async () => ({ state: 'serving' as const })), retireRetainedSource: vi.fn(async () => undefined), hasUnfinishedConversion: () => false, abandonConversion: vi.fn(async () => undefined), @@ -51,6 +52,28 @@ describe('which server an SSH host runs on connect', () => { expect(d.convert).not.toHaveBeenCalled() }) + it('checks the server behind the tunnel on every connect to a converted host', async () => { + const d = deps({ managedEnvironmentId: () => 'env-9' }) + await resolveHostServerOnConnect(target, d) + expect(d.ensureServing).toHaveBeenCalledWith('env-9') + }) + + it('keeps a host whose stopped server could not be started managed, with the reason', async () => { + const detail = 'orcad did not become ready.\nLast lines of orcad.log:\nboom' + const d = deps({ + managedEnvironmentId: () => 'env-9', + ensureServing: vi.fn(async () => ({ state: 'unverifiable' as const, detail })) + }) + await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({ + route: 'managed', + environmentId: 'env-9', + serving: { state: 'unverifiable', detail } + }) + // Neither a relay fallback nor any verdict about the host's terminals. + expect(d.relayTerminals).not.toHaveBeenCalled() + expect(d.autoUpdate).not.toHaveBeenCalled() + }) + it('deploys an empty host directly, and converts a host with state', async () => { const empty = deps({ isEmptyHost: () => true }) await expect(resolveHostServerOnConnect(target, empty)).resolves.toMatchObject({ @@ -306,6 +329,27 @@ describe('which server an SSH host runs on connect', () => { expect(d.recordUpdateFailure).not.toHaveBeenCalled() }) + it('starts a stopped server before the update counts its terminals, as after a reboot', async () => { + const order: string[] = [] + const d = managed({ + ensureServing: vi.fn(async () => { + order.push('start') + return { state: 'started' as const, boundPort: null } + }), + // The restarted server's fresh daemon answers zero sessions, so the update goes ahead. + autoUpdate: vi.fn(async (_id, options) => { + order.push('update') + options.onUpdating() + return { outcome: 'updated' as const, activeVersion: '0.1.0+b' } + }) + }) + await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({ + route: 'managed', + environmentId: 'env-9' + }) + expect(order).toEqual(['start', 'update']) + }) + it('keeps the old version serving while terminals run, and retries on a later connect', async () => { const reason = '2 terminals are running on this host.' const d = managed({ diff --git a/src/main/ssh/ssh-host-server-on-connect.ts b/src/main/ssh/ssh-host-server-on-connect.ts index 43fbae5ae1e..216a71870f7 100644 --- a/src/main/ssh/ssh-host-server-on-connect.ts +++ b/src/main/ssh/ssh-host-server-on-connect.ts @@ -15,9 +15,11 @@ import type { } from '../../shared/orcad-managed-runtime' import type { SshManagedServerRelayReason, + SshManagedServerServingNote, SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' +import type { OrcadManagedServing } from './orcad-managed-serving' import { checkManagedServerUpdate, type ManagedServerUpdateDeps @@ -37,10 +39,15 @@ import { type HostServerReport } from './ssh-host-server-connect-events' -export type HostServerPhase = 'deploying' | 'converting' | 'connecting' | 'updating' +export type HostServerPhase = 'deploying' | 'converting' | 'connecting' | 'updating' | 'starting' export type HostServerOnConnectResult = - | { route: 'managed'; environmentId: string; update?: SshManagedServerUpdateNote } + | { + route: 'managed' + environmentId: string + update?: SshManagedServerUpdateNote + serving?: SshManagedServerServingNote + } | { route: 'relay' reason: SshManagedServerRelayReason @@ -58,6 +65,8 @@ export type HostServerTerminalVerdict = { export type HostServerOnConnectDeps = { managedEnvironmentId: (target: SshTarget) => string | null ensureTunnel: (environmentId: string) => Promise + /** Behind the tunnel: answers, or is started from its activated slot if proven stopped. */ + ensureServing: (environmentId: string) => Promise /** Retires a retained source once retirement is switched on; a failure only defers it. */ retireRetainedSource: (target: SshTarget) => Promise /** False when this build carries no orcad template, so nothing is tried on the host. */ @@ -136,6 +145,11 @@ async function decide( // Why before routing: until the commit lands the server is empty, so finish it or back out. return await convertHost(target, deps, trace, { checkTerminals: false }) } + const serving = await deps.ensureServing(existing) + if (serving.state === 'unverifiable') { + // Still the managed route: a stopped server says nothing about the host's terminals. + return { route: 'managed', environmentId: existing, serving } + } await deps.retireRetainedSource(target).catch((error: unknown) => { console.warn('[ssh] Source retirement deferred to a later connect:', error) }) diff --git a/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts b/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts index b7abdb242ee..379a802112a 100644 --- a/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts +++ b/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts @@ -31,6 +31,27 @@ describe('SSH host server status line', () => { }) }) + it('shows a stopped server starting, and why one could not be started', () => { + expect( + sshHostServerStatusLine(plain, { managedServer: { kind: 'setting-up', phase: 'starting' } }) + ?.text + ).toBe('Starting managed server…') + const detail = 'orcad did not become ready.\nLast lines of orcad.log:\nboom' + expect( + sshHostServerStatusLine(plain, { + managedServer: { + kind: 'managed', + environmentId: 'e', + serving: { state: 'unverifiable', detail } + } + }) + ).toEqual({ + tone: 'warning', + text: 'The managed Orca server isn’t running and couldn’t be started.', + detail + }) + }) + it('offers the move only while live relay terminals keep the host on the relay', () => { expect( sshHostServerStatusLine(plain, { diff --git a/src/renderer/src/components/settings/ssh-host-server-status-copy.ts b/src/renderer/src/components/settings/ssh-host-server-status-copy.ts index f518084ddfb..8e769ed4795 100644 --- a/src/renderer/src/components/settings/ssh-host-server-status-copy.ts +++ b/src/renderer/src/components/settings/ssh-host-server-status-copy.ts @@ -30,6 +30,16 @@ export function sshHostServerStatusLine( ) } } + if (status?.kind === 'managed' && status.serving) { + return { + tone: 'warning', + text: translate( + 'auto.components.settings.sshHostServer.notServing', + 'The managed Orca server isn’t running and couldn’t be started.' + ), + detail: status.serving.detail + } + } if (status?.kind === 'managed' && status.update) { return managedUpdateLine(status.update) } @@ -146,6 +156,11 @@ function settingUpLabel(phase: (typeof SSH_MANAGED_SERVER_PHASES)[number]): stri 'auto.components.settings.sshHostServer.updating', 'Updating managed server…' ) + case 'starting': + return translate( + 'auto.components.settings.sshHostServer.starting', + 'Starting managed server…' + ) } } diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 4eaa74b2e13..0b09b9266ba 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -12417,6 +12417,8 @@ "converting": "Moving this host’s projects to its managed Orca server…", "connecting": "Connecting to the managed Orca server…", "updating": "Updating managed server…", + "starting": "Starting managed server…", + "notServing": "The managed Orca server isn’t running and couldn’t be started.", "hostNewer": "Runs a managed Orca server from a newer Orca; it keeps that version.", "updateDeferred": "Runs a managed Orca server; it updates on a later connect: {{reason}}", "updateFailed": "Runs a managed Orca server on its previous version; updating it failed: {{reason}}", diff --git a/src/shared/orcad-idle-exit.ts b/src/shared/orcad-idle-exit.ts new file mode 100644 index 00000000000..c0034117646 --- /dev/null +++ b/src/shared/orcad-idle-exit.ts @@ -0,0 +1,40 @@ +/** + * Idle exit for an orcad a client launched over SSH, matching the relay: a host nobody has + * used for 15 minutes stops its server, and the next connect starts it again. + */ +import { z } from 'zod' + +/** + * Set only by a client's managed launch. Enables idle exit and names the host's activation + * fence, so an update or rollback in flight keeps the server up. User-started servers never + * carry it. + */ +export const ORCAD_MANAGED_ACTIVATION_ROOT_ENV = 'ORCA_ORCAD_MANAGED_ACTIVATION_ROOT' +/** Test-only quiet period; a client forwards it to the servers it launches. */ +export const ORCAD_E2E_IDLE_TIMEOUT_ENV = 'ORCA_E2E_ORCAD_IDLE_TIMEOUT_MS' +export const ORCAD_IDLE_EXIT_TIMEOUT_MS = 15 * 60_000 +const ORCAD_E2E_IDLE_TIMEOUT_MAX_MS = 60 * 60_000 + +/** The bounded test override, or null when unset or out of range. */ +export function readOrcadE2EIdleTimeoutMs(env: NodeJS.ProcessEnv): number | null { + const raw = env[ORCAD_E2E_IDLE_TIMEOUT_ENV] + const value = raw ? Number(raw) : Number.NaN + return Number.isSafeInteger(value) && value >= 1 && value <= ORCAD_E2E_IDLE_TIMEOUT_MAX_MS + ? value + : null +} + +/** Written to the data root just before an idle stop; the next start reports and clears it. */ +export const ORCAD_IDLE_STOP_RECORD_FILENAME = 'orcad-idle-stop.json' + +export const OrcadIdleStopRecordSchema = z.object({ + schemaVersion: z.literal(1), + kind: z.literal('orcad_idle_stop'), + pid: z.number().int().positive(), + version: z.string().max(255), + quietSince: z.iso.datetime({ offset: true }), + stoppedAt: z.iso.datetime({ offset: true }), + idleTimeoutMs: z.number().int().positive() +}) + +export type OrcadIdleStopRecord = z.infer diff --git a/src/shared/ssh-retained-payload-admission.ts b/src/shared/ssh-retained-payload-admission.ts index 9698cc39a95..ff522dec7e5 100644 --- a/src/shared/ssh-retained-payload-admission.ts +++ b/src/shared/ssh-retained-payload-admission.ts @@ -117,8 +117,18 @@ function admitSshManagedServerStatus(value: unknown): { managedServer?: SshManag return {} } const update = admitSshManagedServerUpdateNote('update' in value ? value.update : undefined) + const serving = + 'serving' in value && + value.serving && + typeof value.serving === 'object' && + 'state' in value.serving && + value.serving.state === 'unverifiable' && + 'detail' in value.serving && + typeof value.serving.detail === 'string' + ? { serving: { state: 'unverifiable' as const, detail: value.serving.detail } } + : {} return { - managedServer: { kind: 'managed', environmentId: value.environmentId, ...update } + managedServer: { kind: 'managed', environmentId: value.environmentId, ...update, ...serving } } } if (value.kind === 'setting-up' && 'phase' in value) { diff --git a/src/shared/ssh-types.ts b/src/shared/ssh-types.ts index 141868da409..35c27193a54 100644 --- a/src/shared/ssh-types.ts +++ b/src/shared/ssh-types.ts @@ -264,7 +264,8 @@ export const SSH_MANAGED_SERVER_PHASES = [ 'deploying', 'converting', 'connecting', - 'updating' + 'updating', + 'starting' ] as const export const SSH_MANAGED_SERVER_UPDATE_STATES = ['host-newer', 'deferred', 'failed'] as const @@ -275,6 +276,9 @@ export type SshManagedServerUpdateNote = { detail?: string } +/** A managed server that is down and could not be started, with why (and orcad.log's tail). */ +export type SshManagedServerServingNote = { state: 'unverifiable'; detail: string } + export const SSH_MANAGED_SERVER_RELAY_REASONS = [ 'orcad_unavailable', 'relay_terminals_live', @@ -288,7 +292,13 @@ export const SSH_MANAGED_SERVER_RELAY_REASONS = [ export type SshManagedServerRelayReason = (typeof SSH_MANAGED_SERVER_RELAY_REASONS)[number] export type SshManagedServerStatus = - | { kind: 'managed'; environmentId: string; update?: SshManagedServerUpdateNote } + | { + kind: 'managed' + environmentId: string + update?: SshManagedServerUpdateNote + /** Set when the server was not running and could not be started; never a terminal verdict. */ + serving?: SshManagedServerServingNote + } | { kind: 'setting-up'; phase: (typeof SSH_MANAGED_SERVER_PHASES)[number] } /** `detail` names the blocker for a refusal, or why orcad can't run on the host. */ | { diff --git a/tests/e2e/ssh-orcad-idle-exit.spec.ts b/tests/e2e/ssh-orcad-idle-exit.spec.ts new file mode 100644 index 00000000000..5ffa23d3428 --- /dev/null +++ b/tests/e2e/ssh-orcad-idle-exit.spec.ts @@ -0,0 +1,240 @@ +/** + * A managed orcad that stops is started again by the client, on a real host: + * + * 1. Idle: quit the client, the server exits and records an idle stop; a relaunched client + * reconnects and the server is running again with that record consumed. + * 2. Killed, then a reboot: a killed server comes back on the next call and adopts the daemon + * that kept its terminal; once both are gone, the next connect starts fresh. + * + * Docker only. `ORCA_E2E_ORCAD_CONVERT_TEMPLATE` names the linux-x64-glibc orcad build, and the + * client forwards a short test-only quiet period to the server it launches. + */ +import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import { expect, test } from './helpers/orca-app' +import { waitForSessionReady } from './helpers/store' +import { createRestartSession } from './helpers/orca-restart' +import { reconnect } from './helpers/orcad-convert-flow' +import { ORCAD_CONVERT_HOST_ENV } from './helpers/orcad-convert-host' +import { + cleanupDockerSshRelayTarget, + DOCKER_SSH_RELAY_REMOTE_REPO_PATH, + execDockerSshRelayTargetCommand, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { ORCAD_E2E_IDLE_TIMEOUT_ENV } from '../../src/shared/orcad-idle-exit' + +const HOST = process.env[ORCAD_CONVERT_HOST_ENV] +const TEMPLATE_SOURCE = process.env.ORCA_E2E_ORCAD_CONVERT_TEMPLATE +// Long enough that a connected client's own traffic never lets it lapse mid-test. +const IDLE_TIMEOUT_MS = 15_000 +const RECORD = '/root/.orca/orcad-idle-stop.json' + +/** PIDs of running orcad slots; empty once every slot has exited. */ +function runningOrcadPids(target: DockerSshRelayTarget): string[] { + return execDockerSshRelayTargetCommand( + target, + 'for f in /root/.orca-remote/orcad-*/.orcad-pid; do pid=$(cat "$f" 2>/dev/null) && ' + + 'kill -0 "$pid" 2>/dev/null && echo "$pid"; done; true' + ) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) +} + +function readIdleStopRecord(target: DockerSshRelayTarget): unknown { + const raw = execDockerSshRelayTargetCommand(target, `cat ${RECORD} 2>/dev/null || true`).trim() + return raw ? JSON.parse(raw) : null +} + +test('a managed orcad stops after idling and starts again on the next connect', async (// oxlint-disable-next-line no-empty-pattern -- Playwright's second fixture arg is testInfo; the first must be an object destructure to opt out of the default fixture set. +{}, testInfo) => { + test.skip( + HOST !== 'docker' || !TEMPLATE_SOURCE, + `Set ${ORCAD_CONVERT_HOST_ENV}=docker and ORCA_E2E_ORCAD_CONVERT_TEMPLATE` + ) + test.setTimeout(15 * 60_000) + const target = startDockerSshRelayTarget(testInfo) + const session = createRestartSession(testInfo, { + ORCA_ORCAD_TEMPLATE_PATH: TEMPLATE_SOURCE!, + [ORCAD_E2E_IDLE_TIMEOUT_ENV]: String(IDLE_TIMEOUT_MS) + }) + let app: ElectronApplication | null = null + try { + const first = await session.launch() + app = first.app + await waitForSessionReady(first.page) + // A managed host is reached through its server, not a relay, so no relay repo is added. + const remote = await first.page.evaluate( + async (input) => { + const { target: created } = await window.api.ssh.addTarget({ target: input }) + const state = await window.api.ssh.connect({ targetId: created.id }) + return { targetId: created.id, managedServer: state?.managedServer ?? null } + }, + { + label: `orcad idle E2E ${Date.now()}`, + host: target.host, + port: target.port, + username: 'root', + identityFile: target.identityFile, + identitiesOnly: true, + relayGracePeriodSeconds: 1 + } + ) + expect(remote.managedServer).toMatchObject({ kind: 'managed' }) + expect(runningOrcadPids(target)).toHaveLength(1) + + // While the client is connected the server stays up past its quiet period. + await first.page.waitForTimeout(IDLE_TIMEOUT_MS * 2) + expect(runningOrcadPids(target)).toHaveLength(1) + + await session.close(app) + app = null + await expect + .poll(() => runningOrcadPids(target), { timeout: 3 * 60_000 }) + .toEqual([]) + .catch((error: unknown) => { + // The server logs what kept it up; without it a timeout explains nothing. + console.error( + execDockerSshRelayTargetCommand(target, 'tail -n 40 /root/.orca-remote/orcad-*/orcad.log') + ) + throw error + }) + expect(readIdleStopRecord(target)).toMatchObject({ + kind: 'orcad_idle_stop', + idleTimeoutMs: IDLE_TIMEOUT_MS + }) + + const second = await session.launch() + app = second.app + await waitForSessionReady(second.page) + const connected = await reconnect(second.page, remote.targetId) + expect(JSON.parse(connected)).toMatchObject({ kind: 'managed' }) + expect(runningOrcadPids(target)).toHaveLength(1) + // The restarted server read the record, so a later crash cannot be mistaken for an idle stop. + expect(readIdleStopRecord(target)).toBeNull() + } finally { + if (app) { + await session.close(app) + } + await session.dispose() + cleanupDockerSshRelayTarget(target) + } +}) + +async function connectManagedHost( + page: Page, + target: DockerSshRelayTarget +): Promise<{ targetId: string; environmentId: string }> { + const connected = await page.evaluate( + async (input) => { + const { target: created } = await window.api.ssh.addTarget({ target: input }) + const state = await window.api.ssh.connect({ targetId: created.id }) + return { targetId: created.id, managedServer: state?.managedServer ?? null } + }, + { + label: `orcad restart E2E ${Date.now()}`, + host: target.host, + port: target.port, + username: 'root', + identityFile: target.identityFile, + identitiesOnly: true, + relayGracePeriodSeconds: 1 + } + ) + const server = connected.managedServer + if (server?.kind !== 'managed') { + throw new Error(`expected a managed server, got ${JSON.stringify(server)}`) + } + return { targetId: connected.targetId, environmentId: server.environmentId } +} + +async function callEnvironment(page: Page, environmentId: string, method: string, params: unknown) { + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params + }) + return response.ok ? { ok: true as const } : { ok: false as const, error: response.error } + }, + { environmentId, method, params } + ) +} + +/** The bracket keeps pgrep from matching the shell that runs it. */ +function processAlive(target: DockerSshRelayTarget, marker: string): boolean { + const found = execDockerSshRelayTargetCommand( + target, + `pgrep -f '[s]leep ${marker}' >/dev/null && echo yes || true` + ) + return found.trim() === 'yes' +} + +test('a killed managed orcad comes back with its terminal, and starts fresh after a reboot', async (// oxlint-disable-next-line no-empty-pattern -- Playwright's second fixture arg is testInfo; the first must be an object destructure to opt out of the default fixture set. +{}, testInfo) => { + test.skip( + HOST !== 'docker' || !TEMPLATE_SOURCE, + `Set ${ORCAD_CONVERT_HOST_ENV}=docker and ORCA_E2E_ORCAD_CONVERT_TEMPLATE` + ) + test.setTimeout(15 * 60_000) + const target = startDockerSshRelayTarget(testInfo) + const session = createRestartSession(testInfo, { ORCA_ORCAD_TEMPLATE_PATH: TEMPLATE_SOURCE! }) + let app: ElectronApplication | null = null + try { + const launched = await session.launch() + app = launched.app + const page = launched.page + await waitForSessionReady(page) + const { targetId, environmentId } = await connectManagedHost(page, target) + // A unique sleep length is the terminal's fingerprint in the process table. + const marker = String(800_000 + Math.floor(Math.random() * 100_000)) + expect( + await callEnvironment(page, environmentId, 'repo.add', { + path: DOCKER_SSH_RELAY_REMOTE_REPO_PATH + }) + ).toMatchObject({ ok: true }) + expect( + await callEnvironment(page, environmentId, 'terminal.create', { + worktree: `path:${DOCKER_SSH_RELAY_REMOTE_REPO_PATH}`, + command: `exec sleep ${marker}` + }) + ).toMatchObject({ ok: true }) + await expect.poll(() => processAlive(target, marker), { timeout: 60_000 }).toBe(true) + + // Killed: the daemon keeps the terminal, and the next call starts orcad, which adopts it. + execDockerSshRelayTargetCommand( + target, + 'kill -TERM $(cat /root/.orca-remote/orcad-*/.orcad-pid)' + ) + await expect.poll(() => runningOrcadPids(target), { timeout: 60_000 }).toEqual([]) + expect(processAlive(target, marker)).toBe(true) + await expect + .poll(async () => (await callEnvironment(page, environmentId, 'terminal.list', {})).ok, { + timeout: 3 * 60_000 + }) + .toBe(true) + expect(runningOrcadPids(target)).toHaveLength(1) + expect(processAlive(target, marker)).toBe(true) + + // A reboot takes orcad, the daemon and its terminal; the next connect starts both fresh. + execDockerSshRelayTargetCommand(target, "pkill -KILL -f '/root/[.]orca-remote/' || true") + execDockerSshRelayTargetCommand(target, `pkill -KILL -f '[s]leep ${marker}' || true`) + await expect.poll(() => runningOrcadPids(target), { timeout: 60_000 }).toEqual([]) + const connected = JSON.parse(await reconnect(page, targetId)) + expect(connected).toMatchObject({ kind: 'managed', environmentId }) + expect(connected).not.toHaveProperty('serving') + expect(runningOrcadPids(target)).toHaveLength(1) + expect(processAlive(target, marker)).toBe(false) + expect(await callEnvironment(page, environmentId, 'terminal.list', {})).toMatchObject({ + ok: true + }) + } finally { + if (app) { + await session.close(app) + } + await session.dispose() + cleanupDockerSshRelayTarget(target) + } +})