From 36b8cd81c7bf5ce68ba99aa7b10aa491f37b3d8d Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Sun, 4 Oct 2026 13:57:56 -0700 Subject: [PATCH] feat(orcad): managed orcad idles out after 15 minutes and is started again whenever it is down (#25121) * feat(orcad): a managed orcad stops after 15 idle minutes and starts again on the next connect A client-launched orcad now exits, like the relay, once no client, terminal, working agent, staged migration or activation fence has been seen for 15 minutes. The exit is the normal graceful shutdown, which leaves the terminal daemon running; the daemon retires only if it proves itself empty. A record in the data root tells the next start (and its readiness health) that the stop was an idle one rather than a crash. On connect and after host resume, a fresh tunnel whose server does not answer starts the activated slot under the activation fence, but only on a proven exit, so a stopped server reads as not running rather than a failure. * fix(orcad): keep orcad-entry under max-lines; idle e2e connects without a relay repo * fix(orcad): deploy and rollback launches carry the managed idle-exit fence The candidate launch in activation and the rollback launch built their own launch spec without the activation root, so a freshly deployed orcad never enabled idle exit; only the wake path did. The field is now required on every launch spec, so the type system covers each launch site. * feat(ssh): start a stopped managed orcad on connect, on restore and after resume Once orcad stopped (idle, kill or host reboot), a connect still resolved managed over a forward to a dead port and every call failed. Every connect now checks the server behind its tunnel, as does a call through a restored environment; a server proven stopped is started from its activated slot under the activation fence, adopting a surviving daemon and its terminals. The status line shows the start, and a start that fails keeps the host managed with the reason and orcad.log's tail, never as a terminal verdict. * fix(ssh): reuse a serving verdict only on the same SSH transport, for 5s A reconnect right after a reboot was answered from the previous transport's cached verdict, so the stopped server was never started. * fix(ssh): key the serving verdict on the tunnel's remote port too * test(ssh): a stopped server starts before the update counts its terminals * feat(ssh): check serving at the bound port, and follow a restarted orcad to a new one The serving check uses the port the tunnel forwards to (the one orcad bound). A restart that binds a different port drops the forward and rebuilds it at the new port, within the same ensure or on the explicit connect check. The tunnel manager class moves to its own file to stay under max-lines. --------- Co-authored-by: m4air --- .github/workflows/e2e.yml | 10 +- config/scripts/ci-e2e-job-selection.mjs | 5 +- config/scripts/pr-e2e-source-routing.mjs | 9 + .../scripts/ssh-docker-ci-sharding.test.mjs | 1 + docs/reference/orcad-operations.md | 32 ++ .../ipc/runtime-environment-managed-tunnel.ts | 32 +- src/main/ipc/ssh-connect-flow.ts | 7 +- src/main/ipc/ssh-host-server-connect.ts | 11 +- .../ipc/ssh-host-server-on-connect-wiring.ts | 2 + src/main/ipc/ssh.ts | 2 + ...untime-daemon-adoption.integration.test.ts | 3 +- src/main/orcad/orcad-entry.ts | 15 +- src/main/orcad/orcad-health.ts | 12 +- .../orcad/orcad-idle-exit-monitor.test.ts | 129 ++++++++ src/main/orcad/orcad-idle-exit-monitor.ts | 116 +++++++ src/main/orcad/orcad-idle-stop-record.test.ts | 57 ++++ src/main/orcad/orcad-idle-stop-record.ts | 60 ++++ .../orcad/orcad-managed-idle-exit-host.ts | 101 ++++++ .../orcad/orcad-managed-idle-exit.test.ts | 134 ++++++++ src/main/orcad/orcad-managed-idle-exit.ts | 116 +++++++ .../orcad-catalog-import.ts | 7 + .../runtime-rpc-client-activity.test.ts | 34 ++ .../runtime-rpc/runtime-rpc-lifecycle.ts | 25 +- .../runtime-rpc/runtime-rpc-shutdown.ts | 15 + .../runtime/runtime-rpc/runtime-rpc-state.ts | 12 + .../orcad-activation-crash-recovery.test.ts | 17 + src/main/ssh/orcad-installed-activation.ts | 8 +- src/main/ssh/orcad-managed-serving-verify.ts | 13 + src/main/ssh/orcad-managed-serving.test.ts | 127 ++++++++ src/main/ssh/orcad-managed-serving.ts | 157 +++++++++ src/main/ssh/orcad-managed-tunnel-manager.ts | 300 ++++++++++++++++++ src/main/ssh/orcad-managed-tunnel-resume.ts | 15 +- src/main/ssh/orcad-managed-tunnel-serving.ts | 45 +++ src/main/ssh/orcad-managed-tunnel.test.ts | 81 ++++- src/main/ssh/orcad-managed-tunnel.ts | 268 +--------------- src/main/ssh/orcad-managed-wake.test.ts | 118 +++++++ src/main/ssh/orcad-managed-wake.ts | 62 ++++ src/main/ssh/orcad-recovery-slot.ts | 5 +- src/main/ssh/orcad-remote-launch-windows.ts | 6 +- src/main/ssh/orcad-remote-launch.test.ts | 12 +- src/main/ssh/orcad-remote-launch.ts | 22 ++ src/main/ssh/orcad-remote-primitives.test.ts | 3 +- ...-remote-shell-commands.integration.test.ts | 3 +- .../ssh/orcad-remote-windows-commands.test.ts | 17 +- src/main/ssh/orcad-rollback-transition.ts | 8 +- ...h-host-server-on-connect-telemetry.test.ts | 1 + .../ssh/ssh-host-server-on-connect.test.ts | 44 +++ src/main/ssh/ssh-host-server-on-connect.ts | 18 +- .../ssh-host-server-status-copy.test.ts | 21 ++ .../settings/ssh-host-server-status-copy.ts | 15 + src/renderer/src/i18n/locales/en.json | 2 + src/shared/orcad-idle-exit.ts | 40 +++ src/shared/ssh-retained-payload-admission.ts | 12 +- src/shared/ssh-types.ts | 14 +- tests/e2e/ssh-orcad-idle-exit.spec.ts | 240 ++++++++++++++ 55 files changed, 2341 insertions(+), 300 deletions(-) create mode 100644 src/main/orcad/orcad-idle-exit-monitor.test.ts create mode 100644 src/main/orcad/orcad-idle-exit-monitor.ts create mode 100644 src/main/orcad/orcad-idle-stop-record.test.ts create mode 100644 src/main/orcad/orcad-idle-stop-record.ts create mode 100644 src/main/orcad/orcad-managed-idle-exit-host.ts create mode 100644 src/main/orcad/orcad-managed-idle-exit.test.ts create mode 100644 src/main/orcad/orcad-managed-idle-exit.ts create mode 100644 src/main/runtime/runtime-rpc-client-activity.test.ts create mode 100644 src/main/ssh/orcad-managed-serving-verify.ts create mode 100644 src/main/ssh/orcad-managed-serving.test.ts create mode 100644 src/main/ssh/orcad-managed-serving.ts create mode 100644 src/main/ssh/orcad-managed-tunnel-manager.ts create mode 100644 src/main/ssh/orcad-managed-tunnel-serving.ts create mode 100644 src/main/ssh/orcad-managed-wake.test.ts create mode 100644 src/main/ssh/orcad-managed-wake.ts create mode 100644 src/shared/orcad-idle-exit.ts create mode 100644 tests/e2e/ssh-orcad-idle-exit.spec.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8770700b818..0bf5b83812e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -343,7 +343,8 @@ jobs: . != "tests/e2e/terminal-ibus-hangul-native.spec.ts" and . != "tests/e2e/orcad-serve-mode-switch.spec.ts" and . != "tests/e2e/ssh-orcad-auto-convert.spec.ts" and - . != "tests/e2e/windows-missing-appdata-startup.spec.ts" + . != "tests/e2e/windows-missing-appdata-startup.spec.ts" and + . != "tests/e2e/ssh-orcad-idle-exit.spec.ts" )' <<<"$TEST_FILES_JSON" > "$RUNNER_TEMP/general-e2e-specs" fi mapfile -t TEST_FILES < "$RUNNER_TEMP/general-e2e-specs" @@ -594,12 +595,13 @@ jobs: retention-days: 7 if-no-files-found: ignore - # #24979 on a real host: a relay-era Docker host converts to managed orcad on connect. Needs the + # #24979 on a real host: a relay-era Docker host converts to managed orcad on connect, and a managed + # orcad idles out and restarts on the next connect. Needs the # orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds. orcad-auto-convert-docker: name: ssh host auto-converts to managed orcad (Docker) needs: [build, prepare-native-cache] - if: inputs.test_files == '' || inputs.ssh_source_changed == 'true' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-auto-convert.spec.ts') + if: inputs.test_files == '' || inputs.ssh_source_changed == 'true' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-auto-convert.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-idle-exit.spec.ts') runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -635,7 +637,7 @@ jobs: ORCA_RELAY_PATH: ${{ github.workspace }}/out/relay run: | export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template" - xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-auto-convert.spec.ts --project=electron-headless --workers=1 + xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-auto-convert.spec.ts tests/e2e/ssh-orcad-idle-exit.spec.ts --project=electron-headless --workers=1 - uses: actions/upload-artifact@v7 if: failure() with: diff --git a/config/scripts/ci-e2e-job-selection.mjs b/config/scripts/ci-e2e-job-selection.mjs index b31e68008eb..62050b9dbcb 100644 --- a/config/scripts/ci-e2e-job-selection.mjs +++ b/config/scripts/ci-e2e-job-selection.mjs @@ -43,6 +43,8 @@ export const ORCAD_SERVE_MODE_SWITCH_E2E_SPEC = 'tests/e2e/orcad-serve-mode-swit export const ORCAD_AUTO_CONVERT_E2E_SPEC = 'tests/e2e/ssh-orcad-auto-convert.spec.ts' // Windows-only; its own job runs it on a Windows runner. export const WINDOWS_MISSING_APPDATA_E2E_SPEC = 'tests/e2e/windows-missing-appdata-startup.spec.ts' +// Runs in the auto-convert job, which builds the template it needs. +export const ORCAD_IDLE_EXIT_E2E_SPEC = 'tests/e2e/ssh-orcad-idle-exit.spec.ts' export const DEDICATED_E2E_SPECS = [ ...DOCKER_SSH_E2E_SPECS, NODE_NETWORK_E2E_SPEC, @@ -50,7 +52,8 @@ export const DEDICATED_E2E_SPECS = [ NATIVE_IME_E2E_SPEC, ORCAD_SERVE_MODE_SWITCH_E2E_SPEC, ORCAD_AUTO_CONVERT_E2E_SPEC, - WINDOWS_MISSING_APPDATA_E2E_SPEC + WINDOWS_MISSING_APPDATA_E2E_SPEC, + ORCAD_IDLE_EXIT_E2E_SPEC ] const dedicatedSpecs = new Set(DEDICATED_E2E_SPECS) const dockerSpecs = new Set(DOCKER_SSH_E2E_SPECS) diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index bab26f13fd0..2284b0ca99c 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -38,6 +38,15 @@ export const PR_E2E_SOURCE_ROUTES = [ file ) }, + { + id: 'ssh.orcad-idle-exit', + specs: ['tests/e2e/ssh-orcad-idle-exit.spec.ts'], + matches: (file) => + isProductSource(file) && + /^src\/(?:main\/(?:orcad\/orcad-(?:idle-|managed-idle-)|ssh\/orcad-(?:managed-wake|managed-tunnel|recovery-slot|remote-launch))|shared\/orcad-idle-exit)/.test( + file + ) + }, { id: 'ssh.localhost-agent-hooks', specs: ['tests/e2e/ssh-localhost.spec.ts'], diff --git a/config/scripts/ssh-docker-ci-sharding.test.mjs b/config/scripts/ssh-docker-ci-sharding.test.mjs index 4afddbe6fed..39ca9e05d60 100644 --- a/config/scripts/ssh-docker-ci-sharding.test.mjs +++ b/config/scripts/ssh-docker-ci-sharding.test.mjs @@ -37,6 +37,7 @@ it('gives every removed SSH spec a dedicated owner even for test-only edits', () 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts', 'tests/e2e/ssh-localhost.spec.ts', 'tests/e2e/ssh-orcad-auto-convert.spec.ts', + 'tests/e2e/ssh-orcad-idle-exit.spec.ts', 'tests/e2e/terminal-ibus-hangul-native.spec.ts', 'tests/e2e/windows-missing-appdata-startup.spec.ts' ]) diff --git a/docs/reference/orcad-operations.md b/docs/reference/orcad-operations.md index 32133bce66d..e0a6dbe5296 100644 --- a/docs/reference/orcad-operations.md +++ b/docs/reference/orcad-operations.md @@ -301,6 +301,38 @@ What differs on a Windows SSH host, and what deliberately does not: versioned directories that nothing deletes while a process runs from them: Windows refuses to delete a running image, and GC treats an in-use slot as live. +## Idle exit (client-managed orcad only) + +An orcad that a desktop client launched over SSH stops itself, like the relay, once its host has +been unused for 15 minutes. The client's launch sets `ORCA_ORCAD_MANAGED_ACTIVATION_ROOT`; an +orcad started by hand, by a supervisor, or as a paired server never carries it and never idles +out. + +"Unused" means every one of these held on every check for the whole period: + +- no client socket open and no RPC request running; +- no terminal in the PTY provider, and the daemon answered with zero live sessions (a daemon + that does not answer keeps orcad up); +- no agent reporting `working`; +- no staged migration into this server; +- no activation fence on the host (an update, rollback, decommission or recovery in flight). + +The stop is the ordinary graceful shutdown, which disconnects from the daemon and never shuts it +down, so it cannot kill a terminal. It then asks the daemon to retire only if the daemon itself +proves it holds no session. Before stopping, orcad writes `/orcad-idle-stop.json`; +the next start reports it once as `health.previousIdleStop` and removes it, so a later crash is +never read as an idle stop. A managed start with no record reports `previousIdleStop: null`. + +The client starts a stopped server again, whatever stopped it (an idle stop, a kill, a host +reboot): on every connect, on every fresh tunnel (including after the client wakes from sleep), +and before a call through an environment the client restored at launch. A server that does not +answer is checked on the host; only a proven exit starts the activated slot, under the activation +fence, and the status line shows "Starting managed server…". A daemon that survived is adopted +with its terminals; after a reboot both start fresh. A process that is live or cannot be proven +gone is left alone, and a start that fails keeps the host managed with the reason and orcad.log's +tail, never as a verdict about its terminals. `ORCA_E2E_ORCAD_IDLE_TIMEOUT_MS` shortens the idle +period for tests; the client forwards it to the servers it launches. + ## Health The readiness payload carries a `health` object: diff --git a/src/main/ipc/runtime-environment-managed-tunnel.ts b/src/main/ipc/runtime-environment-managed-tunnel.ts index d8deb9066dc..fdd1f3f1024 100644 --- a/src/main/ipc/runtime-environment-managed-tunnel.ts +++ b/src/main/ipc/runtime-environment-managed-tunnel.ts @@ -1,7 +1,13 @@ import { resolveEnvironment } from '../../shared/runtime-environment-store' -import type { SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' +import type { + SshManagedServerStatus, + SshManagedServerUpdateNote, + SshTarget +} from '../../shared/ssh-types' import { managedServerUpdateDeps } from '../ssh/managed-server-update-deps' import { ensureOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' +import { verifyOrcadManagedServing } from '../ssh/orcad-managed-serving-verify' +import { setManagedOrcadStartListener } from '../ssh/orcad-managed-serving' import { updateManagedOrcadOnRestore } from '../ssh/orcad-managed-update-on-restore' import { setSshHostServerStatus } from '../ssh/ssh-host-server-status' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' @@ -14,6 +20,8 @@ export async function resolveManagedRuntimeEnvironment( ): Promise> { const environment = resolveEnvironment(userDataPath, selector) await ensureOrcadManagedTunnel(userDataPath, environment.id) + // Why: a server that stopped (idle, killed, host rebooted) starts before the call that needs it. + await verifyOrcadManagedServing(userDataPath, environment.id) // Why here: an auto-restored host may never see an SSH connect, so it would never update. void updateManagedOrcadOnRestore(environment.id, () => ({ ...managedServerUpdateDeps(userDataPath), @@ -32,12 +40,30 @@ function publishRestoreUpdate( phase: 'updating' | 'settled', note?: SshManagedServerUpdateNote ): void { - setSshHostServerStatus( - target.id, + publishHostServerStatus( + target, phase === 'updating' ? { kind: 'setting-up', phase: 'updating' } : { kind: 'managed', environmentId, ...(note ? { update: note } : {}) } ) +} + +/** Shows a managed server's start on the host's status line, wherever the start began. */ +export function installManagedOrcadStartStatus(): void { + setManagedOrcadStartListener({ + starting: (target) => + publishHostServerStatus(target, { kind: 'setting-up', phase: 'starting' }), + settled: (target, environmentId, serving) => + publishHostServerStatus(target, { + kind: 'managed', + environmentId, + ...(serving.state === 'unverifiable' ? { serving } : {}) + }) + }) +} + +function publishHostServerStatus(target: SshTarget, status: SshManagedServerStatus): void { + setSshHostServerStatus(target.id, status) // Only a host with a connection state has a status line to refresh. const state = connectionManager?.getState(target.id) if (state) { diff --git a/src/main/ipc/ssh-connect-flow.ts b/src/main/ipc/ssh-connect-flow.ts index 1e36532c850..d2f16a07a8f 100644 --- a/src/main/ipc/ssh-connect-flow.ts +++ b/src/main/ipc/ssh-connect-flow.ts @@ -179,7 +179,12 @@ async function doConnect( throw createCancelledConnectAttemptError() } if (server?.route === 'managed') { - return publishManagedServerConnect(targetId, server.environmentId, server.update) + return publishManagedServerConnect( + targetId, + server.environmentId, + server.update, + server.serving + ) } if (server) { recordRelayDecision(target, server) diff --git a/src/main/ipc/ssh-host-server-connect.ts b/src/main/ipc/ssh-host-server-connect.ts index 45b7ea00051..ff7735f7d79 100644 --- a/src/main/ipc/ssh-host-server-connect.ts +++ b/src/main/ipc/ssh-host-server-connect.ts @@ -2,6 +2,7 @@ import { getAppEnvironment } from '../../shared/app-environment' import type { SshConnectionState, + SshManagedServerServingNote, SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' @@ -38,9 +39,15 @@ export async function decideHostServer( export function publishManagedServerConnect( targetId: string, environmentId: string, - update?: SshManagedServerUpdateNote + update?: SshManagedServerUpdateNote, + serving?: SshManagedServerServingNote ): SshConnectionState { - const managedServer = { kind: 'managed' as const, environmentId, ...(update ? { update } : {}) } + const managedServer = { + kind: 'managed' as const, + environmentId, + ...(update ? { update } : {}), + ...(serving ? { serving } : {}) + } setSshHostServerStatus(targetId, managedServer) const state: SshConnectionState = { ...(connectionManager!.getState(targetId) ?? { targetId, reconnectAttempt: 0 }), diff --git a/src/main/ipc/ssh-host-server-on-connect-wiring.ts b/src/main/ipc/ssh-host-server-on-connect-wiring.ts index fa78cd2e02f..bbd72437f31 100644 --- a/src/main/ipc/ssh-host-server-on-connect-wiring.ts +++ b/src/main/ipc/ssh-host-server-on-connect-wiring.ts @@ -14,6 +14,7 @@ import { assessOrcadMigrationTerminals } from '../ssh/orcad-migration-terminal-g import { hasOrcadTemplate } from '../ssh/orcad-artifact-materializer' import { managedServerUpdateDeps } from '../ssh/managed-server-update-deps' import { ensureOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' +import { verifyOrcadManagedServing } from '../ssh/orcad-managed-serving-verify' import { convertSshTargetToManagedOrcad } from '../ssh/orcad-runtime-conversion' import { orcadMigrationDestinationFor } from '../ssh/orcad-runtime-conversion-wiring' import { createManagedOrcadEnvironment } from '../ssh/orcad-runtime-deployment' @@ -49,6 +50,7 @@ export function hostServerOnConnectDeps(userDataPath: string): HostServerOnConne ensureTunnel: async (environmentId) => { await ensureOrcadManagedTunnel(userDataPath, environmentId) }, + ensureServing: (environmentId) => verifyOrcadManagedServing(userDataPath, environmentId), retireRetainedSource: async (target) => { if (isOrcadSourceRetirementEnabled()) { await retireRetainedOrcadSourceChain(userDataPath, store, target, runTargetLifecycle) diff --git a/src/main/ipc/ssh.ts b/src/main/ipc/ssh.ts index 9c34afb4fd9..7f97f2d3577 100644 --- a/src/main/ipc/ssh.ts +++ b/src/main/ipc/ssh.ts @@ -41,6 +41,7 @@ import { } from '../ssh/ssh-connection-generation' import { resetSshProviderAuthorities } from '../ssh/ssh-provider-authority' import { activeSessions } from './ssh-active-relay-sessions' +import { installManagedOrcadStartStatus } from './runtime-environment-managed-tunnel' import { registerAdvertisedUrlRefresh, unregisterAdvertisedUrlRefresh @@ -190,6 +191,7 @@ export function registerSshHandlers( setPersistedStore(store) reconcileManagedOrcadSshTargets(getAppEnvironment().getPath('userData'), store) registerAdvertisedUrlRefresh(getCurrentMainWindow) + installManagedOrcadStartStatus() registerCredentialHandler() diff --git a/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts b/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts index a80a6c9b9ec..8e3e2d3be9a 100644 --- a/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts +++ b/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts @@ -179,7 +179,8 @@ async function launch(slot: Slot, userDataDir: string): Promise fullVersion: slot.version, userDataDir, bindHost: '127.0.0.1', - port: 0 + port: 0, + activationRoot: join(userDataDir, '.orcad-activation-transaction') }) ) ).trim() diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index ea311efb7d8..c72fb0eab2b 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -26,6 +26,7 @@ import type { OrcadRuntimeCleanup } from './orcad-runtime-lifetime' import { installOrcadStopRequestListeners } from './orcad-stop-request-listener' import { prepareOrcadManagedStop } from './orcad-managed-stop-admission' import type { OrcadManagedStopContext } from '../../shared/orcad-stop-request' +import { beginOrcadIdleExit, bindOrcadIdleShutdown } from './orcad-managed-idle-exit-host' import { changedAiVaultSearchSettings, type AiVaultSearchSettings @@ -191,6 +192,7 @@ async function startOrcadRuntime( const { resolvePushGatewayOrigin } = await import('../runtime/push/push-gateway-origin') const runtimeUserDataPath = getAppEnvironment().getPath('userData') + const idleExitStartup = beginOrcadIdleExit(runtimeUserDataPath) const { store: profileStore, authority: profileStateAuthority } = await createOrcadProfileStateStartup(runtimeUserDataPath) const observedPaneIdentities = new AgentStatusObservedPaneIdentities() @@ -386,7 +388,11 @@ async function startOrcadRuntime( // Why in the readiness payload: this is the one message a supervisor and a deploy // transaction both read, and a green orcad with a dead daemon is exactly the // looks-healthy-but-useless state they must not activate. - health: await collectOrcadHealth(getAppEnvironment().getVersion(), profileStateAuthority) + health: await collectOrcadHealth( + getAppEnvironment().getVersion(), + profileStateAuthority, + idleExitStartup.previousIdleStop + ) } await new ServeReadinessPublisher().publish( @@ -396,6 +402,12 @@ async function startOrcadRuntime( : { mode: options.json ? 'json' : 'human' } ) + await idleExitStartup.start({ + rpc, + agentStates: () => agentHookServer.getStatusSnapshot(), + hasStagedMigration: () => profileStore.hasStagedOrcadMigrationCatalog(), + registerCleanup + }) return { readiness } } @@ -427,4 +439,5 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise { return { buildHash: computeOrcadBuildHash(), @@ -173,6 +180,7 @@ export async function collectOrcadHealth( pid: process.pid, terminalDaemon: await collectTerminalDaemonHealth(), ...(profileStateAuthority ? { profileStateAuthority } : {}), - stopRequests: ORCAD_STOP_REQUESTS_CAPABILITY + stopRequests: ORCAD_STOP_REQUESTS_CAPABILITY, + ...(previousIdleStop !== undefined ? { previousIdleStop } : {}) } } diff --git a/src/main/orcad/orcad-idle-exit-monitor.test.ts b/src/main/orcad/orcad-idle-exit-monitor.test.ts new file mode 100644 index 00000000000..fbe31074a52 --- /dev/null +++ b/src/main/orcad/orcad-idle-exit-monitor.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it, vi } from 'vitest' +import { + OrcadIdleExitMonitor, + resolveOrcadIdlePollMs, + type OrcadIdleProbe, + type OrcadIdleVerdict +} from './orcad-idle-exit-monitor' + +function harness(options: { timeoutMs?: number; lastClientActivityAt?: number } = {}) { + let now = 1_000 + let lastActivity = options.lastClientActivityAt ?? 0 + const verdicts: Record = { clients: 'idle', terminals: 'idle' } + const probes: OrcadIdleProbe[] = Object.keys(verdicts).map((name) => ({ + name, + read: () => { + const verdict = verdicts[name] + if (verdict instanceof Error) { + throw verdict + } + return verdict + } + })) + const onIdle = vi.fn() + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: options.timeoutMs ?? 100, + probes, + lastClientActivityAt: () => lastActivity, + onIdle, + now: () => now, + log: () => {} + }) + return { + monitor, + onIdle, + verdicts, + advance: (ms: number) => (now += ms), + touch: () => (lastActivity = now) + } +} + +describe('OrcadIdleExitMonitor', () => { + it('fires once every probe has stayed idle for the whole quiet period', async () => { + const h = harness() + expect(await h.monitor.check()).toBe(false) + h.advance(99) + expect(await h.monitor.check()).toBe(false) + h.advance(1) + expect(await h.monitor.check()).toBe(true) + expect(h.onIdle).toHaveBeenCalledWith({ quietSince: 1_000, stoppedAt: 1_100, timeoutMs: 100 }) + h.advance(1_000) + expect(await h.monitor.check()).toBe(false) + expect(h.onIdle).toHaveBeenCalledTimes(1) + }) + + it('restarts the quiet period whenever any probe is busy', async () => { + const h = harness() + await h.monitor.check() + h.advance(90) + h.verdicts.terminals = 'busy' + expect(await h.monitor.check()).toBe(false) + h.verdicts.terminals = 'idle' + h.advance(10) + expect(await h.monitor.check()).toBe(false) + h.advance(99) + expect(await h.monitor.check()).toBe(false) + h.advance(1) + expect(await h.monitor.check()).toBe(true) + }) + + it.each([ + ['unverifiable', 'unverifiable' as const], + ['throwing', new Error('daemon did not answer')] + ])('treats a %s probe as busy, never as idle', async (_label, verdict) => { + const h = harness() + h.verdicts.terminals = verdict + for (let i = 0; i < 5; i += 1) { + expect(await h.monitor.check()).toBe(false) + h.advance(100) + } + expect(h.onIdle).not.toHaveBeenCalled() + }) + + it('counts a request that came and went between checks as activity', async () => { + const h = harness() + await h.monitor.check() + h.advance(80) + h.touch() + h.advance(20) + expect(await h.monitor.check()).toBe(false) + h.advance(80) + expect(await h.monitor.check()).toBe(true) + }) + + it('does not fire after it was stopped', async () => { + const h = harness() + await h.monitor.check() + h.monitor.stop() + h.advance(1_000) + expect(await h.monitor.check()).toBe(false) + expect(h.onIdle).not.toHaveBeenCalled() + }) + + it('polls often enough for a short test timeout and at most once a minute', () => { + expect(resolveOrcadIdlePollMs(15 * 60_000)).toBe(60_000) + expect(resolveOrcadIdlePollMs(2_000)).toBe(400) + expect(resolveOrcadIdlePollMs(10)).toBe(250) + }) + + it('stops itself on a timer once idle', async () => { + vi.useFakeTimers() + try { + const onIdle = vi.fn() + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: 1_000, + probes: [{ name: 'clients', read: () => 'idle' }], + lastClientActivityAt: () => 0, + onIdle, + log: () => {} + }) + monitor.start() + await vi.advanceTimersByTimeAsync(1_600) + expect(onIdle).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(5_000) + expect(onIdle).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/orcad/orcad-idle-exit-monitor.ts b/src/main/orcad/orcad-idle-exit-monitor.ts new file mode 100644 index 00000000000..58a053adffc --- /dev/null +++ b/src/main/orcad/orcad-idle-exit-monitor.ts @@ -0,0 +1,116 @@ +/** + * Decides when a managed orcad has been unused long enough to stop. + * + * Every probe must answer `idle` on the same check, continuously for the whole quiet period. + * A probe that throws or cannot answer counts as busy: silence is never evidence of idleness. + */ + +export type OrcadIdleVerdict = 'idle' | 'busy' | 'unverifiable' + +export type OrcadIdleProbe = { + name: string + read: () => OrcadIdleVerdict | Promise +} + +export type OrcadIdleExitEvidence = { quietSince: number; stoppedAt: number; timeoutMs: number } + +export type OrcadIdleExitMonitorOptions = { + timeoutMs: number + probes: readonly OrcadIdleProbe[] + /** Any client request restarts the quiet period, even one that came and went between checks. */ + lastClientActivityAt: () => number + onIdle: (evidence: OrcadIdleExitEvidence) => void + now?: () => number + pollMs?: number + log?: (line: string) => void +} + +export function resolveOrcadIdlePollMs(timeoutMs: number): number { + return Math.min(60_000, Math.max(250, Math.floor(timeoutMs / 5))) +} + +export class OrcadIdleExitMonitor { + private timer: ReturnType | null = null + private quietSince: number | null = null + private blocker: string | null = null + private stopped = false + private readonly now: () => number + private readonly pollMs: number + private readonly log: (line: string) => void + + constructor(private readonly options: OrcadIdleExitMonitorOptions) { + this.now = options.now ?? Date.now + this.pollMs = options.pollMs ?? resolveOrcadIdlePollMs(options.timeoutMs) + this.log = options.log ?? ((line) => console.error(line)) + } + + start(): void { + this.schedule() + } + + stop(): void { + this.stopped = true + if (this.timer) { + clearTimeout(this.timer) + this.timer = null + } + } + + /** One check; resolves true once the quiet period elapsed and `onIdle` fired. */ + async check(): Promise { + const blocker = await this.findBlocker() + if (this.stopped) { + return false + } + const now = this.now() + if (blocker) { + if (this.quietSince !== null || this.blocker !== blocker) { + this.log(`[orcad] idle exit waiting: ${blocker}`) + } + this.quietSince = null + this.blocker = blocker + return false + } + if (this.quietSince === null) { + this.quietSince = now + this.blocker = null + this.log(`[orcad] idle; stopping after ${this.options.timeoutMs}ms unless a client returns`) + } + const quietSince = Math.max(this.quietSince, this.options.lastClientActivityAt()) + if (now - quietSince < this.options.timeoutMs) { + return false + } + this.stop() + this.options.onIdle({ quietSince, stoppedAt: now, timeoutMs: this.options.timeoutMs }) + return true + } + + private async findBlocker(): Promise { + for (const probe of this.options.probes) { + let verdict: OrcadIdleVerdict + try { + verdict = await probe.read() + } catch { + verdict = 'unverifiable' + } + if (verdict !== 'idle') { + return `${probe.name} ${verdict}` + } + } + return null + } + + private schedule(): void { + if (this.stopped) { + return + } + this.timer = setTimeout(() => { + this.timer = null + void this.check() + .catch((error: unknown) => this.log(`[orcad] idle check failed: ${String(error)}`)) + .finally(() => this.schedule()) + }, this.pollMs) + // Never the reason the process stays alive. + this.timer.unref?.() + } +} diff --git a/src/main/orcad/orcad-idle-stop-record.test.ts b/src/main/orcad/orcad-idle-stop-record.test.ts new file mode 100644 index 00000000000..2e5c5659f7e --- /dev/null +++ b/src/main/orcad/orcad-idle-stop-record.test.ts @@ -0,0 +1,57 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + consumeOrcadIdleStopRecord, + orcadIdleStopRecordPath, + writeOrcadIdleStopRecord +} from './orcad-idle-stop-record' + +let root: string + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orcad-idle-stop-')) +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) + vi.restoreAllMocks() +}) + +describe('orcad idle-stop record', () => { + it('lets the next start tell an idle stop apart from a crash, exactly once', () => { + writeOrcadIdleStopRecord( + root, + { + quietSince: Date.parse('2026-10-03T10:00:00Z'), + stoppedAt: Date.parse('2026-10-03T10:15:00Z'), + timeoutMs: 900_000 + }, + '1.2.3' + ) + + expect(consumeOrcadIdleStopRecord(root)).toMatchObject({ + kind: 'orcad_idle_stop', + pid: process.pid, + version: '1.2.3', + quietSince: '2026-10-03T10:00:00.000Z', + stoppedAt: '2026-10-03T10:15:00.000Z', + idleTimeoutMs: 900_000 + }) + // A crash after this start must not inherit the earlier idle stop. + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + }) + + it('reads a start with no record as "not an idle stop"', () => { + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + }) + + it('drops a corrupt record instead of trusting it', () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + writeFileSync(orcadIdleStopRecordPath(root), '{"kind":"orcad_idle_stop"') + + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + expect(existsSync(orcadIdleStopRecordPath(root))).toBe(false) + }) +}) diff --git a/src/main/orcad/orcad-idle-stop-record.ts b/src/main/orcad/orcad-idle-stop-record.ts new file mode 100644 index 00000000000..3656d09415d --- /dev/null +++ b/src/main/orcad/orcad-idle-stop-record.ts @@ -0,0 +1,60 @@ +/** + * The record that tells a clean idle stop apart from a crash. Written just before an idle + * stop; the next start reports it once and removes it, so a later crash never inherits it. + */ +import { rmSync } from 'node:fs' +import { join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { writeDurableSecureJsonFile } from '../../shared/secure-file' +import { + ORCAD_IDLE_STOP_RECORD_FILENAME, + OrcadIdleStopRecordSchema, + type OrcadIdleStopRecord +} from '../../shared/orcad-idle-exit' +import type { OrcadIdleExitEvidence } from './orcad-idle-exit-monitor' + +const RECORD_MAX_BYTES = 16 * 1024 + +export function orcadIdleStopRecordPath(userDataPath: string): string { + return join(userDataPath, ORCAD_IDLE_STOP_RECORD_FILENAME) +} + +export function writeOrcadIdleStopRecord( + userDataPath: string, + evidence: OrcadIdleExitEvidence, + version: string +): void { + const record: OrcadIdleStopRecord = { + schemaVersion: 1, + kind: 'orcad_idle_stop', + pid: process.pid, + version, + quietSince: new Date(evidence.quietSince).toISOString(), + stoppedAt: new Date(evidence.stoppedAt).toISOString(), + idleTimeoutMs: evidence.timeoutMs + } + if (!writeDurableSecureJsonFile(orcadIdleStopRecordPath(userDataPath), record)) { + throw new Error('orcad_idle_stop_record_permissions_unconfirmed') + } +} + +/** The previous run's idle stop, or null when it ended any other way (or never ran). */ +export function consumeOrcadIdleStopRecord(userDataPath: string): OrcadIdleStopRecord | null { + const path = orcadIdleStopRecordPath(userDataPath) + let record: OrcadIdleStopRecord | null = null + try { + const raw = readNodeFileSyncWithinLimit(path, RECORD_MAX_BYTES).buffer.toString('utf8') + record = OrcadIdleStopRecordSchema.parse(JSON.parse(raw)) + } catch (error) { + if (isMissing(error)) { + return null + } + console.error('[orcad] ignoring an unreadable idle-stop record:', error) + } + rmSync(path, { force: true }) + return record +} + +function isMissing(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT' +} diff --git a/src/main/orcad/orcad-managed-idle-exit-host.ts b/src/main/orcad/orcad-managed-idle-exit-host.ts new file mode 100644 index 00000000000..022e2fda01b --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit-host.ts @@ -0,0 +1,101 @@ +/** Binds managed idle exit to this orcad's RPC server, PTY provider and terminal daemon. */ +import type { RuntimeRpcClientActivity } from '../runtime/runtime-rpc/runtime-rpc-shutdown' +import type { OrcadIdleExitEvidence } from './orcad-idle-exit-monitor' +import { + activationFenceExists, + installOrcadManagedIdleExit, + resolveOrcadManagedIdleExit, + type OrcadManagedIdleExitConfig +} from './orcad-managed-idle-exit' +import { consumeOrcadIdleStopRecord, writeOrcadIdleStopRecord } from './orcad-idle-stop-record' +import type { OrcadIdleStopRecord } from '../../shared/orcad-idle-exit' + +let requestIdleShutdown: ((reason: string) => void) | null = null + +/** main binds its shutdown once signal handling exists; the quiet period outlasts that gap. */ +export function bindOrcadIdleShutdown(request: (reason: string) => void): void { + requestIdleShutdown = request +} + +type OrcadIdleExitRuntimePorts = { + rpc: { readClientActivity(): RuntimeRpcClientActivity } + agentStates: () => readonly { state: string }[] + hasStagedMigration: () => boolean + /** Shutdown stops the monitor first, so a signal stop is never recorded as an idle one. */ + registerCleanup: (cleanup: () => void) => void +} + +/** + * Runs under the instance lock at startup: reads the previous run's idle-stop record before + * anything this run does could be mistaken for it. Inert for an orcad no client launched. + */ +export function beginOrcadIdleExit(userDataPath: string): { + previousIdleStop: OrcadIdleStopRecord | null | undefined + start: (ports: OrcadIdleExitRuntimePorts) => Promise +} { + const config = resolveOrcadManagedIdleExit(process.env) + if (!config) { + return { previousIdleStop: undefined, start: async () => {} } + } + const previousIdleStop = consumeOrcadIdleStopRecord(userDataPath) + if (previousIdleStop) { + console.error(`[orcad] the previous run stopped idle at ${previousIdleStop.stoppedAt}`) + } + const version = process.env.ORCA_VERSION ?? '0.0.0-orcad' + return { + previousIdleStop, + start: (ports) => startOrcadManagedIdleExit({ ...ports, config, userDataPath, version }) + } +} + +async function startOrcadManagedIdleExit( + input: OrcadIdleExitRuntimePorts & { + config: OrcadManagedIdleExitConfig + userDataPath: string + version: string + } +): Promise { + const { getLocalPtyProvider } = await import('../ipc/pty') + const { getDaemonEndpointFacts } = await import('../daemon/daemon-init') + const { countLiveOrcadDaemonSessions, retireOrcadDaemonIfIdle } = + await import('./orcad-daemon-retirement') + const dispose = installOrcadManagedIdleExit({ + config: input.config, + ports: { + readClientActivity: () => input.rpc.readClientActivity(), + listTerminals: () => getLocalPtyProvider().listProcesses(), + countDaemonSessions: countLiveOrcadDaemonSessions, + hasDaemon: () => getDaemonEndpointFacts() !== null, + agentStates: input.agentStates, + hasStagedMigration: input.hasStagedMigration, + activationFenceExists + }, + stop: (evidence) => { + void stopForIdle(input, evidence, retireOrcadDaemonIfIdle).finally(() => + requestIdleShutdown?.('idle') + ) + } + }) + input.registerCleanup(dispose) +} + +async function stopForIdle( + input: { userDataPath: string; version: string }, + evidence: OrcadIdleExitEvidence, + retireDaemon: () => Promise<{ retirement: string; reason: string | null }> +): Promise { + console.error(`[orcad] stopping: no client, terminal or job for ${evidence.timeoutMs}ms`) + try { + writeOrcadIdleStopRecord(input.userDataPath, evidence, input.version) + } catch (error) { + console.error('[orcad] could not record the idle stop:', error) + } + // The daemon leaves only if it proves itself empty; a busy one stays up with its terminals. + const outcome = await retireDaemon().catch((error: unknown) => ({ + retirement: 'unverifiable', + reason: String(error) + })) + console.error( + `[orcad] terminal daemon on idle stop: ${outcome.retirement}${outcome.reason ? ` (${outcome.reason})` : ''}` + ) +} diff --git a/src/main/orcad/orcad-managed-idle-exit.test.ts b/src/main/orcad/orcad-managed-idle-exit.test.ts new file mode 100644 index 00000000000..f3f8ed10e74 --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit.test.ts @@ -0,0 +1,134 @@ +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + activationFenceExists, + createOrcadIdleProbes, + resolveOrcadManagedIdleExit, + type OrcadManagedIdleExitPorts +} from './orcad-managed-idle-exit' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_IDLE_EXIT_TIMEOUT_MS, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV +} from '../../shared/orcad-idle-exit' + +const config = { timeoutMs: 1_000, activationRoot: '/home/u/.orca-remote/.fence' } + +function idlePorts(): OrcadManagedIdleExitPorts { + return { + readClientActivity: () => ({ openConnections: 0, requestsInFlight: 0, lastRequestAt: 0 }), + listTerminals: async () => [], + countDaemonSessions: async () => 0, + hasDaemon: () => true, + agentStates: () => [{ state: 'done' }], + hasStagedMigration: () => false, + activationFenceExists: async () => false + } +} + +async function verdicts(ports: OrcadManagedIdleExitPorts): Promise> { + const entries = await Promise.all( + createOrcadIdleProbes(config, ports).map(async (probe) => [probe.name, await probe.read()]) + ) + return Object.fromEntries(entries) +} + +describe('resolveOrcadManagedIdleExit', () => { + it('stays off for a server the user started or paired', () => { + expect(resolveOrcadManagedIdleExit({})).toBeNull() + expect(resolveOrcadManagedIdleExit({ [ORCAD_E2E_IDLE_TIMEOUT_ENV]: '50' })).toBeNull() + }) + + it('matches the relay quiet period for a managed launch', () => { + expect(resolveOrcadManagedIdleExit({ [ORCAD_MANAGED_ACTIVATION_ROOT_ENV]: '/f' })).toEqual({ + timeoutMs: ORCAD_IDLE_EXIT_TIMEOUT_MS, + activationRoot: '/f' + }) + expect(ORCAD_IDLE_EXIT_TIMEOUT_MS).toBe(15 * 60_000) + }) + + it.each([ + ['3000', 3_000], + ['0', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['-1', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['1.5', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['abc', ORCAD_IDLE_EXIT_TIMEOUT_MS], + [String(2 * 60 * 60_000), ORCAD_IDLE_EXIT_TIMEOUT_MS] + ])('accepts only a bounded test timeout (%s)', (raw, expected) => { + expect( + resolveOrcadManagedIdleExit({ + [ORCAD_MANAGED_ACTIVATION_ROOT_ENV]: '/f', + [ORCAD_E2E_IDLE_TIMEOUT_ENV]: raw + })?.timeoutMs + ).toBe(expected) + }) +}) + +describe('createOrcadIdleProbes', () => { + it('reads an unused host as idle on every probe', async () => { + expect(await verdicts(idlePorts())).toEqual({ + clients: 'idle', + terminals: 'idle', + agents: 'idle', + migration: 'idle', + activation: 'idle' + }) + }) + + it.each<[string, Partial, string]>([ + [ + 'an open client socket', + { + readClientActivity: () => ({ openConnections: 1, requestsInFlight: 0, lastRequestAt: 0 }) + }, + 'clients' + ], + [ + 'a request still running', + { + readClientActivity: () => ({ openConnections: 0, requestsInFlight: 1, lastRequestAt: 0 }) + }, + 'clients' + ], + ['an in-process terminal', { listTerminals: async () => [{ id: 'pty-1' }] }, 'terminals'], + ['a live daemon session', { countDaemonSessions: async () => 2 }, 'terminals'], + ['a working agent', { agentStates: () => [{ state: 'working' }] }, 'agents'], + ['a staged migration', { hasStagedMigration: () => true }, 'migration'], + ['a held activation fence', { activationFenceExists: async () => true }, 'activation'] + ])('reads %s as busy', async (_label, override, probe) => { + expect((await verdicts({ ...idlePorts(), ...override }))[probe]).toBe('busy') + }) + + it('treats a daemon that did not answer as unverifiable, not as no terminals', async () => { + const result = await verdicts({ ...idlePorts(), countDaemonSessions: async () => null }) + expect(result.terminals).toBe('unverifiable') + }) + + it('needs only the provider census when this orcad runs without a daemon', async () => { + const result = await verdicts({ + ...idlePorts(), + hasDaemon: () => false, + countDaemonSessions: async () => null + }) + expect(result.terminals).toBe('idle') + }) +}) + +describe('activationFenceExists', () => { + let root: string | null = null + afterEach(() => { + if (root) { + rmSync(root, { recursive: true, force: true }) + } + }) + + it('follows the fence directory a client holds during an update', async () => { + root = mkdtempSync(join(tmpdir(), 'orcad-fence-')) + const fence = join(root, '.orcad-activation-transaction') + expect(await activationFenceExists(fence)).toBe(false) + mkdirSync(fence) + expect(await activationFenceExists(fence)).toBe(true) + }) +}) diff --git a/src/main/orcad/orcad-managed-idle-exit.ts b/src/main/orcad/orcad-managed-idle-exit.ts new file mode 100644 index 00000000000..857e55e3f91 --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit.ts @@ -0,0 +1,116 @@ +/** + * What a managed orcad counts as "in use" before an idle stop, and the stop itself. + * + * The stop is the ordinary graceful shutdown, which disconnects from the terminal daemon and + * never shuts it down, so no terminal can be killed by it. Terminals are still a blocker: a + * host with live terminals keeps its server so a returning client finds it serving. + */ +import { stat } from 'node:fs/promises' +import type { RuntimeRpcClientActivity } from '../runtime/runtime-rpc/runtime-rpc-shutdown' +import { + ORCAD_MANAGED_ACTIVATION_ROOT_ENV, + ORCAD_IDLE_EXIT_TIMEOUT_MS, + readOrcadE2EIdleTimeoutMs +} from '../../shared/orcad-idle-exit' +import { + OrcadIdleExitMonitor, + type OrcadIdleExitEvidence, + type OrcadIdleProbe, + type OrcadIdleVerdict +} from './orcad-idle-exit-monitor' + +export type OrcadManagedIdleExitConfig = { timeoutMs: number; activationRoot: string } + +/** Null for any orcad a client did not launch: a user-started or paired server never idles out. */ +export function resolveOrcadManagedIdleExit( + env: NodeJS.ProcessEnv +): OrcadManagedIdleExitConfig | null { + const activationRoot = env[ORCAD_MANAGED_ACTIVATION_ROOT_ENV] + if (!activationRoot) { + return null + } + return { + timeoutMs: readOrcadE2EIdleTimeoutMs(env) ?? ORCAD_IDLE_EXIT_TIMEOUT_MS, + activationRoot + } +} + +export type OrcadManagedIdleExitPorts = { + readClientActivity: () => RuntimeRpcClientActivity + /** Every terminal the PTY provider knows, daemon-owned or in-process. */ + listTerminals: () => Promise + /** Live daemon sessions across generations; null when a daemon did not answer. */ + countDaemonSessions: () => Promise + hasDaemon: () => boolean + agentStates: () => readonly { state: string }[] + hasStagedMigration: () => boolean + /** Exists while a client holds the host's activation fence (update, rollback, decommission). */ + activationFenceExists: (root: string) => Promise +} + +export function createOrcadIdleProbes( + config: OrcadManagedIdleExitConfig, + ports: OrcadManagedIdleExitPorts +): OrcadIdleProbe[] { + const verdict = (busy: boolean): OrcadIdleVerdict => (busy ? 'busy' : 'idle') + return [ + { + name: 'clients', + read: () => { + const activity = ports.readClientActivity() + return verdict(activity.openConnections > 0 || activity.requestsInFlight > 0) + } + }, + { + name: 'terminals', + read: async () => { + if ((await ports.listTerminals()).length > 0) { + return 'busy' + } + // Why read the daemon too: it outlives this process and may hold sessions no provider lists. + if (!ports.hasDaemon()) { + return 'idle' + } + const live = await ports.countDaemonSessions() + return live === null ? 'unverifiable' : verdict(live > 0) + } + }, + { + name: 'agents', + read: () => verdict(ports.agentStates().some((entry) => entry.state === 'working')) + }, + { name: 'migration', read: () => verdict(ports.hasStagedMigration()) }, + { + name: 'activation', + read: async () => verdict(await ports.activationFenceExists(config.activationRoot)) + } + ] +} + +export async function activationFenceExists(root: string): Promise { + try { + await stat(root) + return true + } catch (error) { + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT') { + return false + } + throw error + } +} + +export function installOrcadManagedIdleExit(input: { + config: OrcadManagedIdleExitConfig + ports: OrcadManagedIdleExitPorts + /** Records the clean stop, then runs the same graceful shutdown as SIGTERM. */ + stop: (evidence: OrcadIdleExitEvidence) => void +}): () => void { + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: input.config.timeoutMs, + probes: createOrcadIdleProbes(input.config, input.ports), + lastClientActivityAt: () => input.ports.readClientActivity().lastRequestAt, + onIdle: input.stop + }) + monitor.start() + return () => monitor.stop() +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts index d6fdf15122d..cadb4a6121f 100644 --- a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts +++ b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts @@ -191,6 +191,13 @@ export class OrcadCatalogImportPersistence { context.runtime.terminalScrollbackSnapshotStorage ) } + + /** A migration into this server that is staged but neither committed nor aborted. */ + hasStagedOrcadMigrationCatalog(): boolean { + return ( + (this[orcadCatalogImportContext].runtime.state.orcadMigrationStagedCatalogs?.length ?? 0) > 0 + ) + } } function commitPreparedCatalog( diff --git a/src/main/runtime/runtime-rpc-client-activity.test.ts b/src/main/runtime/runtime-rpc-client-activity.test.ts new file mode 100644 index 00000000000..12b6c3a5206 --- /dev/null +++ b/src/main/runtime/runtime-rpc-client-activity.test.ts @@ -0,0 +1,34 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' +import { OrcaRuntimeRpcServer } from './runtime-rpc' +import { readRuntimeMetadata } from './runtime-metadata' +import { sendRequest } from './runtime-rpc-test-harness' + +describe('OrcaRuntimeRpcServer client activity', () => { + it('records each request so an idle host restarts its quiet period', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-activity-')) + const server = new OrcaRuntimeRpcServer({ runtime: new OrcaRuntimeService(), userDataPath }) + await server.start() + try { + const before = server.readClientActivity() + expect(before).toMatchObject({ openConnections: 0, requestsInFlight: 0 }) + + const metadata = readRuntimeMetadata(userDataPath) + await new Promise((resolve) => setTimeout(resolve, 5)) + await sendRequest(metadata!.transports[0]!.endpoint, { + id: 'req_status', + authToken: metadata!.authToken, + method: 'status.get' + }) + + const after = server.readClientActivity() + expect(after.requestsInFlight).toBe(0) + expect(after.lastRequestAt).toBeGreaterThan(before.lastRequestAt) + } finally { + await server.stop() + } + }) +}) diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts index 4c595b3bbab..3a1df4ce8ba 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts @@ -45,7 +45,7 @@ export class RuntimeRpcLifecycle extends RuntimeRpcWebSocketDispatch { // Why: the `.catch` guarantees reply() always fires so a throw can't strand the client or leak the AbortController. socketTransport.onMessage((msg, reply, context) => { - void this.handleMessage(msg, context) + void this.trackClientRequest(() => this.handleMessage(msg, context)) .then((response) => { reply(JSON.stringify(response)) }) @@ -222,17 +222,22 @@ export class RuntimeRpcLifecycle extends RuntimeRpcWebSocketDispatch { deviceRegistry, e2eeKeypair, onText: (socket, plaintext, reply, sendBinary) => { - void this.handleWebSocketMessage( - plaintext, - reply, - sendBinary, - undefined, - socket.ws, - socket.device.deviceToken, - socket + void this.trackClientRequest(() => + this.handleWebSocketMessage( + plaintext, + reply, + sendBinary, + undefined, + socket.ws, + socket.device.deviceToken, + socket + ) ) }, - onBinary: (socket, bytes) => this.handleWebSocketBinaryMessage(bytes, socket.ws), + onBinary: (socket, bytes) => { + this.lastClientRequestAt = Date.now() + this.handleWebSocketBinaryMessage(bytes, socket.ws) + }, onReady: () => { // Why: first authenticated mobile/remote client (direct WS and // cloud relay both attach here) starts path-candidate tracking. diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts b/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts index f79834fbcc2..c16bcda5510 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts @@ -1,11 +1,26 @@ import { RuntimeRpcMobilePairing } from './runtime-rpc-mobile-pairing' +export type RuntimeRpcClientActivity = { + openConnections: number + requestsInFlight: number + lastRequestAt: number +} + export class RuntimeRpcShutdown extends RuntimeRpcMobilePairing { /** Why: test-only seam — runs one ownership check instead of waiting out the poll interval. */ checkRuntimeMetadataOwnership(): Promise { return this.metadataOwnershipWatch?.check() ?? Promise.resolve() } + /** What a host's idle exit reads to know whether any client is still using this server. */ + readClientActivity(): RuntimeRpcClientActivity { + return { + openConnections: this.mobileSocketWiring?.connectionCount ?? 0, + requestsInFlight: this.clientRequestsInFlight, + lastRequestAt: this.lastClientRequestAt + } + } + async stop(): Promise { // Why: STA-2370 — refuse new widens, then let any in-flight pairing widen settle into the live // transport arrays before snapshotting them, so a racing rebind can't strand a wide 0.0.0.0 listener diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-state.ts b/src/main/runtime/runtime-rpc/runtime-rpc-state.ts index 20ddbf70e38..bd9a5ad2106 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-state.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-state.ts @@ -92,6 +92,8 @@ export class RuntimeRpcState { protected activeAskLongPolls = 0 protected activeBrowserHostLongPolls = 0 protected readonly activeBrowserHostLongPollsByDevice = new Map() + protected clientRequestsInFlight = 0 + protected lastClientRequestAt = Date.now() constructor({ runtime, @@ -137,4 +139,14 @@ export class RuntimeRpcState { this.pushUnregisterOutbox = new PushUnregisterOutbox(userDataPath) this.runtime.configureNotificationDismissalStore(userDataPath) } + + /** Counts a client message for idle exit, from receipt until its dispatch settles. */ + protected trackClientRequest(work: () => Promise): Promise { + this.clientRequestsInFlight += 1 + this.lastClientRequestAt = Date.now() + return work().finally(() => { + this.clientRequestsInFlight -= 1 + this.lastClientRequestAt = Date.now() + }) + } } diff --git a/src/main/ssh/orcad-activation-crash-recovery.test.ts b/src/main/ssh/orcad-activation-crash-recovery.test.ts index 8a61515f1a7..758291c3838 100644 --- a/src/main/ssh/orcad-activation-crash-recovery.test.ts +++ b/src/main/ssh/orcad-activation-crash-recovery.test.ts @@ -231,3 +231,20 @@ describe('recovery refusals keep the fence', () => { expect(host.fence).toBe(false) }) }) + +describe('every launch is a managed one', () => { + it.each(scenarios)( + '%s starts orcad with idle exit and its activation fence', + async (_n, scenario, run) => { + host = scenario() + await run() + const launches = host.commands.filter((command) => command.includes('nohup')) + expect(launches.length).toBeGreaterThan(0) + for (const launch of launches) { + expect(launch).toContain( + "ORCA_ORCAD_MANAGED_ACTIVATION_ROOT='/home/u/.orca-remote/.orcad-activation-transaction'" + ) + } + } + ) +}) diff --git a/src/main/ssh/orcad-installed-activation.ts b/src/main/ssh/orcad-installed-activation.ts index 97c6272b0f9..eb33ccb41f0 100644 --- a/src/main/ssh/orcad-installed-activation.ts +++ b/src/main/ssh/orcad-installed-activation.ts @@ -27,7 +27,10 @@ import { orcadStopFreedTheHost } from './orcad-remote-process-control' import { joinRemotePath } from './ssh-remote-platform' import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' import { preflightInstalledOrcad } from './orcad-remote-preflight' -import type { OrcadActivationLockControl } from './orcad-activation-lock' +import { + orcadActivationTransactionRoot, + type OrcadActivationLockControl +} from './orcad-activation-lock' import type { OrcadActivateTransaction } from './orcad-activation-transaction' import { createOrcadActivationTransaction, @@ -222,7 +225,8 @@ export async function activateInstalledOrcad( fullVersion, userDataDir: options.userDataDir, bindHost: options.bindHost, - port: options.port + port: options.port, + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) }) } catch (error) { if (isUnconfirmedSshCommandTermination(error)) { diff --git a/src/main/ssh/orcad-managed-serving-verify.ts b/src/main/ssh/orcad-managed-serving-verify.ts new file mode 100644 index 00000000000..65a4b3dedaf --- /dev/null +++ b/src/main/ssh/orcad-managed-serving-verify.ts @@ -0,0 +1,13 @@ +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import type { OrcadManagedServing } from './orcad-managed-serving' +import { verifyManagedTunnelServing } from './orcad-managed-tunnel' + +/** After the tunnel: the server answers, or was proven stopped and started; never throws. */ +export function verifyOrcadManagedServing( + userDataPath: string, + selector: string +): Promise { + return Promise.resolve() + .then(() => verifyManagedTunnelServing(resolveEnvironment(userDataPath, selector))) + .catch((error: unknown) => ({ state: 'unverifiable', detail: String(error) })) +} diff --git a/src/main/ssh/orcad-managed-serving.test.ts b/src/main/ssh/orcad-managed-serving.test.ts new file mode 100644 index 00000000000..171157a1575 --- /dev/null +++ b/src/main/ssh/orcad-managed-serving.test.ts @@ -0,0 +1,127 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./orcad-remote-context', () => ({ + resolveOrcadRemoteContext: vi.fn(async () => ({ + connection: {}, + host: {}, + remoteHome: '/home/u', + userDataDir: '/home/u/.orca' + })) +})) +vi.mock('./orcad-managed-wake', () => ({ wakeStoppedManagedOrcad: vi.fn() })) + +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { + ensureManagedOrcadServing, + resetManagedOrcadServingForTests, + setManagedOrcadStartListener, + type OrcadManagedServingInput +} from './orcad-managed-serving' + +const listener = { starting: vi.fn(), settled: vi.fn() } +let generation = 1 +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the transport generation is read; the remote context is mocked. +const connection = { getTransportGeneration: () => generation } as never + +function input(probe: () => Promise): OrcadManagedServingInput { + return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the id is read. + environment: { id: 'env-1' } as never, + target: { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' }, + connection, + remotePort: 6768, + probe: vi.fn(probe) + } +} + +beforeEach(() => { + vi.clearAllMocks() + resetManagedOrcadServingForTests() + setManagedOrcadStartListener(listener) + vi.spyOn(console, 'info').mockImplementation(() => {}) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) + +describe('ensureManagedOrcadServing', () => { + it('costs one round trip when the server answers', async () => { + expect(await ensureManagedOrcadServing(input(async () => true))).toEqual({ state: 'serving' }) + expect(wakeStoppedManagedOrcad).not.toHaveBeenCalled() + expect(listener.starting).not.toHaveBeenCalled() + }) + + it('starts a stopped server from its slot and shows the start on the status line', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockImplementation(async (_slot, onStarting) => { + onStarting?.() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only health is read. + return { outcome: 'started', readiness: { health: { previousIdleStop: null } } as never } + }) + + expect(await ensureManagedOrcadServing(input(async () => false))).toEqual({ + state: 'started', + boundPort: null + }) + expect(vi.mocked(wakeStoppedManagedOrcad).mock.calls[0]?.[0]).toMatchObject({ port: 6768 }) + expect(listener.starting).toHaveBeenCalledOnce() + expect(listener.settled).toHaveBeenCalledWith(expect.anything(), 'env-1', { + state: 'started', + boundPort: null + }) + }) + + it('stays unverifiable with the reason when the start fails, never a terminal verdict', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockImplementation(async (_slot, onStarting) => { + onStarting?.() + throw new Error('orcad did not become ready.\nLast lines of orcad.log:\nboom') + }) + + const serving = await ensureManagedOrcadServing(input(async () => false)) + expect(serving).toEqual({ + state: 'unverifiable', + detail: 'orcad did not become ready.\nLast lines of orcad.log:\nboom' + }) + expect(listener.settled).toHaveBeenCalledWith(expect.anything(), 'env-1', serving) + }) + + it('leaves a live process that did not answer alone', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'serving' }) + expect(await ensureManagedOrcadServing(input(async () => false))).toEqual({ state: 'serving' }) + expect(listener.starting).not.toHaveBeenCalled() + }) + + it.each(['fenced', 'unverifiable', 'not-activated'] as const)( + 'does not start a server whose host says %s', + async (outcome) => { + vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome }) + const serving = await ensureManagedOrcadServing(input(async () => false)) + expect(serving.state).toBe('unverifiable') + expect(listener.starting).not.toHaveBeenCalled() + } + ) + + it('shares one check between a fresh tunnel and the connect right after it', async () => { + let now = 0 + const probe = vi.fn(async () => true) + const first = input(probe) + await Promise.all([ + ensureManagedOrcadServing(first, () => now), + ensureManagedOrcadServing(first, () => now) + ]) + now = 4_000 + await ensureManagedOrcadServing(first, () => now) + expect(probe).toHaveBeenCalledOnce() + now = 6_000 + await ensureManagedOrcadServing(first, () => now) + expect(probe).toHaveBeenCalledTimes(2) + }) + + it('never answers a new SSH transport from an earlier verdict, as after a reboot', async () => { + const probe = vi.fn(async () => true) + await ensureManagedOrcadServing(input(probe), () => 0) + generation += 1 + await ensureManagedOrcadServing(input(probe), () => 1) + expect(probe).toHaveBeenCalledTimes(2) + // A rebind to the port a restarted server bound is a different server address. + await ensureManagedOrcadServing({ ...input(probe), remotePort: 40_001 }, () => 2) + expect(probe).toHaveBeenCalledTimes(3) + }) +}) diff --git a/src/main/ssh/orcad-managed-serving.ts b/src/main/ssh/orcad-managed-serving.ts new file mode 100644 index 00000000000..64e59132031 --- /dev/null +++ b/src/main/ssh/orcad-managed-serving.ts @@ -0,0 +1,157 @@ +/** + * Making sure a managed orcad is serving before a client relies on it: a server that answers + * costs one round trip; one that does not is checked on the host and, only if proven stopped + * (an idle stop, a kill, a host reboot), started from its activated slot. A daemon that survived + * is adopted by the new orcad with its terminals; after a reboot both start fresh. + * + * Never throws. A server that cannot be started stays `unverifiable` with the reason, and is + * never read as evidence that its terminals exited. + */ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import { orcadBoundPort } from './orcad-managed-bound-port' +import { managedOrcadSlot } from './orcad-managed-runtime-context' +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import type { SshConnection } from './ssh-connection' + +export type OrcadManagedServing = + | { state: 'serving' } + /** `boundPort` is the port the restarted server bound, which a forward must follow. */ + | { state: 'started'; boundPort: number | null } + | { state: 'unverifiable'; detail: string } + +export type OrcadManagedServingInput = { + environment: KnownRuntimeEnvironment + target: SshTarget + connection: SshConnection + remotePort: number + probe: (environment: KnownRuntimeEnvironment, timeoutMs: number) => Promise +} + +const PROBE_TIMEOUT_MS = 5_000 +// Why: a connect checks right after its fresh tunnel did; one verdict serves both, on that +// transport and port only, so a kill, reboot or rebind is never answered from cache. +const VERDICT_REUSE_MS = 5_000 + +const inFlight = new Map>() +type RecentVerdict = { + at: number + connection: SshConnection + generation: number + remotePort: number + serving: OrcadManagedServing +} +const recent = new Map() +export type ManagedOrcadStartListener = { + /** Shows "Starting managed server…" for the host. */ + starting: (target: SshTarget) => void + /** The start finished; an `unverifiable` result carries why, with orcad.log's tail. */ + settled: (target: SshTarget, environmentId: string, serving: OrcadManagedServing) => void +} + +let startListener: ManagedOrcadStartListener | null = null + +/** The SSH status wiring registers where a start is shown. */ +export function setManagedOrcadStartListener(listener: ManagedOrcadStartListener | null): void { + startListener = listener +} + +export function ensureManagedOrcadServing( + input: OrcadManagedServingInput, + now: () => number = Date.now +): Promise { + const id = input.environment.id + const generation = input.connection.getTransportGeneration() + const cached = recent.get(id) + if ( + cached && + cached.connection === input.connection && + cached.generation === generation && + cached.remotePort === input.remotePort && + now() - cached.at < VERDICT_REUSE_MS + ) { + return Promise.resolve(cached.serving) + } + const pending = inFlight.get(id) + if (pending) { + return pending + } + const operation = checkAndStart(input).then((serving) => { + recent.set(id, { + at: now(), + connection: input.connection, + generation, + remotePort: input.remotePort, + serving + }) + return serving + }) + const settled = operation.finally(() => inFlight.delete(id)) + inFlight.set(id, settled) + return settled +} + +/** Test-only: forget cached verdicts. */ +export function resetManagedOrcadServingForTests(): void { + inFlight.clear() + recent.clear() +} + +async function checkAndStart(input: OrcadManagedServingInput): Promise { + if (await input.probe(input.environment, PROBE_TIMEOUT_MS)) { + return { state: 'serving' } + } + let starting = false + const serving = await wakeIfStopped(input, () => { + starting = true + startListener?.starting(input.target) + }) + if (starting) { + startListener?.settled(input.target, input.environment.id, serving) + } + return serving +} + +async function wakeIfStopped( + input: OrcadManagedServingInput, + onStarting: () => void +): Promise { + const label = input.target.label + try { + const context = await resolveOrcadRemoteContext(input.target, input.connection) + const wake = await wakeStoppedManagedOrcad( + managedOrcadSlot(context, input.remotePort), + onStarting + ) + if (wake.outcome === 'started') { + const idle = wake.readiness.health?.previousIdleStop + const cause = idle + ? `it had stopped after idling at ${idle.stoppedAt}` + : 'it had stopped without an idle-stop record (crash, signal or host restart)' + console.info(`[ssh] Started the managed Orca server on ${label}; ${cause}.`) + return { state: 'started', boundPort: orcadBoundPort(wake.readiness) } + } + // A live process that did not answer may still be starting; it is not restarted. + if (wake.outcome === 'serving') { + return { state: 'serving' } + } + const detail = wakeRefusal(wake.outcome) + console.warn(`[ssh] The managed Orca server on ${label} is not answering: ${detail}`) + return { state: 'unverifiable', detail } + } catch (error) { + console.warn(`[ssh] Could not start the managed Orca server on ${label}:`, error) + return { state: 'unverifiable', detail: error instanceof Error ? error.message : String(error) } + } +} + +function wakeRefusal(outcome: 'not-activated' | 'unverifiable' | 'fenced'): string { + switch (outcome) { + case 'fenced': + return 'An update, rollback or recovery holds this host; it was not started.' + case 'not-activated': + return 'This host has no activated managed server to start.' + case 'unverifiable': + return 'Whether the server process is still running could not be proven, so it was not started.' + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-manager.ts b/src/main/ssh/orcad-managed-tunnel-manager.ts new file mode 100644 index 00000000000..ce549278cde --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-manager.ts @@ -0,0 +1,300 @@ +/** + * One SSH forward per managed environment, owned across reconnects and host resume, and the + * serving check that starts a stopped server behind it. + */ +import { + getRuntimeSshAccess, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { SshConnection } from './ssh-connection' +import type { SshConnectionManager } from './ssh-connection-manager' +import { SshPortForwardManager } from './ssh-port-forward' +import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport' +import { + OrcadManagedTunnelResumeRecovery, + type ActiveOrcadTunnel, + type OrcadManagedServingCheck, + type OrcadManagedTunnelProbe, + type OrcadManagedTunnelResumeOptions +} from './orcad-managed-tunnel-resume' +import type { OrcadManagedServing } from './orcad-managed-serving' +import { checkManagedTunnelServing, type OrcadTunnelServing } from './orcad-managed-tunnel-serving' +import type { getSshTargetRegistryStore } from './ssh-target-registry' +import { + environmentForwardChecks, + forwardToVerifiedOrcad, + PERSISTED_PORT_TARGETING, + type OrcadManagedTunnelTargeting, + type OrcadTunnelStartChecks +} from './orcad-managed-tunnel-target' + +export type OrcadManagedTunnelDependencies = { + getConnectionManager: () => SshConnectionManager | null + getTargetStore: () => ReturnType + forwardManager?: SshPortForwardManager + probeTunnel?: OrcadManagedTunnelProbe + targeting?: OrcadManagedTunnelTargeting + /** Runs after a fresh forward is up; starts a server that stopped (e.g. after idling). */ + ensureServing?: OrcadManagedServingCheck +} + +export class OrcadManagedTunnelManager { + private readonly active = new Map() + private readonly inFlight = new Map>() + private readonly ownershipGenerations = new Map() + private readonly forwards: SshPortForwardManager + private readonly resumeRecovery: OrcadManagedTunnelResumeRecovery + private readonly targeting: OrcadManagedTunnelTargeting + private managerGeneration = 0 + + constructor(private readonly dependencies: OrcadManagedTunnelDependencies) { + this.forwards = + dependencies.forwardManager ?? + new SshPortForwardManager({}, [new OrcadManagedTunnelTransportProvider()]) + this.targeting = dependencies.targeting ?? PERSISTED_PORT_TARGETING + this.resumeRecovery = new OrcadManagedTunnelResumeRecovery({ + active: this.active, + forwards: this.forwards, + getConnectionManager: dependencies.getConnectionManager, + getManagerGeneration: () => this.managerGeneration, + getTargetStore: dependencies.getTargetStore, + inFlight: this.inFlight, + ownershipGenerations: this.ownershipGenerations, + probeTunnel: dependencies.probeTunnel, + targeting: this.targeting, + checkServing: (environment) => this.checkServing(environment) + }) + this.forwards.setCallbacks({ + onForwardClosed: (entry) => { + for (const [environmentId, active] of this.active) { + if (active.forwardId === entry.id) { + this.active.delete(environmentId) + } + } + } + }) + } + + ensure( + environment: KnownRuntimeEnvironment, + resolveCurrent: () => KnownRuntimeEnvironment | null = () => environment + ): Promise { + if (!getRuntimeSshAccess(environment)) { + return Promise.resolve() + } + const pending = this.inFlight.get(environment.id) + if (pending) { + return pending + } + const operation = this.ensureManagedTunnel(environment, resolveCurrent).finally(() => { + if (this.inFlight.get(environment.id) === operation) { + this.inFlight.delete(environment.id) + } + }) + this.inFlight.set(environment.id, operation) + return operation + } + + async start( + environmentId: string, + target: SshTarget, + connection: SshConnection, + remotePort: number, + checks: OrcadTunnelStartChecks = {} + ): Promise { + if (!target.generation) { + throw new Error('Managed Orca SSH target has no registration generation.') + } + const managerGeneration = this.managerGeneration + const ownershipGeneration = (this.ownershipGenerations.get(environmentId) ?? 0) + 1 + const transportGeneration = connection.getTransportGeneration() + const stillCurrent = (): boolean => + this.managerGeneration === managerGeneration && + this.ownershipGenerations.get(environmentId) === ownershipGeneration && + connection.getTransportGeneration() === transportGeneration + await this.close(environmentId) + if (!stillCurrent()) { + throw new Error('Orca SSH tunnel setup was superseded.') + } + const forward = await forwardToVerifiedOrcad({ + targetId: target.id, + connection, + forwards: this.forwards, + localPort: 0, + label: 'Managed Orca server', + remotePort, + rereadRemotePort: checks.rereadRemotePort, + verify: checks.verify, + stillCurrent + }) + if (!forward) { + throw new Error('Orca SSH tunnel setup was superseded.') + } + this.active.set(environmentId, { + connection, + forwardId: forward.id, + localPort: forward.localPort, + remotePort: forward.remotePort, + preferredPort: checks.preferredPort ?? remotePort, + sshTargetGeneration: target.generation, + targetId: target.id, + transportGeneration + }) + return forward.localPort + } + + async close(environmentId: string): Promise { + this.ownershipGenerations.set( + environmentId, + (this.ownershipGenerations.get(environmentId) ?? 0) + 1 + ) + const active = this.active.get(environmentId) + if (!active) { + return + } + this.active.delete(environmentId) + await this.forwards.removeForwardAndWait(active.forwardId) + } + + dispose(): void { + this.managerGeneration += 1 + this.active.clear() + this.inFlight.clear() + this.ownershipGenerations.clear() + this.resumeRecovery.dispose() + this.forwards.dispose() + } + + /** The server behind the tunnel answers, or is started; a moved port rebuilds the forward. */ + async verifyServing(environment: KnownRuntimeEnvironment): Promise { + if (!this.active.has(environment.id)) { + await this.ensure(environment) + } + const serving = await this.checkServing(environment) + if (serving.rebind) { + await this.ensure(environment) + } + return serving + } + + /** Never rebuilds, so it is safe inside a build: a moved port only drops the forward. */ + checkServing(environment: KnownRuntimeEnvironment): Promise { + return checkManagedTunnelServing({ + environment, + active: this.active, + getTarget: (id) => this.dependencies.getTargetStore()?.getTarget(id), + forwards: this.forwards, + ensureServing: this.dependencies.ensureServing + }) + } + + recoverAfterHostResume(options: OrcadManagedTunnelResumeOptions): Promise { + return this.resumeRecovery.recover(options) + } + + private async ensureManagedTunnel( + environment: KnownRuntimeEnvironment, + resolveCurrent: () => KnownRuntimeEnvironment | null, + rebound = false + ): Promise { + const deployment = getRuntimeSshAccess(environment) + if (!deployment || environment.connectionDependency !== 'ssh-tunnel') { + throw new Error('Managed orcad environment is missing its SSH tunnel dependency.') + } + const targetStore = this.dependencies.getTargetStore() + const connectionManager = this.dependencies.getConnectionManager() + if (!targetStore || !connectionManager) { + throw new Error('SSH is unavailable on this client; the managed Orca server is unverifiable.') + } + const target = targetStore.getTarget(deployment.sshTargetId) + if (!target || target.generation !== deployment.sshTargetGeneration) { + throw new Error( + 'The SSH registration for this managed Orca server was removed or re-created.' + ) + } + if (getManagedOrcadFenceEnvironmentId(target) !== environment.id) { + throw new Error('The SSH target is no longer owned by this managed Orca server.') + } + + const managerGeneration = this.managerGeneration + const ownershipGeneration = this.ownershipGenerations.get(environment.id) ?? 0 + const stillOwned = (): boolean => { + const currentTarget = targetStore.getTarget(target.id) + const currentEnvironment = resolveCurrent() + const currentAccess = currentEnvironment ? getRuntimeSshAccess(currentEnvironment) : undefined + return ( + this.managerGeneration === managerGeneration && + (this.ownershipGenerations.get(environment.id) ?? 0) === ownershipGeneration && + currentTarget?.generation === target.generation && + getManagedOrcadFenceEnvironmentId(currentTarget) === environment.id && + currentEnvironment?.id === environment.id && + currentEnvironment.runtimeId === environment.runtimeId && + (currentEnvironment.pairingRevision ?? currentEnvironment.createdAt) === + (environment.pairingRevision ?? environment.createdAt) && + currentEnvironment.connectionDependency === 'ssh-tunnel' && + currentAccess?.sshTargetId === deployment.sshTargetId && + currentAccess.sshTargetGeneration === deployment.sshTargetGeneration && + currentAccess.localPort === deployment.localPort && + currentAccess.remotePort === deployment.remotePort + ) + } + const connection = await connectionManager.connect(target) + if (!stillOwned()) { + return + } + const transportGeneration = connection.getTransportGeneration() + const active = this.active.get(environment.id) + if ( + active?.connection === connection && + active.transportGeneration === transportGeneration && + active.targetId === target.id && + active.sshTargetGeneration === target.generation && + active.localPort === deployment.localPort && + active.preferredPort === deployment.remotePort + ) { + return + } + const checks = await environmentForwardChecks(this.targeting, { + environment, + target, + connection + }) + if (active && stillOwned()) { + await this.forwards.removeForwardAndWait(active.forwardId) + if (this.active.get(environment.id) === active) { + this.active.delete(environment.id) + } + } + if (!stillOwned()) { + return + } + const forward = await forwardToVerifiedOrcad({ + targetId: target.id, + connection, + forwards: this.forwards, + localPort: deployment.localPort, + label: `Managed Orca server: ${environment.name}`, + ...checks, + stillCurrent: () => + stillOwned() && connection.getTransportGeneration() === transportGeneration + }) + if (!forward) { + return + } + this.active.set(environment.id, { + connection, + forwardId: forward.id, + localPort: forward.localPort, + remotePort: forward.remotePort, + preferredPort: deployment.remotePort, + sshTargetGeneration: target.generation, + targetId: target.id, + transportGeneration + }) + if ((await this.checkServing(environment)).rebind && !rebound) { + await this.ensureManagedTunnel(environment, resolveCurrent, true) + } + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-resume.ts b/src/main/ssh/orcad-managed-tunnel-resume.ts index b42e0830ac7..203d722b835 100644 --- a/src/main/ssh/orcad-managed-tunnel-resume.ts +++ b/src/main/ssh/orcad-managed-tunnel-resume.ts @@ -7,6 +7,8 @@ import { type RuntimeSshTunnelLink } from '../../shared/runtime-environments' import type { SshConnection } from './ssh-connection' +import type { SshTarget } from '../../shared/ssh-types' +import type { OrcadManagedServing } from './orcad-managed-serving' import type { SshConnectionManager } from './ssh-connection-manager' import type { SshPortForwardManager } from './ssh-port-forward' import type { getSshTargetRegistryStore } from './ssh-target-registry' @@ -33,6 +35,13 @@ export type OrcadManagedTunnelProbe = ( timeoutMs: number ) => Promise +export type OrcadManagedServingCheck = (input: { + environment: KnownRuntimeEnvironment + target: SshTarget + connection: SshConnection + remotePort: number +}) => Promise + export type OrcadManagedTunnelResumeOptions = { attempts: number resolveEnvironment: (environmentId: string) => KnownRuntimeEnvironment | null @@ -49,6 +58,8 @@ type ResumeRecoveryDependencies = { ownershipGenerations: Map probeTunnel?: OrcadManagedTunnelProbe targeting: OrcadManagedTunnelTargeting + /** Never rebuilds: a server restarted on a new port drops this forward for the next ensure. */ + checkServing?: (environment: KnownRuntimeEnvironment) => Promise } type ResolvedManagedTunnelEnvironment = { @@ -222,6 +233,8 @@ export class OrcadManagedTunnelResumeRecovery { targetId: active.targetId, transportGeneration }) + // A server that idled out while this client slept is started here, not reported as lost. + await this.dependencies.checkServing?.(current.environment) } private resolveEnvironment( @@ -265,7 +278,7 @@ export class OrcadManagedTunnelResumeRecovery { } } -async function probeManagedOrcadTunnel( +export async function probeManagedOrcadTunnel( environment: KnownRuntimeEnvironment, timeoutMs: number ): Promise { diff --git a/src/main/ssh/orcad-managed-tunnel-serving.ts b/src/main/ssh/orcad-managed-tunnel-serving.ts new file mode 100644 index 00000000000..bd545c75360 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-serving.ts @@ -0,0 +1,45 @@ +/** + * The serving check behind an established managed tunnel. A restarted orcad may bind a port + * other than the one the tunnel forwards to; the forward is then dropped, and the next build + * forwards to the port the new server actually bound. + */ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { OrcadManagedServing } from './orcad-managed-serving' +import type { ActiveOrcadTunnel, OrcadManagedServingCheck } from './orcad-managed-tunnel-resume' +import type { SshPortForwardManager } from './ssh-port-forward' +import type { SshTarget } from '../../shared/ssh-types' + +export type OrcadTunnelServing = OrcadManagedServing & { rebind?: true } + +export async function checkManagedTunnelServing(input: { + environment: KnownRuntimeEnvironment + active: Map + getTarget: (targetId: string) => SshTarget | null | undefined + forwards: Pick + ensureServing: OrcadManagedServingCheck | undefined +}): Promise { + const { environment, active: tunnels } = input + const active = tunnels.get(environment.id) + const target = active && input.getTarget(active.targetId) + if (!active || !target || !input.ensureServing) { + return { state: 'unverifiable', detail: 'The managed server tunnel is not up.' } + } + const serving = await input.ensureServing({ + environment, + target, + connection: active.connection, + remotePort: active.remotePort + }) + if ( + serving.state !== 'started' || + serving.boundPort === null || + serving.boundPort === active.remotePort + ) { + return serving + } + if (tunnels.get(environment.id) === active) { + tunnels.delete(environment.id) + } + await input.forwards.removeForwardAndWait(active.forwardId) + return { ...serving, rebind: true } +} diff --git a/src/main/ssh/orcad-managed-tunnel.test.ts b/src/main/ssh/orcad-managed-tunnel.test.ts index 338516074d3..0a285126c7a 100644 --- a/src/main/ssh/orcad-managed-tunnel.test.ts +++ b/src/main/ssh/orcad-managed-tunnel.test.ts @@ -84,6 +84,7 @@ function setup(overrides: Partial = {}, targeting?: OrcadManagedTunne }) ) const removeForwardAndWait = vi.fn().mockResolvedValue(null) + const ensureServing = vi.fn().mockResolvedValue({ state: 'serving' }) // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the members the tunnel manager calls. const forwardManager = { setCallbacks: vi.fn(), @@ -104,12 +105,14 @@ function setup(overrides: Partial = {}, targeting?: OrcadManagedTunne getTargetStore: () => ({ getTarget: vi.fn(() => target) }) as unknown as SshConnectionStore, forwardManager, probeTunnel, - targeting + targeting, + ensureServing }) return { addForward, connect, connection, + ensureServing, getConnection, getState, manager, @@ -356,6 +359,38 @@ describe.each(['orcadDeployment', 'sshAccess'] as const)( expect(state.probeTunnel).not.toHaveBeenCalled() }) + it('checks that the server is running only when it sets up a fresh forward', async () => { + const state = setup() + + await state.manager.ensure(environment()) + await state.manager.ensure(environment()) + expect(state.ensureServing).toHaveBeenCalledOnce() + expect(state.ensureServing).toHaveBeenCalledWith({ + environment: expect.objectContaining({ id: 'environment-1' }), + target: state.target, + connection: state.connection, + remotePort: 6_768 + }) + + state.setTransportGeneration(4) + await state.manager.ensure(environment()) + expect(state.ensureServing).toHaveBeenCalledTimes(2) + }) + + it('starts a server that stopped while the client slept once the tunnel is rebuilt', async () => { + const state = setup() + await state.manager.ensure(environment()) + state.probeTunnel.mockResolvedValue(false) + + await state.manager.recoverAfterHostResume(resumeOptions()) + + expect(state.reconnect).toHaveBeenCalledOnce() + expect(state.ensureServing).toHaveBeenCalledTimes(2) + expect(state.ensureServing).toHaveBeenLastCalledWith( + expect.objectContaining({ target: state.target, remotePort: 6_768 }) + ) + }) + it('keeps a healthy managed tunnel intact after host resume', async () => { const state = setup() await state.manager.ensure(environment()) @@ -534,6 +569,50 @@ describe('OrcadManagedTunnelManager bound port', () => { expect(state.verifyIdentity).toHaveBeenCalledOnce() }) + it('follows a restarted server to the port it bound, within the same ensure', async () => { + const state = boundPortSetup([6_768, 58_520]) + state.ensureServing + .mockResolvedValueOnce({ state: 'started', boundPort: 58_520 }) + .mockResolvedValue({ state: 'serving' }) + await state.manager.ensure(createEnvironment('orcadDeployment')) + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + expect(state.addForward).toHaveBeenLastCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 58_520, + 'Managed Orca server: Managed server' + ) + expect(state.ensureServing).toHaveBeenLastCalledWith( + expect.objectContaining({ remotePort: 58_520 }) + ) + }) + + it('rebuilds after an explicit check finds the server restarted on another port', async () => { + const state = boundPortSetup([6_768, 58_520]) + const environment = createEnvironment('orcadDeployment') + await state.manager.ensure(environment) + state.ensureServing + .mockResolvedValueOnce({ state: 'started', boundPort: 58_520 }) + .mockResolvedValue({ state: 'serving' }) + + await expect(state.manager.verifyServing(environment)).resolves.toMatchObject({ + state: 'started', + rebind: true + }) + expect(state.addForward).toHaveBeenCalledTimes(2) + expect(state.addForward).toHaveBeenLastCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 58_520, + 'Managed Orca server: Managed server' + ) + }) + it('reuses a verified tunnel without reading the port again', async () => { const state = boundPortSetup([58_520]) const environment = createEnvironment('orcadDeployment') diff --git a/src/main/ssh/orcad-managed-tunnel.ts b/src/main/ssh/orcad-managed-tunnel.ts index 660651e83f9..f0b6509e38c 100644 --- a/src/main/ssh/orcad-managed-tunnel.ts +++ b/src/main/ssh/orcad-managed-tunnel.ts @@ -1,273 +1,23 @@ -import { - getRuntimeSshAccess, - type KnownRuntimeEnvironment -} from '../../shared/runtime-environments' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' import { resolveEnvironment } from '../../shared/runtime-environment-store' import type { SshTarget } from '../../shared/ssh-types' -import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' import type { SshConnection } from './ssh-connection' -import type { SshConnectionManager } from './ssh-connection-manager' -import { SshPortForwardManager } from './ssh-port-forward' -import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport' -import { - OrcadManagedTunnelResumeRecovery, - type ActiveOrcadTunnel, - type OrcadManagedTunnelProbe, - type OrcadManagedTunnelResumeOptions -} from './orcad-managed-tunnel-resume' +import { probeManagedOrcadTunnel } from './orcad-managed-tunnel-resume' +import { ensureManagedOrcadServing } from './orcad-managed-serving' +import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry' import { - environmentForwardChecks, - forwardToVerifiedOrcad, MANAGED_ORCAD_TUNNEL_TARGETING, - PERSISTED_PORT_TARGETING, - type OrcadManagedTunnelTargeting, type OrcadTunnelStartChecks } from './orcad-managed-tunnel-target' -type OrcadManagedTunnelDependencies = { - getConnectionManager: () => SshConnectionManager | null - getTargetStore: () => ReturnType - forwardManager?: SshPortForwardManager - probeTunnel?: OrcadManagedTunnelProbe - targeting?: OrcadManagedTunnelTargeting -} - -export class OrcadManagedTunnelManager { - private readonly active = new Map() - private readonly inFlight = new Map>() - private readonly ownershipGenerations = new Map() - private readonly forwards: SshPortForwardManager - private readonly resumeRecovery: OrcadManagedTunnelResumeRecovery - private readonly targeting: OrcadManagedTunnelTargeting - private managerGeneration = 0 - - constructor(private readonly dependencies: OrcadManagedTunnelDependencies) { - this.forwards = - dependencies.forwardManager ?? - new SshPortForwardManager({}, [new OrcadManagedTunnelTransportProvider()]) - this.targeting = dependencies.targeting ?? PERSISTED_PORT_TARGETING - this.resumeRecovery = new OrcadManagedTunnelResumeRecovery({ - active: this.active, - forwards: this.forwards, - getConnectionManager: dependencies.getConnectionManager, - getManagerGeneration: () => this.managerGeneration, - getTargetStore: dependencies.getTargetStore, - inFlight: this.inFlight, - ownershipGenerations: this.ownershipGenerations, - probeTunnel: dependencies.probeTunnel, - targeting: this.targeting - }) - this.forwards.setCallbacks({ - onForwardClosed: (entry) => { - for (const [environmentId, active] of this.active) { - if (active.forwardId === entry.id) { - this.active.delete(environmentId) - } - } - } - }) - } - - ensure( - environment: KnownRuntimeEnvironment, - resolveCurrent: () => KnownRuntimeEnvironment | null = () => environment - ): Promise { - if (!getRuntimeSshAccess(environment)) { - return Promise.resolve() - } - const pending = this.inFlight.get(environment.id) - if (pending) { - return pending - } - const operation = this.ensureManagedTunnel(environment, resolveCurrent).finally(() => { - if (this.inFlight.get(environment.id) === operation) { - this.inFlight.delete(environment.id) - } - }) - this.inFlight.set(environment.id, operation) - return operation - } - - async start( - environmentId: string, - target: SshTarget, - connection: SshConnection, - remotePort: number, - checks: OrcadTunnelStartChecks = {} - ): Promise { - if (!target.generation) { - throw new Error('Managed Orca SSH target has no registration generation.') - } - const managerGeneration = this.managerGeneration - const ownershipGeneration = (this.ownershipGenerations.get(environmentId) ?? 0) + 1 - const transportGeneration = connection.getTransportGeneration() - const stillCurrent = (): boolean => - this.managerGeneration === managerGeneration && - this.ownershipGenerations.get(environmentId) === ownershipGeneration && - connection.getTransportGeneration() === transportGeneration - await this.close(environmentId) - if (!stillCurrent()) { - throw new Error('Orca SSH tunnel setup was superseded.') - } - const forward = await forwardToVerifiedOrcad({ - targetId: target.id, - connection, - forwards: this.forwards, - localPort: 0, - label: 'Managed Orca server', - remotePort, - rereadRemotePort: checks.rereadRemotePort, - verify: checks.verify, - stillCurrent - }) - if (!forward) { - throw new Error('Orca SSH tunnel setup was superseded.') - } - this.active.set(environmentId, { - connection, - forwardId: forward.id, - localPort: forward.localPort, - remotePort: forward.remotePort, - preferredPort: checks.preferredPort ?? remotePort, - sshTargetGeneration: target.generation, - targetId: target.id, - transportGeneration - }) - return forward.localPort - } - - async close(environmentId: string): Promise { - this.ownershipGenerations.set( - environmentId, - (this.ownershipGenerations.get(environmentId) ?? 0) + 1 - ) - const active = this.active.get(environmentId) - if (!active) { - return - } - this.active.delete(environmentId) - await this.forwards.removeForwardAndWait(active.forwardId) - } - - dispose(): void { - this.managerGeneration += 1 - this.active.clear() - this.inFlight.clear() - this.ownershipGenerations.clear() - this.resumeRecovery.dispose() - this.forwards.dispose() - } - - recoverAfterHostResume(options: OrcadManagedTunnelResumeOptions): Promise { - return this.resumeRecovery.recover(options) - } - - private async ensureManagedTunnel( - environment: KnownRuntimeEnvironment, - resolveCurrent: () => KnownRuntimeEnvironment | null - ): Promise { - const deployment = getRuntimeSshAccess(environment) - if (!deployment || environment.connectionDependency !== 'ssh-tunnel') { - throw new Error('Managed orcad environment is missing its SSH tunnel dependency.') - } - const targetStore = this.dependencies.getTargetStore() - const connectionManager = this.dependencies.getConnectionManager() - if (!targetStore || !connectionManager) { - throw new Error('SSH is unavailable on this client; the managed Orca server is unverifiable.') - } - const target = targetStore.getTarget(deployment.sshTargetId) - if (!target || target.generation !== deployment.sshTargetGeneration) { - throw new Error( - 'The SSH registration for this managed Orca server was removed or re-created.' - ) - } - if (getManagedOrcadFenceEnvironmentId(target) !== environment.id) { - throw new Error('The SSH target is no longer owned by this managed Orca server.') - } - - const managerGeneration = this.managerGeneration - const ownershipGeneration = this.ownershipGenerations.get(environment.id) ?? 0 - const stillOwned = (): boolean => { - const currentTarget = targetStore.getTarget(target.id) - const currentEnvironment = resolveCurrent() - const currentAccess = currentEnvironment ? getRuntimeSshAccess(currentEnvironment) : undefined - return ( - this.managerGeneration === managerGeneration && - (this.ownershipGenerations.get(environment.id) ?? 0) === ownershipGeneration && - currentTarget?.generation === target.generation && - getManagedOrcadFenceEnvironmentId(currentTarget) === environment.id && - currentEnvironment?.id === environment.id && - currentEnvironment.runtimeId === environment.runtimeId && - (currentEnvironment.pairingRevision ?? currentEnvironment.createdAt) === - (environment.pairingRevision ?? environment.createdAt) && - currentEnvironment.connectionDependency === 'ssh-tunnel' && - currentAccess?.sshTargetId === deployment.sshTargetId && - currentAccess.sshTargetGeneration === deployment.sshTargetGeneration && - currentAccess.localPort === deployment.localPort && - currentAccess.remotePort === deployment.remotePort - ) - } - const connection = await connectionManager.connect(target) - if (!stillOwned()) { - return - } - const transportGeneration = connection.getTransportGeneration() - const active = this.active.get(environment.id) - if ( - active?.connection === connection && - active.transportGeneration === transportGeneration && - active.targetId === target.id && - active.sshTargetGeneration === target.generation && - active.localPort === deployment.localPort && - active.preferredPort === deployment.remotePort - ) { - return - } - const checks = await environmentForwardChecks(this.targeting, { - environment, - target, - connection - }) - if (active && stillOwned()) { - await this.forwards.removeForwardAndWait(active.forwardId) - if (this.active.get(environment.id) === active) { - this.active.delete(environment.id) - } - } - if (!stillOwned()) { - return - } - const forward = await forwardToVerifiedOrcad({ - targetId: target.id, - connection, - forwards: this.forwards, - localPort: deployment.localPort, - label: `Managed Orca server: ${environment.name}`, - ...checks, - stillCurrent: () => - stillOwned() && connection.getTransportGeneration() === transportGeneration - }) - if (!forward) { - return - } - this.active.set(environment.id, { - connection, - forwardId: forward.id, - localPort: forward.localPort, - remotePort: forward.remotePort, - preferredPort: deployment.remotePort, - sshTargetGeneration: target.generation, - targetId: target.id, - transportGeneration - }) - } -} +export { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' const managedTunnels = new OrcadManagedTunnelManager({ getConnectionManager: getSshConnectionManager, getTargetStore: getSshTargetRegistryStore, - targeting: MANAGED_ORCAD_TUNNEL_TARGETING + targeting: MANAGED_ORCAD_TUNNEL_TARGETING, + ensureServing: (input) => ensureManagedOrcadServing({ ...input, probe: probeManagedOrcadTunnel }) }) export async function ensureOrcadManagedTunnel( @@ -288,6 +38,10 @@ function resolveEnvironmentOrNull(userDataPath: string, id: string) { } } +export function verifyManagedTunnelServing(environment: KnownRuntimeEnvironment) { + return managedTunnels.verifyServing(environment) +} + export function disposeOrcadManagedTunnels(): void { managedTunnels.dispose() } diff --git a/src/main/ssh/orcad-managed-wake.test.ts b/src/main/ssh/orcad-managed-wake.test.ts new file mode 100644 index 00000000000..4b99c79e1ca --- /dev/null +++ b/src/main/ssh/orcad-managed-wake.test.ts @@ -0,0 +1,118 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' +import type * as InstallLock from './ssh-relay-install-lock' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn() +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' +import { FakeOrcadHost, OLD } from './orcad-activation-host-test-harness' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV +} from '../../shared/orcad-idle-exit' + +let host = new FakeOrcadHost() + +const slot = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked, so the connection is never used. + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + readinessTimeoutMs: 50, + sleep: async () => {} +} + +function launches(): string[] { + return host.commands.filter((command) => command.includes('nohup')) +} + +/** The slot's process exited on its own, as an idle stop leaves it. */ +function stoppedHost(): FakeOrcadHost { + const stopped = FakeOrcadHost.deployedOld() + stopped.alive.clear() + return stopped +} + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + vi.mocked(acquireInstallLock).mockImplementation(async () => host.acquireFence()) +}) + +afterEach(() => { + vi.unstubAllEnvs() +}) + +describe('wakeStoppedManagedOrcad', () => { + it('starts a stopped active slot as a managed launch and releases the fence', async () => { + host = stoppedHost() + + const wake = await wakeStoppedManagedOrcad(slot) + + expect(wake.outcome).toBe('started') + expect([...host.alive]).toEqual([OLD]) + expect(host.fence).toBe(false) + expect(launches()).toHaveLength(1) + expect(launches()[0]).toContain( + `${ORCAD_MANAGED_ACTIVATION_ROOT_ENV}='/home/u/.orca-remote/.orcad-activation-transaction'` + ) + expect(launches()[0]).not.toContain(ORCAD_E2E_IDLE_TIMEOUT_ENV) + }) + + it('forwards the test-only idle timeout to the server it starts', async () => { + vi.stubEnv(ORCAD_E2E_IDLE_TIMEOUT_ENV, '3000') + host = stoppedHost() + + await wakeStoppedManagedOrcad(slot) + + expect(launches()[0]).toContain(`${ORCAD_E2E_IDLE_TIMEOUT_ENV}='3000'`) + }) + + it('leaves a running server alone', async () => { + host = FakeOrcadHost.deployedOld() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' }) + expect(launches()).toEqual([]) + expect(acquireInstallLock).not.toHaveBeenCalled() + }) + + it('never starts a second process when the slot state is unprovable', async () => { + host = stoppedHost() + host.pidFiles.clear() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'unverifiable' }) + expect(launches()).toEqual([]) + }) + + it('defers to an update or recovery that holds the activation fence', async () => { + host = stoppedHost() + host.fence = true + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'fenced' }) + expect(launches()).toEqual([]) + expect(host.fence).toBe(true) + }) + + it('has nothing to start on a host with no activated server', async () => { + host = new FakeOrcadHost() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'not-activated' }) + expect(launches()).toEqual([]) + }) +}) diff --git a/src/main/ssh/orcad-managed-wake.ts b/src/main/ssh/orcad-managed-wake.ts new file mode 100644 index 00000000000..aa89056b4c9 --- /dev/null +++ b/src/main/ssh/orcad-managed-wake.ts @@ -0,0 +1,62 @@ +/** + * Starting a managed orcad that is installed and activated but not running, most often one + * that stopped itself after idling. A stopped server is just "not running": it is neither a + * failure nor evidence about terminals, which the daemon owns and which outlive orcad. + */ +import type { ServeReadiness } from '../server/serve-readiness' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { orcadActivationFenceExists, withOrcadActivationLock } from './orcad-activation-lock' +import { orcadLivenessProbeCommand, parseOrcadLiveness } from './orcad-remote-launch' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import { + ensureOrcadSlotServing, + orcadSlotDir, + resolveOrcadSlotIdentity, + type OrcadSlotOptions +} from './orcad-recovery-slot' + +export type OrcadManagedWake = + | { outcome: 'serving' | 'not-activated' | 'unverifiable' } + /** An update, rollback or recovery holds the host; it owns which slot serves. */ + | { outcome: 'fenced' } + | { outcome: 'started'; readiness: ServeReadiness } + +/** Launches the active slot only on proven exit; a live or unprovable process is left alone. */ +export async function wakeStoppedManagedOrcad( + options: OrcadSlotOptions, + onStarting: () => void = () => {} +): Promise { + const before = await readOrcadActivationRecord(options) + if (!before.active) { + return { outcome: 'not-activated' } + } + const liveness = await slotLiveness(options, before.active) + if (liveness !== 'DEAD') { + return { outcome: liveness === 'LIVE' ? 'serving' : 'unverifiable' } + } + if (await orcadActivationFenceExists(options)) { + return { outcome: 'fenced' } + } + return withOrcadActivationLock(options, async () => { + // Re-read under the fence: another client may have activated or started a slot meanwhile. + const active = (await readOrcadActivationRecord(options)).active + if (!active) { + return { outcome: 'not-activated' } + } + const identity = await resolveOrcadSlotIdentity(options, active) + onStarting() + return { outcome: 'started', readiness: await ensureOrcadSlotServing(options, identity) } + }) +} + +async function slotLiveness( + options: OrcadSlotOptions, + version: string +): Promise<'LIVE' | 'DEAD' | 'UNKNOWN'> { + return parseOrcadLiveness( + await execOrcadRemote( + options, + orcadLivenessProbeCommand(options.host, orcadSlotDir(options, version)) + ) + ) +} diff --git a/src/main/ssh/orcad-recovery-slot.ts b/src/main/ssh/orcad-recovery-slot.ts index 8e4cec5a768..1fb374a8eab 100644 --- a/src/main/ssh/orcad-recovery-slot.ts +++ b/src/main/ssh/orcad-recovery-slot.ts @@ -21,6 +21,7 @@ import { type OrcadStopOutcome } from './orcad-remote-process-control' import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { orcadActivationTransactionRoot } from './orcad-activation-lock' import { initialOrcadActivationAdmissionCommand, parseInitialOrcadActivationAdmission @@ -70,7 +71,9 @@ export function launchOrcadSlot( fullVersion: identity.version, userDataDir: options.userDataDir, bindHost: options.bindHost, - port: options.port + port: options.port, + // Every SSH launch is client-managed, so every one may idle out and be woken on connect. + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) }, expectation(identity) ) diff --git a/src/main/ssh/orcad-remote-launch-windows.ts b/src/main/ssh/orcad-remote-launch-windows.ts index 1a8c733063e..5aa0fedb192 100644 --- a/src/main/ssh/orcad-remote-launch-windows.ts +++ b/src/main/ssh/orcad-remote-launch-windows.ts @@ -33,7 +33,7 @@ import { ORCAD_READINESS_FILENAME, ORCAD_WINDOWS_PROCESS_FILENAME } from './orcad-remote-host-support' -import type { OrcadLaunchSpec } from './orcad-remote-launch' +import { orcadManagedLaunchEnv, type OrcadLaunchSpec } from './orcad-remote-launch' export class OrcadWindowsLaunchRefusedError extends Error { readonly code = 'orcad_windows_launch_refused' @@ -87,6 +87,10 @@ export function windowsOrcadLaunchCommand( `ORCA_VERSION=${spec.fullVersion}`, WINDOWS_BREAKAWAY_ENV_FLAG, `ORCA_USER_DATA=${spec.userDataDir}`, + ...orcadManagedLaunchEnv(spec).flatMap(([name, value]) => [ + WINDOWS_BREAKAWAY_ENV_FLAG, + `${name}=${value}` + ]), ORCAD_WINDOWS_BREAKAWAY_CONTRACT.argsFlag, '--json', '--bind', diff --git a/src/main/ssh/orcad-remote-launch.test.ts b/src/main/ssh/orcad-remote-launch.test.ts index 0be9ddc76c1..3730deb0ee1 100644 --- a/src/main/ssh/orcad-remote-launch.test.ts +++ b/src/main/ssh/orcad-remote-launch.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { ORCAD_MANAGED_ACTIVATION_ROOT_ENV } from '../../shared/orcad-idle-exit' import { ORCAD_READINESS_FILENAME, @@ -25,7 +26,8 @@ const SPEC = { fullVersion: '0.2.0+bb01', userDataDir: '/home/u/.orca', bindHost: '127.0.0.1', - port: 7777 + port: 7777, + activationRoot: '/home/u/.orca-remote/.orcad-activation-transaction' } const READY_LINE = JSON.stringify({ @@ -58,6 +60,14 @@ describe('orcadLaunchCommand', () => { expect(command).toContain(`ORCA_USER_DATA='${SPEC.userDataDir}'`) }) + it('names the activation fence on every launch, which enables idle exit', () => { + const command = orcadLaunchCommand(posix, SPEC) + expect(command).toContain(`${ORCAD_MANAGED_ACTIVATION_ROOT_ENV}='${SPEC.activationRoot}'`) + expect(command.indexOf(ORCAD_MANAGED_ACTIVATION_ROOT_ENV)).toBeLessThan( + command.indexOf('nohup') + ) + }) + it('declares the Windows refusal instead of emitting a command that cannot work', () => { expect(() => orcadLaunchCommand(windows, SPEC)).toThrow(OrcadRemoteLaunchUnsupportedError) }) diff --git a/src/main/ssh/orcad-remote-launch.ts b/src/main/ssh/orcad-remote-launch.ts index c133ddb4a88..28754f0c755 100644 --- a/src/main/ssh/orcad-remote-launch.ts +++ b/src/main/ssh/orcad-remote-launch.ts @@ -26,6 +26,11 @@ import type { ServeReadiness } from '../server/serve-readiness' import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime' import { ORCAD_STOP_REQUEST_FILENAME } from '../../shared/orcad-stop-request' import { windowsOrcadLivenessProbeCommand } from './orcad-remote-liveness-windows' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV, + readOrcadE2EIdleTimeoutMs +} from '../../shared/orcad-idle-exit' export { ORCAD_LOG_FILENAME, @@ -44,6 +49,22 @@ export type OrcadLaunchSpec = { /** Loopback by default; the client reaches it through an SSH local port-forward. */ bindHost: string port: number + /** The host's activation fence. Every SSH launch is client-managed, so every one may idle out. */ + activationRoot: string +} + +/** Env a managed launch adds; an older orcad ignores both. */ +export function orcadManagedLaunchEnv( + spec: OrcadLaunchSpec, + env: NodeJS.ProcessEnv = process.env +): [string, string][] { + const e2eTimeout = readOrcadE2EIdleTimeoutMs(env) + return [ + [ORCAD_MANAGED_ACTIVATION_ROOT_ENV, spec.activationRoot], + ...(e2eTimeout === null + ? [] + : [[ORCAD_E2E_IDLE_TIMEOUT_ENV, String(e2eTimeout)] satisfies [string, string]]) + ] } /** @@ -73,6 +94,7 @@ export function orcadLaunchCommand(host: RemoteHostPlatform, spec: OrcadLaunchSp 'umask 077 &&', `ORCA_VERSION=${shellEscape(spec.fullVersion)}`, `ORCA_USER_DATA=${shellEscape(spec.userDataDir)}`, + ...orcadManagedLaunchEnv(spec).map(([name, value]) => `${name}=${shellEscape(value)}`), // Keep $! equal to the runtime PID rather than a waiting shell's PID. `exec nohup "$orcad_runtime" ${entry}`, `--json --bind ${shellEscape(spec.bindHost)} --port ${String(spec.port)}`, diff --git a/src/main/ssh/orcad-remote-primitives.test.ts b/src/main/ssh/orcad-remote-primitives.test.ts index 8cc38a43eb0..290496deddb 100644 --- a/src/main/ssh/orcad-remote-primitives.test.ts +++ b/src/main/ssh/orcad-remote-primitives.test.ts @@ -212,7 +212,8 @@ describe('installed build identity and readiness', () => { fullVersion: '0.2.0+bb01', userDataDir: '/home/u/.orca', bindHost: '127.0.0.1', - port: 7777 + port: 7777, + activationRoot: '/home/u/.orca-remote/.orcad-activation-transaction' }, expectation ) diff --git a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts index 1ca251a2e1b..87022993655 100644 --- a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts +++ b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts @@ -134,7 +134,8 @@ async function launchTestRuntime( fullVersion: '0.2.0+bb01', userDataDir: dataDir, bindHost: '127.0.0.1', - port: 0 + port: 0, + activationRoot: join(dataDir, '.orcad-activation-transaction') }) if (legacyWrapper) { // The trailing command retains the old macOS waiting-shell behavior on every POSIX shell. diff --git a/src/main/ssh/orcad-remote-windows-commands.test.ts b/src/main/ssh/orcad-remote-windows-commands.test.ts index 7ea3489576d..d12f05cd66f 100644 --- a/src/main/ssh/orcad-remote-windows-commands.test.ts +++ b/src/main/ssh/orcad-remote-windows-commands.test.ts @@ -54,7 +54,8 @@ const spec: OrcadLaunchSpec = { fullVersion: '0.2.0+bb01', userDataDir: 'C:/Users/u/.orca', bindHost: '127.0.0.1', - port: 7777 + port: 7777, + activationRoot: `${base}/.orcad-activation-transaction` } function windowsConn(): { conn: SshConnection; writes: [string, string][] } { @@ -107,10 +108,22 @@ describe('Windows orcad commands run node.exe directly', () => { `${NODE} ${SCRIPT} slot-runtime ${slot} clear-stop-request` ) expect(windowsOrcadLaunchCommand(host, spec, SLOT_NODE)).toMatchInlineSnapshot( - `"C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.js --windows-breakaway-launch --stdout-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-readiness --stderr-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.log --process-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-process.json --env ORCA_VERSION=0.2.0+bb01 --env ORCA_USER_DATA=C:/Users/u/.orca --orcad-args --json --bind "127.0.0.1" --port "7777""` + `"C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.js --windows-breakaway-launch --stdout-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-readiness --stderr-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.log --process-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-process.json --env ORCA_VERSION=0.2.0+bb01 --env ORCA_USER_DATA=C:/Users/u/.orca --env ORCA_ORCAD_MANAGED_ACTIVATION_ROOT=C:/Users/u/.orca-remote/.orcad-activation-transaction --orcad-args --json --bind "127.0.0.1" --port "7777""` ) }) + it('passes the managed idle-exit fence through the breakaway environment', () => { + const command = windowsOrcadLaunchCommand( + host, + { ...spec, activationRoot: 'C:/Users/u/.orca-remote/.orcad-activation-transaction' }, + SLOT_NODE + ) + expect(command).toContain( + '--env ORCA_ORCAD_MANAGED_ACTIVATION_ROOT=C:/Users/u/.orca-remote/.orcad-activation-transaction --orcad-args' + ) + expect(command).not.toMatch(/[%$`']/u) + }) + it('never polls across SSH: runtime, launch, then one host-side wait', async () => { mockExec .mockResolvedValueOnce(encoded('__ORCAD_RUNTIME__', SLOT_NODE)) diff --git a/src/main/ssh/orcad-rollback-transition.ts b/src/main/ssh/orcad-rollback-transition.ts index a3d653affe5..fe9f27b78ce 100644 --- a/src/main/ssh/orcad-rollback-transition.ts +++ b/src/main/ssh/orcad-rollback-transition.ts @@ -21,7 +21,10 @@ import { } from './orcad-state-snapshot' import { orcadStopFreedTheHost } from './orcad-remote-process-control' import { joinRemotePath } from './ssh-remote-platform' -import type { OrcadActivationLockControl } from './orcad-activation-lock' +import { + orcadActivationTransactionRoot, + type OrcadActivationLockControl +} from './orcad-activation-lock' import type { OrcadRollbackTransaction } from './orcad-activation-transaction' import { createOrcadRollbackTransaction, @@ -209,7 +212,8 @@ export async function rollbackOrcadLocked( fullVersion: safety.target, userDataDir: options.userDataDir, bindHost: options.bindHost, - port: options.port + port: options.port, + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) }) } catch (error) { if (isUnconfirmedSshCommandTermination(error)) { diff --git a/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts b/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts index 7fa8a704a9d..82210ea7cf8 100644 --- a/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts +++ b/src/main/ssh/ssh-host-server-on-connect-telemetry.test.ts @@ -19,6 +19,7 @@ function deps(overrides: Partial = {}): HostServerOnCon return { managedEnvironmentId: () => null, ensureTunnel: vi.fn(async () => undefined), + ensureServing: vi.fn(async () => ({ state: 'serving' as const })), retireRetainedSource: vi.fn(async () => undefined), hasUnfinishedConversion: () => false, abandonConversion: vi.fn(async () => undefined), diff --git a/src/main/ssh/ssh-host-server-on-connect.test.ts b/src/main/ssh/ssh-host-server-on-connect.test.ts index 5d16e591192..fcc5836a190 100644 --- a/src/main/ssh/ssh-host-server-on-connect.test.ts +++ b/src/main/ssh/ssh-host-server-on-connect.test.ts @@ -17,6 +17,7 @@ function deps(overrides: Partial = {}): HostServerOnCon return { managedEnvironmentId: () => null, ensureTunnel: vi.fn(async () => undefined), + ensureServing: vi.fn(async () => ({ state: 'serving' as const })), retireRetainedSource: vi.fn(async () => undefined), hasUnfinishedConversion: () => false, abandonConversion: vi.fn(async () => undefined), @@ -51,6 +52,28 @@ describe('which server an SSH host runs on connect', () => { expect(d.convert).not.toHaveBeenCalled() }) + it('checks the server behind the tunnel on every connect to a converted host', async () => { + const d = deps({ managedEnvironmentId: () => 'env-9' }) + await resolveHostServerOnConnect(target, d) + expect(d.ensureServing).toHaveBeenCalledWith('env-9') + }) + + it('keeps a host whose stopped server could not be started managed, with the reason', async () => { + const detail = 'orcad did not become ready.\nLast lines of orcad.log:\nboom' + const d = deps({ + managedEnvironmentId: () => 'env-9', + ensureServing: vi.fn(async () => ({ state: 'unverifiable' as const, detail })) + }) + await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({ + route: 'managed', + environmentId: 'env-9', + serving: { state: 'unverifiable', detail } + }) + // Neither a relay fallback nor any verdict about the host's terminals. + expect(d.relayTerminals).not.toHaveBeenCalled() + expect(d.autoUpdate).not.toHaveBeenCalled() + }) + it('deploys an empty host directly, and converts a host with state', async () => { const empty = deps({ isEmptyHost: () => true }) await expect(resolveHostServerOnConnect(target, empty)).resolves.toMatchObject({ @@ -306,6 +329,27 @@ describe('which server an SSH host runs on connect', () => { expect(d.recordUpdateFailure).not.toHaveBeenCalled() }) + it('starts a stopped server before the update counts its terminals, as after a reboot', async () => { + const order: string[] = [] + const d = managed({ + ensureServing: vi.fn(async () => { + order.push('start') + return { state: 'started' as const, boundPort: null } + }), + // The restarted server's fresh daemon answers zero sessions, so the update goes ahead. + autoUpdate: vi.fn(async (_id, options) => { + order.push('update') + options.onUpdating() + return { outcome: 'updated' as const, activeVersion: '0.1.0+b' } + }) + }) + await expect(resolveHostServerOnConnect(target, d)).resolves.toEqual({ + route: 'managed', + environmentId: 'env-9' + }) + expect(order).toEqual(['start', 'update']) + }) + it('keeps the old version serving while terminals run, and retries on a later connect', async () => { const reason = '2 terminals are running on this host.' const d = managed({ diff --git a/src/main/ssh/ssh-host-server-on-connect.ts b/src/main/ssh/ssh-host-server-on-connect.ts index 43fbae5ae1e..216a71870f7 100644 --- a/src/main/ssh/ssh-host-server-on-connect.ts +++ b/src/main/ssh/ssh-host-server-on-connect.ts @@ -15,9 +15,11 @@ import type { } from '../../shared/orcad-managed-runtime' import type { SshManagedServerRelayReason, + SshManagedServerServingNote, SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' +import type { OrcadManagedServing } from './orcad-managed-serving' import { checkManagedServerUpdate, type ManagedServerUpdateDeps @@ -37,10 +39,15 @@ import { type HostServerReport } from './ssh-host-server-connect-events' -export type HostServerPhase = 'deploying' | 'converting' | 'connecting' | 'updating' +export type HostServerPhase = 'deploying' | 'converting' | 'connecting' | 'updating' | 'starting' export type HostServerOnConnectResult = - | { route: 'managed'; environmentId: string; update?: SshManagedServerUpdateNote } + | { + route: 'managed' + environmentId: string + update?: SshManagedServerUpdateNote + serving?: SshManagedServerServingNote + } | { route: 'relay' reason: SshManagedServerRelayReason @@ -58,6 +65,8 @@ export type HostServerTerminalVerdict = { export type HostServerOnConnectDeps = { managedEnvironmentId: (target: SshTarget) => string | null ensureTunnel: (environmentId: string) => Promise + /** Behind the tunnel: answers, or is started from its activated slot if proven stopped. */ + ensureServing: (environmentId: string) => Promise /** Retires a retained source once retirement is switched on; a failure only defers it. */ retireRetainedSource: (target: SshTarget) => Promise /** False when this build carries no orcad template, so nothing is tried on the host. */ @@ -136,6 +145,11 @@ async function decide( // Why before routing: until the commit lands the server is empty, so finish it or back out. return await convertHost(target, deps, trace, { checkTerminals: false }) } + const serving = await deps.ensureServing(existing) + if (serving.state === 'unverifiable') { + // Still the managed route: a stopped server says nothing about the host's terminals. + return { route: 'managed', environmentId: existing, serving } + } await deps.retireRetainedSource(target).catch((error: unknown) => { console.warn('[ssh] Source retirement deferred to a later connect:', error) }) diff --git a/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts b/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts index b7abdb242ee..379a802112a 100644 --- a/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts +++ b/src/renderer/src/components/settings/ssh-host-server-status-copy.test.ts @@ -31,6 +31,27 @@ describe('SSH host server status line', () => { }) }) + it('shows a stopped server starting, and why one could not be started', () => { + expect( + sshHostServerStatusLine(plain, { managedServer: { kind: 'setting-up', phase: 'starting' } }) + ?.text + ).toBe('Starting managed server…') + const detail = 'orcad did not become ready.\nLast lines of orcad.log:\nboom' + expect( + sshHostServerStatusLine(plain, { + managedServer: { + kind: 'managed', + environmentId: 'e', + serving: { state: 'unverifiable', detail } + } + }) + ).toEqual({ + tone: 'warning', + text: 'The managed Orca server isn’t running and couldn’t be started.', + detail + }) + }) + it('offers the move only while live relay terminals keep the host on the relay', () => { expect( sshHostServerStatusLine(plain, { diff --git a/src/renderer/src/components/settings/ssh-host-server-status-copy.ts b/src/renderer/src/components/settings/ssh-host-server-status-copy.ts index f518084ddfb..8e769ed4795 100644 --- a/src/renderer/src/components/settings/ssh-host-server-status-copy.ts +++ b/src/renderer/src/components/settings/ssh-host-server-status-copy.ts @@ -30,6 +30,16 @@ export function sshHostServerStatusLine( ) } } + if (status?.kind === 'managed' && status.serving) { + return { + tone: 'warning', + text: translate( + 'auto.components.settings.sshHostServer.notServing', + 'The managed Orca server isn’t running and couldn’t be started.' + ), + detail: status.serving.detail + } + } if (status?.kind === 'managed' && status.update) { return managedUpdateLine(status.update) } @@ -146,6 +156,11 @@ function settingUpLabel(phase: (typeof SSH_MANAGED_SERVER_PHASES)[number]): stri 'auto.components.settings.sshHostServer.updating', 'Updating managed server…' ) + case 'starting': + return translate( + 'auto.components.settings.sshHostServer.starting', + 'Starting managed server…' + ) } } diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 4eaa74b2e13..0b09b9266ba 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -12417,6 +12417,8 @@ "converting": "Moving this host’s projects to its managed Orca server…", "connecting": "Connecting to the managed Orca server…", "updating": "Updating managed server…", + "starting": "Starting managed server…", + "notServing": "The managed Orca server isn’t running and couldn’t be started.", "hostNewer": "Runs a managed Orca server from a newer Orca; it keeps that version.", "updateDeferred": "Runs a managed Orca server; it updates on a later connect: {{reason}}", "updateFailed": "Runs a managed Orca server on its previous version; updating it failed: {{reason}}", diff --git a/src/shared/orcad-idle-exit.ts b/src/shared/orcad-idle-exit.ts new file mode 100644 index 00000000000..c0034117646 --- /dev/null +++ b/src/shared/orcad-idle-exit.ts @@ -0,0 +1,40 @@ +/** + * Idle exit for an orcad a client launched over SSH, matching the relay: a host nobody has + * used for 15 minutes stops its server, and the next connect starts it again. + */ +import { z } from 'zod' + +/** + * Set only by a client's managed launch. Enables idle exit and names the host's activation + * fence, so an update or rollback in flight keeps the server up. User-started servers never + * carry it. + */ +export const ORCAD_MANAGED_ACTIVATION_ROOT_ENV = 'ORCA_ORCAD_MANAGED_ACTIVATION_ROOT' +/** Test-only quiet period; a client forwards it to the servers it launches. */ +export const ORCAD_E2E_IDLE_TIMEOUT_ENV = 'ORCA_E2E_ORCAD_IDLE_TIMEOUT_MS' +export const ORCAD_IDLE_EXIT_TIMEOUT_MS = 15 * 60_000 +const ORCAD_E2E_IDLE_TIMEOUT_MAX_MS = 60 * 60_000 + +/** The bounded test override, or null when unset or out of range. */ +export function readOrcadE2EIdleTimeoutMs(env: NodeJS.ProcessEnv): number | null { + const raw = env[ORCAD_E2E_IDLE_TIMEOUT_ENV] + const value = raw ? Number(raw) : Number.NaN + return Number.isSafeInteger(value) && value >= 1 && value <= ORCAD_E2E_IDLE_TIMEOUT_MAX_MS + ? value + : null +} + +/** Written to the data root just before an idle stop; the next start reports and clears it. */ +export const ORCAD_IDLE_STOP_RECORD_FILENAME = 'orcad-idle-stop.json' + +export const OrcadIdleStopRecordSchema = z.object({ + schemaVersion: z.literal(1), + kind: z.literal('orcad_idle_stop'), + pid: z.number().int().positive(), + version: z.string().max(255), + quietSince: z.iso.datetime({ offset: true }), + stoppedAt: z.iso.datetime({ offset: true }), + idleTimeoutMs: z.number().int().positive() +}) + +export type OrcadIdleStopRecord = z.infer diff --git a/src/shared/ssh-retained-payload-admission.ts b/src/shared/ssh-retained-payload-admission.ts index 9698cc39a95..ff522dec7e5 100644 --- a/src/shared/ssh-retained-payload-admission.ts +++ b/src/shared/ssh-retained-payload-admission.ts @@ -117,8 +117,18 @@ function admitSshManagedServerStatus(value: unknown): { managedServer?: SshManag return {} } const update = admitSshManagedServerUpdateNote('update' in value ? value.update : undefined) + const serving = + 'serving' in value && + value.serving && + typeof value.serving === 'object' && + 'state' in value.serving && + value.serving.state === 'unverifiable' && + 'detail' in value.serving && + typeof value.serving.detail === 'string' + ? { serving: { state: 'unverifiable' as const, detail: value.serving.detail } } + : {} return { - managedServer: { kind: 'managed', environmentId: value.environmentId, ...update } + managedServer: { kind: 'managed', environmentId: value.environmentId, ...update, ...serving } } } if (value.kind === 'setting-up' && 'phase' in value) { diff --git a/src/shared/ssh-types.ts b/src/shared/ssh-types.ts index 141868da409..35c27193a54 100644 --- a/src/shared/ssh-types.ts +++ b/src/shared/ssh-types.ts @@ -264,7 +264,8 @@ export const SSH_MANAGED_SERVER_PHASES = [ 'deploying', 'converting', 'connecting', - 'updating' + 'updating', + 'starting' ] as const export const SSH_MANAGED_SERVER_UPDATE_STATES = ['host-newer', 'deferred', 'failed'] as const @@ -275,6 +276,9 @@ export type SshManagedServerUpdateNote = { detail?: string } +/** A managed server that is down and could not be started, with why (and orcad.log's tail). */ +export type SshManagedServerServingNote = { state: 'unverifiable'; detail: string } + export const SSH_MANAGED_SERVER_RELAY_REASONS = [ 'orcad_unavailable', 'relay_terminals_live', @@ -288,7 +292,13 @@ export const SSH_MANAGED_SERVER_RELAY_REASONS = [ export type SshManagedServerRelayReason = (typeof SSH_MANAGED_SERVER_RELAY_REASONS)[number] export type SshManagedServerStatus = - | { kind: 'managed'; environmentId: string; update?: SshManagedServerUpdateNote } + | { + kind: 'managed' + environmentId: string + update?: SshManagedServerUpdateNote + /** Set when the server was not running and could not be started; never a terminal verdict. */ + serving?: SshManagedServerServingNote + } | { kind: 'setting-up'; phase: (typeof SSH_MANAGED_SERVER_PHASES)[number] } /** `detail` names the blocker for a refusal, or why orcad can't run on the host. */ | { diff --git a/tests/e2e/ssh-orcad-idle-exit.spec.ts b/tests/e2e/ssh-orcad-idle-exit.spec.ts new file mode 100644 index 00000000000..5ffa23d3428 --- /dev/null +++ b/tests/e2e/ssh-orcad-idle-exit.spec.ts @@ -0,0 +1,240 @@ +/** + * A managed orcad that stops is started again by the client, on a real host: + * + * 1. Idle: quit the client, the server exits and records an idle stop; a relaunched client + * reconnects and the server is running again with that record consumed. + * 2. Killed, then a reboot: a killed server comes back on the next call and adopts the daemon + * that kept its terminal; once both are gone, the next connect starts fresh. + * + * Docker only. `ORCA_E2E_ORCAD_CONVERT_TEMPLATE` names the linux-x64-glibc orcad build, and the + * client forwards a short test-only quiet period to the server it launches. + */ +import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import { expect, test } from './helpers/orca-app' +import { waitForSessionReady } from './helpers/store' +import { createRestartSession } from './helpers/orca-restart' +import { reconnect } from './helpers/orcad-convert-flow' +import { ORCAD_CONVERT_HOST_ENV } from './helpers/orcad-convert-host' +import { + cleanupDockerSshRelayTarget, + DOCKER_SSH_RELAY_REMOTE_REPO_PATH, + execDockerSshRelayTargetCommand, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { ORCAD_E2E_IDLE_TIMEOUT_ENV } from '../../src/shared/orcad-idle-exit' + +const HOST = process.env[ORCAD_CONVERT_HOST_ENV] +const TEMPLATE_SOURCE = process.env.ORCA_E2E_ORCAD_CONVERT_TEMPLATE +// Long enough that a connected client's own traffic never lets it lapse mid-test. +const IDLE_TIMEOUT_MS = 15_000 +const RECORD = '/root/.orca/orcad-idle-stop.json' + +/** PIDs of running orcad slots; empty once every slot has exited. */ +function runningOrcadPids(target: DockerSshRelayTarget): string[] { + return execDockerSshRelayTargetCommand( + target, + 'for f in /root/.orca-remote/orcad-*/.orcad-pid; do pid=$(cat "$f" 2>/dev/null) && ' + + 'kill -0 "$pid" 2>/dev/null && echo "$pid"; done; true' + ) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) +} + +function readIdleStopRecord(target: DockerSshRelayTarget): unknown { + const raw = execDockerSshRelayTargetCommand(target, `cat ${RECORD} 2>/dev/null || true`).trim() + return raw ? JSON.parse(raw) : null +} + +test('a managed orcad stops after idling and starts again on the next connect', async (// oxlint-disable-next-line no-empty-pattern -- Playwright's second fixture arg is testInfo; the first must be an object destructure to opt out of the default fixture set. +{}, testInfo) => { + test.skip( + HOST !== 'docker' || !TEMPLATE_SOURCE, + `Set ${ORCAD_CONVERT_HOST_ENV}=docker and ORCA_E2E_ORCAD_CONVERT_TEMPLATE` + ) + test.setTimeout(15 * 60_000) + const target = startDockerSshRelayTarget(testInfo) + const session = createRestartSession(testInfo, { + ORCA_ORCAD_TEMPLATE_PATH: TEMPLATE_SOURCE!, + [ORCAD_E2E_IDLE_TIMEOUT_ENV]: String(IDLE_TIMEOUT_MS) + }) + let app: ElectronApplication | null = null + try { + const first = await session.launch() + app = first.app + await waitForSessionReady(first.page) + // A managed host is reached through its server, not a relay, so no relay repo is added. + const remote = await first.page.evaluate( + async (input) => { + const { target: created } = await window.api.ssh.addTarget({ target: input }) + const state = await window.api.ssh.connect({ targetId: created.id }) + return { targetId: created.id, managedServer: state?.managedServer ?? null } + }, + { + label: `orcad idle E2E ${Date.now()}`, + host: target.host, + port: target.port, + username: 'root', + identityFile: target.identityFile, + identitiesOnly: true, + relayGracePeriodSeconds: 1 + } + ) + expect(remote.managedServer).toMatchObject({ kind: 'managed' }) + expect(runningOrcadPids(target)).toHaveLength(1) + + // While the client is connected the server stays up past its quiet period. + await first.page.waitForTimeout(IDLE_TIMEOUT_MS * 2) + expect(runningOrcadPids(target)).toHaveLength(1) + + await session.close(app) + app = null + await expect + .poll(() => runningOrcadPids(target), { timeout: 3 * 60_000 }) + .toEqual([]) + .catch((error: unknown) => { + // The server logs what kept it up; without it a timeout explains nothing. + console.error( + execDockerSshRelayTargetCommand(target, 'tail -n 40 /root/.orca-remote/orcad-*/orcad.log') + ) + throw error + }) + expect(readIdleStopRecord(target)).toMatchObject({ + kind: 'orcad_idle_stop', + idleTimeoutMs: IDLE_TIMEOUT_MS + }) + + const second = await session.launch() + app = second.app + await waitForSessionReady(second.page) + const connected = await reconnect(second.page, remote.targetId) + expect(JSON.parse(connected)).toMatchObject({ kind: 'managed' }) + expect(runningOrcadPids(target)).toHaveLength(1) + // The restarted server read the record, so a later crash cannot be mistaken for an idle stop. + expect(readIdleStopRecord(target)).toBeNull() + } finally { + if (app) { + await session.close(app) + } + await session.dispose() + cleanupDockerSshRelayTarget(target) + } +}) + +async function connectManagedHost( + page: Page, + target: DockerSshRelayTarget +): Promise<{ targetId: string; environmentId: string }> { + const connected = await page.evaluate( + async (input) => { + const { target: created } = await window.api.ssh.addTarget({ target: input }) + const state = await window.api.ssh.connect({ targetId: created.id }) + return { targetId: created.id, managedServer: state?.managedServer ?? null } + }, + { + label: `orcad restart E2E ${Date.now()}`, + host: target.host, + port: target.port, + username: 'root', + identityFile: target.identityFile, + identitiesOnly: true, + relayGracePeriodSeconds: 1 + } + ) + const server = connected.managedServer + if (server?.kind !== 'managed') { + throw new Error(`expected a managed server, got ${JSON.stringify(server)}`) + } + return { targetId: connected.targetId, environmentId: server.environmentId } +} + +async function callEnvironment(page: Page, environmentId: string, method: string, params: unknown) { + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params + }) + return response.ok ? { ok: true as const } : { ok: false as const, error: response.error } + }, + { environmentId, method, params } + ) +} + +/** The bracket keeps pgrep from matching the shell that runs it. */ +function processAlive(target: DockerSshRelayTarget, marker: string): boolean { + const found = execDockerSshRelayTargetCommand( + target, + `pgrep -f '[s]leep ${marker}' >/dev/null && echo yes || true` + ) + return found.trim() === 'yes' +} + +test('a killed managed orcad comes back with its terminal, and starts fresh after a reboot', async (// oxlint-disable-next-line no-empty-pattern -- Playwright's second fixture arg is testInfo; the first must be an object destructure to opt out of the default fixture set. +{}, testInfo) => { + test.skip( + HOST !== 'docker' || !TEMPLATE_SOURCE, + `Set ${ORCAD_CONVERT_HOST_ENV}=docker and ORCA_E2E_ORCAD_CONVERT_TEMPLATE` + ) + test.setTimeout(15 * 60_000) + const target = startDockerSshRelayTarget(testInfo) + const session = createRestartSession(testInfo, { ORCA_ORCAD_TEMPLATE_PATH: TEMPLATE_SOURCE! }) + let app: ElectronApplication | null = null + try { + const launched = await session.launch() + app = launched.app + const page = launched.page + await waitForSessionReady(page) + const { targetId, environmentId } = await connectManagedHost(page, target) + // A unique sleep length is the terminal's fingerprint in the process table. + const marker = String(800_000 + Math.floor(Math.random() * 100_000)) + expect( + await callEnvironment(page, environmentId, 'repo.add', { + path: DOCKER_SSH_RELAY_REMOTE_REPO_PATH + }) + ).toMatchObject({ ok: true }) + expect( + await callEnvironment(page, environmentId, 'terminal.create', { + worktree: `path:${DOCKER_SSH_RELAY_REMOTE_REPO_PATH}`, + command: `exec sleep ${marker}` + }) + ).toMatchObject({ ok: true }) + await expect.poll(() => processAlive(target, marker), { timeout: 60_000 }).toBe(true) + + // Killed: the daemon keeps the terminal, and the next call starts orcad, which adopts it. + execDockerSshRelayTargetCommand( + target, + 'kill -TERM $(cat /root/.orca-remote/orcad-*/.orcad-pid)' + ) + await expect.poll(() => runningOrcadPids(target), { timeout: 60_000 }).toEqual([]) + expect(processAlive(target, marker)).toBe(true) + await expect + .poll(async () => (await callEnvironment(page, environmentId, 'terminal.list', {})).ok, { + timeout: 3 * 60_000 + }) + .toBe(true) + expect(runningOrcadPids(target)).toHaveLength(1) + expect(processAlive(target, marker)).toBe(true) + + // A reboot takes orcad, the daemon and its terminal; the next connect starts both fresh. + execDockerSshRelayTargetCommand(target, "pkill -KILL -f '/root/[.]orca-remote/' || true") + execDockerSshRelayTargetCommand(target, `pkill -KILL -f '[s]leep ${marker}' || true`) + await expect.poll(() => runningOrcadPids(target), { timeout: 60_000 }).toEqual([]) + const connected = JSON.parse(await reconnect(page, targetId)) + expect(connected).toMatchObject({ kind: 'managed', environmentId }) + expect(connected).not.toHaveProperty('serving') + expect(runningOrcadPids(target)).toHaveLength(1) + expect(processAlive(target, marker)).toBe(false) + expect(await callEnvironment(page, environmentId, 'terminal.list', {})).toMatchObject({ + ok: true + }) + } finally { + if (app) { + await session.close(app) + } + await session.dispose() + cleanupDockerSshRelayTarget(target) + } +})