// Versioned-install plumbing for the remote relay. // // Why this exists: the relay used to install into a single shared directory // (~/.orca-remote/relay-v0.1.0) which the deploy step would overwrite in place // on every cross-version push. A daemon already loaded into memory then served // new clients off rewritten on-disk code, producing protocol drift and a // reconnect loop. We now install each (RELAY_VERSION + content-hash) bundle // into its own directory and never mutate it after the install finishes, // matching VS Code's `~/.vscode-server/bin//` layout. // // See: docs/ssh-relay-versioned-install-dirs.md import { join } from 'node:path' import { existsSync, readFileSync } from 'node:fs' import type { SshConnection } from './ssh-connection' import { RELAY_REMOTE_DIR } from './relay-protocol' import { execCommand } from './ssh-relay-deploy-helpers' import { acquireInstallLockParentCommand, listRelayBaseDirsCommand, lockMtimeEpochCommand, probeDirectoryExistsCommand, probeFileExistsCommand, probeRelayInstalledCommand, relayLivenessProbeCommand, removeRemoteTreeCommand, tryCreateInstallLockCommand, writeRemoteEmptyFileCommand } from './ssh-remote-commands' import { getRemoteHostPlatform, isWindowsRemoteHost, joinRemotePath, remoteBasename, type RemoteHostPlatform, type RemotePathFlavor } from './ssh-remote-platform' import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' import { isSshSessionLimitError } from './ssh-session-limit-error' // Why: the GC pass and the version-dir parser must agree on what counts as a // relay install dir. Single source of truth for both. The pattern matches the // new layout `relay-${RELAY_VERSION}+${hash}` and the legacy `relay-v${VERSION}` // so the GC eventually drains the old layout once its daemons idle out. const RELAY_VERSION_DIR_REGEX = /^relay-(v?\d+\.\d+\.\d+(\+[0-9a-f]+)?)$/ // Why: legacy dirs from before `.install-complete` was introduced (i.e. the // `relay-v0.1.0` shape with no content-hash suffix). They are missing the // install-complete sentinel by definition and need a separate liveness-only // GC check so they actually drain after the legacy daemon dies, instead of // living on remote disks forever. const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/ const INSTALL_LOCK_NAME = '.install-lock' const INSTALL_COMPLETE_NAME = '.install-complete' const INSTALL_LOCK_POLL_MS = 1_000 const INSTALL_LOCK_TIMEOUT_MS = 120_000 // Why: a stale lock dir from a crashed installer must be recoverable without // user intervention. After the timeout we check the lock's mtime; if it's // older than this window the previous installer is assumed dead and we steal // the lock. 2 minutes is well above a normal `npm install node-pty` runtime // (10–60s on slow hosts) so a slow concurrent installer is not falsely // declared dead. const INSTALL_LOCK_STALE_MS = 120_000 const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64') type RelayInstalledProbeOptions = { rethrowSessionLimitErrors?: boolean signal?: AbortSignal } function execHostCommand( conn: SshConnection, host: RemoteHostPlatform, command: string, options?: { signal?: AbortSignal } ): Promise { return execCommand(conn, command, { wrapCommand: host.commandDialect !== 'powershell', signal: options?.signal }) } /** * Read the local relay's content-hashed version (e.g. "0.1.0+0a5fe134d020") * from `${localRelayDir}/.version`. Throws on missing/empty so the caller * never silently falls back to a path where a daemon from a different code * generation may already be running — that fallback is the failure mode the * versioned-install design exists to prevent. */ export function readLocalFullVersion(localRelayDir: string): string { const versionFile = join(localRelayDir, '.version') if (!existsSync(versionFile)) { throw new Error( `Orca's local relay build is missing its version marker at ${versionFile}. ` + `This usually indicates a packaging or build problem; reinstall Orca.` ) } const v = readFileSync(versionFile, 'utf-8').trim() if (!v) { throw new Error( `Orca's local relay version marker at ${versionFile} is empty. ` + `This usually indicates a packaging or build problem; reinstall Orca.` ) } return v } /** * Compute the absolute remote install directory for a given content-hashed * version. The format is `${remoteHome}/${RELAY_REMOTE_DIR}/relay-${fullVersion}`. */ export function computeRemoteRelayDir( remoteHome: string, fullVersion: string, pathFlavor: RemotePathFlavor = 'posix' ): string { const host = pathFlavor === 'windows' ? getRemoteHostPlatform('win32-x64') : getRemoteHostPlatform('linux-x64') return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, `relay-${fullVersion}`) } /** * Probe whether a fully-installed relay already exists at remoteRelayDir. * * "Fully installed" means: the directory contains relay.js, its isolated * relay-watcher.js child, and the .install-complete sentinel written at the * end of a successful install. Missing artifacts force a complete re-deploy. */ export async function isRelayAlreadyInstalled( conn: SshConnection, remoteRelayDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: RelayInstalledProbeOptions ): Promise { try { const probe = await execHostCommand( conn, host, probeRelayInstalledCommand(host, remoteRelayDir), { signal: options?.signal } ) return probe.trim() === 'OK' } catch (err) { if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) { throw err } return false } } /** * Acquire the per-version install lock via atomic `mkdir`. Returns when the * caller owns the lock; throws if the lock could not be acquired within * INSTALL_LOCK_TIMEOUT_MS even after one stale-lock recovery attempt. * * Why mkdir: POSIX `mkdir` is atomic and fails with EEXIST if the dir already * exists, giving us a free mutex. A second concurrent caller polls and * eventually either acquires the lock or steals it after the stale window. */ export async function acquireInstallLock( conn: SshConnection, remoteRelayDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { const lockDir = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME) // Why: the parent dir may not exist yet on a first install. mkdir -p is // safe to run multiple times — it's a no-op if the dir already exists. await execHostCommand(conn, host, acquireInstallLockParentCommand(host, remoteRelayDir)) let start = Date.now() let recoveredOnce = false while (true) { try { const result = await execHostCommand(conn, host, tryCreateInstallLockCommand(host, lockDir)) if (result.trim().endsWith('OK')) { return } } catch { /* mkdir failed with non-zero — fall through to BUSY treatment */ } if (Date.now() - start >= INSTALL_LOCK_TIMEOUT_MS) { if (recoveredOnce) { throw new Error( `Could not acquire relay install lock at ${lockDir} after ${ INSTALL_LOCK_TIMEOUT_MS / 1000 }s; another install is in progress or the lock is wedged.` ) } // Stale-lock recovery: if the lock dir's mtime is older than the stale // window, the previous installer crashed. Steal it and retry once, // resetting the timeout window so a single post-recovery race doesn't // immediately exhaust the budget. const ageOk = await isLockStale(conn, lockDir, host) if (ageOk) { console.warn(`[ssh-relay] Stealing stale install lock at ${lockDir}`) await execHostCommand(conn, host, removeRemoteTreeCommand(host, lockDir)).catch(() => {}) recoveredOnce = true start = Date.now() continue } throw new Error( `Could not acquire relay install lock at ${lockDir} after ${ INSTALL_LOCK_TIMEOUT_MS / 1000 }s and the lock is not yet stale.` ) } await new Promise((r) => setTimeout(r, INSTALL_LOCK_POLL_MS)) } } async function isLockStale( conn: SshConnection, lockDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { try { // Why: `stat` flags differ between GNU coreutils (Linux) and BSD (macOS). // We try GNU first, then BSD; both produce a Unix epoch in seconds on // stdout. If both fail we conservatively treat the lock as not stale. const out = await execHostCommand(conn, host, lockMtimeEpochCommand(host, lockDir)) const mtimeSec = Number.parseInt(out.trim(), 10) if (!Number.isFinite(mtimeSec)) { return false } const ageMs = Date.now() - mtimeSec * 1000 return ageMs > INSTALL_LOCK_STALE_MS } catch { return false } } /** * Mark the install as complete and release the lock. Sentinel ordering is: * write `.install-complete` FIRST, then remove `.install-lock`. This ensures * a sibling dir is never observed by GC as "complete but locked", which * would lead GC to skip a recoverable dir indefinitely. */ export async function finalizeInstall( conn: SshConnection, remoteRelayDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { const sentinel = joinRemotePath(host, remoteRelayDir, INSTALL_COMPLETE_NAME) const lock = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME) await execHostCommand(conn, host, writeRemoteEmptyFileCommand(host, sentinel)) await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock)).catch(() => {}) } /** * Release the install lock without writing the completion sentinel. Called * from the failure path so the dir remains a recoverable partial that the * next deploy detects (alreadyInstalled=false) and re-runs upload+install. */ export async function abandonInstall( conn: SshConnection, remoteRelayDir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST ): Promise { const lock = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME) await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock)).catch(() => {}) } /** * Garbage-collect old version directories. Removes a sibling dir under * `${remoteHome}/${RELAY_REMOTE_DIR}/` only if ALL of: * * - it matches the relay-version-dir regex (allowlist) * - it is NOT the current version dir * - it has no live `relay-*.sock` (pgrep + connectability probe) * - it contains `.install-complete` (a fully-installed dir, not a partial) * - it does NOT contain `.install-lock` (no in-progress install) * * Best-effort: any error is logged and swallowed; GC must never block the * user from connecting. */ export async function gcOldRelayVersions( conn: SshConnection, remoteHome: string, currentDirAbsPath: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: { windowsNodePath?: string windowsSockNames?: string[] } ): Promise { const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) const currentDirName = remoteBasename(currentDirAbsPath, host) let listing: string try { listing = await execHostCommand(conn, host, listRelayBaseDirsCommand(host, baseDir)) } catch { return } const candidates = listing .split('\n') .map((s) => s.trim()) .filter(Boolean) .filter((name) => RELAY_VERSION_DIR_REGEX.test(name)) .filter((name) => name !== currentDirName) if (candidates.length === 0) { return } const removed: string[] = [] const kept: string[] = [] for (const name of candidates) { const dir = joinRemotePath(host, baseDir, name) try { const safe = await isCandidateSafeToRemove(conn, dir, name, host, options) if (!safe) { kept.push(name) continue } await execHostCommand(conn, host, removeRemoteTreeCommand(host, dir)) removed.push(name) } catch (err) { console.warn( `[ssh-relay] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}` ) kept.push(name) } } if (removed.length > 0) { const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : '' console.log( `[ssh-relay] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}` ) } } async function isCandidateSafeToRemove( conn: SshConnection, dir: string, name: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: { windowsNodePath?: string windowsSockNames?: string[] } ): Promise { const isLegacy = LEGACY_RELAY_DIR_REGEX.test(name) const lockDir = joinRemotePath(host, dir, INSTALL_LOCK_NAME) const lockProbe = await execHostCommand( conn, host, probeDirectoryExistsCommand(host, lockDir) ).catch(() => 'OPEN') const locked = lockProbe.trim() === 'LOCKED' if (locked) { // Why: a locked dir is normally unsafe to remove — but a STALE lock // (mtime older than INSTALL_LOCK_STALE_MS) means the previous installer // crashed and is never coming back. If the dir also has the // .install-complete sentinel (touch succeeded but the rm-lock at the // end of finalizeInstall failed), removing the dir is safe — no // installer is racing us, and the daemon (if any) keeps running off // its already-loaded code regardless of disk state. if (!(await isLockStale(conn, lockDir, host))) { return false } process.stderr.write?.(`[ssh-relay] GC: lock at ${lockDir} is stale; treating as recoverable\n`) } // Legacy dirs (relay-v0.1.0) predate .install-complete. Skip the sentinel // check for them and rely solely on the live-socket probe — that's the // only signal we have that a legacy daemon is still serving clients. if (!isLegacy) { const completePath = joinRemotePath(host, dir, INSTALL_COMPLETE_NAME) const completeProbe = await execHostCommand( conn, host, probeFileExistsCommand(host, completePath) ).catch(() => 'PARTIAL') if (completeProbe.trim() !== 'COMPLETE') { // Crashed-install partial; leave for the next deploy to recover. return false } } const sockAlive = await hasLiveRelaySocket(conn, dir, host, options) if (sockAlive) { return false } return true } async function hasLiveRelaySocket( conn: SshConnection, dir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: { windowsNodePath?: string windowsSockNames?: string[] } ): Promise { try { // Why: `ls -1 dir/relay-*.sock 2>/dev/null` lists socket files. For each, // we test -S to confirm it's a socket inode. We do NOT attempt to open // the socket here — `test -S` is sufficient for the GC decision and a // connect-and-close probe would race with a daemon that's about to idle. const windowsOptions = isWindowsRemoteHost(host) && options?.windowsNodePath ? { nodePath: options.windowsNodePath, pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) => windowsRelayPipePathsForSocketName(host, dir, sockName) ) } : undefined const out = await execHostCommand( conn, host, relayLivenessProbeCommand(host, dir, windowsOptions) ) return out.includes('ALIVE') } catch { return false } }