orca/src/main/ssh/ssh-relay-versioned-install.ts

// Versioned-install plumbing for the remote relay.
//
// Why this exists: the relay used to install into a single shared directory
// (~/.orca-remote/relay-v0.1.0) which the deploy step would overwrite in place
// on every cross-version push. A daemon already loaded into memory then served
// new clients off rewritten on-disk code, producing protocol drift and a
// reconnect loop. We now install each (RELAY_VERSION + content-hash) bundle
// into its own directory and never mutate it after the install finishes,
// matching VS Code's `~/.vscode-server/bin/<commit>/` layout.
//
// See: docs/ssh-relay-versioned-install-dirs.md

import { join } from 'node:path'
import { existsSync, readFileSync } from 'node:fs'
import type { SshConnection } from './ssh-connection'
import { RELAY_REMOTE_DIR } from './relay-protocol'
import { execCommand } from './ssh-relay-deploy-helpers'
import { lockAgeSecondsCommand } from './ssh-relay-install-lock-commands'
import { INSTALL_LOCK_STALE_MS, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock'
import {
  listRelayBaseDirsCommand,
  probeDirectoryExistsCommand,
  probeFileExistsCommand,
  probeRelayInstalledCommand,
  relayLivenessProbeCommand,
  removeRemoteTreeCommand,
  writeRemoteEmptyFileCommand
} from './ssh-remote-commands'
import {
  getRemoteHostPlatform,
  isWindowsRemoteHost,
  joinRemotePath,
  remoteBasename,
  type RemoteHostPlatform,
  type RemotePathFlavor
} from './ssh-remote-platform'
import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints'
import { isSshSessionLimitError } from './ssh-session-limit-error'

// Why: the GC pass and the version-dir parser must agree on what counts as a
// relay install dir. Single source of truth for both. The pattern matches the
// new layout `relay-${RELAY_VERSION}+${hash}` and the legacy `relay-v${VERSION}`
// so the GC eventually drains the old layout once its daemons idle out.
const RELAY_VERSION_DIR_REGEX = /^relay-(v?\d+\.\d+\.\d+(\+[0-9a-f]+)?)$/

// Why: legacy dirs from before `.install-complete` was introduced (i.e. the
// `relay-v0.1.0` shape with no content-hash suffix). They are missing the
// install-complete sentinel by definition and need a separate liveness-only
// GC check so they actually drain after the legacy daemon dies, instead of
// living on remote disks forever.
const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/

const INSTALL_COMPLETE_NAME = '.install-complete'
const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64')

type RelayInstalledProbeOptions = {
  rethrowSessionLimitErrors?: boolean
  signal?: AbortSignal
}

function execHostCommand(
  conn: SshConnection,
  host: RemoteHostPlatform,
  command: string,
  options?: { signal?: AbortSignal }
): Promise<string> {
  return execCommand(conn, command, {
    wrapCommand: host.commandDialect !== 'powershell',
    signal: options?.signal
  })
}

/**
 * Read the local relay's content-hashed version (e.g. "0.1.0+0a5fe134d020")
 * from `${localRelayDir}/.version`. Throws on missing/empty so the caller
 * never silently falls back to a path where a daemon from a different code
 * generation may already be running — that fallback is the failure mode the
 * versioned-install design exists to prevent.
 */
export function readLocalFullVersion(localRelayDir: string): string {
  const versionFile = join(localRelayDir, '.version')
  if (!existsSync(versionFile)) {
    throw new Error(
      `Orca's local relay build is missing its version marker at ${versionFile}. ` +
        `This usually indicates a packaging or build problem; reinstall Orca.`
    )
  }
  const v = readFileSync(versionFile, 'utf-8').trim()
  if (!v) {
    throw new Error(
      `Orca's local relay version marker at ${versionFile} is empty. ` +
        `This usually indicates a packaging or build problem; reinstall Orca.`
    )
  }
  return v
}

/**
 * Compute the absolute remote install directory for a given content-hashed
 * version. The format is `${remoteHome}/${RELAY_REMOTE_DIR}/relay-${fullVersion}`.
 */
export function computeRemoteRelayDir(
  remoteHome: string,
  fullVersion: string,
  pathFlavor: RemotePathFlavor = 'posix'
): string {
  const host =
    pathFlavor === 'windows'
      ? getRemoteHostPlatform('win32-x64')
      : getRemoteHostPlatform('linux-x64')
  return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, `relay-${fullVersion}`)
}

/**
 * Probe whether a fully-installed relay already exists at remoteRelayDir.
 *
 * "Fully installed" means: the directory contains relay.js, its isolated
 * relay-watcher.js child, and the .install-complete sentinel written at the
 * end of a successful install. Missing artifacts force a complete re-deploy.
 */
export async function isRelayAlreadyInstalled(
  conn: SshConnection,
  remoteRelayDir: string,
  host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
  options?: RelayInstalledProbeOptions
): Promise<boolean> {
  try {
    const probe = await execHostCommand(
      conn,
      host,
      probeRelayInstalledCommand(host, remoteRelayDir),
      { signal: options?.signal }
    )
    return probe.trim() === 'OK'
  } catch (err) {
    options?.signal?.throwIfAborted()
    if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) {
      throw err
    }
    return false
  }
}

async function isRelayInstallLockStale(
  conn: SshConnection,
  lockDir: string,
  host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
): Promise<boolean> {
  try {
    // Why: compute age on the remote host so clock skew between two Orca
    // clients cannot make a live repair lock look older than it is.
    const out = await execHostCommand(conn, host, lockAgeSecondsCommand(host, lockDir))
    const ageSec = Number.parseInt(out.trim(), 10)
    if (!Number.isFinite(ageSec) || ageSec < 0) {
      return false
    }
    return ageSec * 1000 > INSTALL_LOCK_STALE_MS
  } catch {
    return false
  }
}

/**
 * Mark the install as complete and release the lock. Sentinel ordering is:
 * write `.install-complete` FIRST, then remove `.install-lock`. This ensures
 * a sibling dir is never observed by GC as "complete but locked", which
 * would lead GC to skip a recoverable dir indefinitely.
 */
export async function finalizeInstall(
  conn: SshConnection,
  remoteRelayDir: string,
  host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
  options?: { signal?: AbortSignal }
): Promise<void> {
  const sentinel = joinRemotePath(host, remoteRelayDir, INSTALL_COMPLETE_NAME)
  const lock = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME)
  await execHostCommand(conn, host, writeRemoteEmptyFileCommand(host, sentinel), {
    signal: options?.signal
  })
  await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock), {
    signal: options?.signal
  }).catch(() => {})
  options?.signal?.throwIfAborted()
}

/**
 * Release the install lock without writing the completion sentinel. Called
 * from the failure path so the dir remains a recoverable partial that the
 * next deploy detects (alreadyInstalled=false) and re-runs upload+install.
 */
export async function abandonInstall(
  conn: SshConnection,
  remoteRelayDir: string,
  host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
): Promise<void> {
  const lock = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME)
  await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock)).catch(() => {})
}

/**
 * Garbage-collect old version directories. Removes a sibling dir under
 * `${remoteHome}/${RELAY_REMOTE_DIR}/` only if ALL of:
 *
 *   - it matches the relay-version-dir regex (allowlist)
 *   - it is NOT the current version dir
 *   - it has no live `relay-*.sock` (pgrep + connectability probe)
 *   - it contains `.install-complete` (a fully-installed dir, not a partial)
 *   - it does NOT contain `.install-lock` (no in-progress install)
 *
 * Best-effort: any error is logged and swallowed; GC must never block the
 * user from connecting.
 */
export async function gcOldRelayVersions(
  conn: SshConnection,
  remoteHome: string,
  currentDirAbsPath: string,
  host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
  options?: {
    windowsNodePath?: string
    windowsSockNames?: string[]
  }
): Promise<void> {
  const baseDir = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR)
  const currentDirName = remoteBasename(currentDirAbsPath, host)
  let listing: string
  try {
    listing = await execHostCommand(conn, host, listRelayBaseDirsCommand(host, baseDir))
  } catch {
    return
  }
  const candidates = listing
    .split('\n')
    .map((s) => s.trim())
    .filter(Boolean)
    .filter((name) => RELAY_VERSION_DIR_REGEX.test(name))
    .filter((name) => name !== currentDirName)

  if (candidates.length === 0) {
    return
  }

  const removed: string[] = []
  const kept: string[] = []
  for (const name of candidates) {
    const dir = joinRemotePath(host, baseDir, name)
    try {
      const safe = await isCandidateSafeToRemove(conn, dir, name, host, options)
      if (!safe) {
        kept.push(name)
        continue
      }
      await execHostCommand(conn, host, removeRemoteTreeCommand(host, dir))
      removed.push(name)
    } catch (err) {
      console.warn(
        `[ssh-relay] GC failed for ${dir}: ${err instanceof Error ? err.message : String(err)}`
      )
      kept.push(name)
    }
  }

  if (removed.length > 0) {
    const keptSuffix = kept.length > 0 ? ` (kept: ${kept.join(', ')})` : ''
    console.log(
      `[ssh-relay] GC: removed ${removed.length} stale version dir(s): ${removed.join(', ')}${keptSuffix}`
    )
  }
}

async function isCandidateSafeToRemove(
  conn: SshConnection,
  dir: string,
  name: string,
  host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
  options?: {
    windowsNodePath?: string
    windowsSockNames?: string[]
  }
): Promise<boolean> {
  const isLegacy = LEGACY_RELAY_DIR_REGEX.test(name)

  const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME)
  const lockProbe = await execHostCommand(
    conn,
    host,
    probeDirectoryExistsCommand(host, lockDir)
  ).catch(() => 'OPEN')
  const locked = lockProbe.trim() === 'LOCKED'

  if (locked) {
    // Why: a locked dir is normally unsafe to remove — but a STALE lock
    // (remote age older than INSTALL_LOCK_STALE_MS) means the previous installer
    // crashed and is never coming back. If the dir also has the
    // .install-complete sentinel (touch succeeded but the rm-lock at the
    // end of finalizeInstall failed), removing the dir is safe — no
    // installer is racing us, and the daemon (if any) keeps running off
    // its already-loaded code regardless of disk state.
    if (!(await isRelayInstallLockStale(conn, lockDir, host))) {
      return false
    }
    process.stderr.write?.(`[ssh-relay] GC: lock at ${lockDir} is stale; treating as recoverable\n`)
  }

  // Legacy dirs (relay-v0.1.0) predate .install-complete. Skip the sentinel
  // check for them and rely solely on the live-socket probe — that's the
  // only signal we have that a legacy daemon is still serving clients.
  if (!isLegacy) {
    const completePath = joinRemotePath(host, dir, INSTALL_COMPLETE_NAME)
    const completeProbe = await execHostCommand(
      conn,
      host,
      probeFileExistsCommand(host, completePath)
    ).catch(() => 'PARTIAL')
    if (completeProbe.trim() !== 'COMPLETE') {
      // Crashed-install partial; leave for the next deploy to recover.
      return false
    }
  }

  const sockAlive = await hasLiveRelaySocket(conn, dir, host, options)
  if (sockAlive) {
    return false
  }
  return true
}

async function hasLiveRelaySocket(
  conn: SshConnection,
  dir: string,
  host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
  options?: {
    windowsNodePath?: string
    windowsSockNames?: string[]
  }
): Promise<boolean> {
  try {
    // Why: `ls -1 dir/relay-*.sock 2>/dev/null` lists socket files. For each,
    // we test -S to confirm it's a socket inode. We do NOT attempt to open
    // the socket here — `test -S` is sufficient for the GC decision and a
    // connect-and-close probe would race with a daemon that's about to idle.
    const windowsOptions =
      isWindowsRemoteHost(host) && options?.windowsNodePath
        ? {
            nodePath: options.windowsNodePath,
            pipePaths: (options.windowsSockNames ?? []).flatMap((sockName) =>
              windowsRelayPipePathsForSocketName(host, dir, sockName)
            )
          }
        : undefined
    const out = await execHostCommand(
      conn,
      host,
      relayLivenessProbeCommand(host, dir, windowsOptions)
    )
    return out.includes('ALIVE')
  } catch {
    return false
  }
}