From 9ddcc9b9f0b43af3e676f75e2bce05556ea5e258 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Thu, 1 Oct 2026 01:10:47 -0700 Subject: [PATCH] fix(ssh): collect relay versions only when provably exited; runtimes/ store GC (#24130) * fix(ssh): relay version GC deletes only on an exited verdict and keeps the previous build The relay records .relay-pid in its version dir once it owns its socket. GC calls a relay version dir exited only when that PID is provably dead and every relay-*.sock refuses a connection; a dir without a PID file keeps the test -S rule. The most recently completed other relay build is pinned like orcad's rollback target. Design D5 GC liveness. * feat(ssh): collect the shared runtimes/ Node store and give it its own owner runtimes/ gets its own owner in the install model, so no version-dir GC (new or old clients, whose listings are prefix-scoped) can list or delete it. A store pass removes node- only when no retained dir references it, it is neither a current pin nor the newest other verified runtime, and a ps or /proc check ran and found no process using it. Legacy relay-*/orcad-* dirs are read for references and reported as diagnostics only (design D10 two-step). Wired behind orcad GC's nodeRuntimePins. * fix(ssh): runtime store process check holds runtimes reached through a symlinked home /proc exe resolves symlinks and argv keeps whatever spelling launched the runtime, so filtering on the exact $root path missed in-use runtimes on hosts like /home -> /var/home. Filter on the store segment instead; the parser already attributes holds root-agnostically. * test(ssh): wait for the holder process to spawn instead of a fixed delay --------- Co-authored-by: m4air --- src/main/ssh/orcad-remote-gc.test.ts | 20 ++ src/main/ssh/orcad-remote-gc.ts | 28 +- .../ssh/relay-version-dir-liveness.test.ts | 131 ++++++++ src/main/ssh/relay-version-dir-liveness.ts | 65 ++++ .../ssh/remote-install-gc-termination.test.ts | 16 +- src/main/ssh/remote-install-gc.ts | 67 ++++- src/main/ssh/remote-install-model.test.ts | 36 ++- src/main/ssh/remote-install-model.ts | 40 ++- .../remote-install-previous-version.test.ts | 160 ++++++++++ .../ssh/remote-install-previous-version.ts | 106 +++++++ .../ssh/remote-node-runtime-store-gc.test.ts | 279 ++++++++++++++++++ src/main/ssh/remote-node-runtime-store-gc.ts | 249 ++++++++++++++++ .../remote-node-runtime-store-inventory.ts | 140 +++++++++ .../ssh-relay-cross-version-isolation.test.ts | 7 + src/main/ssh/ssh-relay-deploy.ts | 1 + src/main/ssh/ssh-relay-endpoint-incumbent.ts | 4 +- .../ssh/ssh-relay-versioned-install.test.ts | 5 + src/main/ssh/ssh-remote-commands.ts | 2 +- src/relay/relay-daemon.ts | 2 + src/relay/relay-pid-publication.test.ts | 41 +++ src/relay/relay-pid-publication.ts | 39 +++ src/shared/relay-artifacts.ts | 3 + 22 files changed, 1396 insertions(+), 45 deletions(-) create mode 100644 src/main/ssh/relay-version-dir-liveness.test.ts create mode 100644 src/main/ssh/relay-version-dir-liveness.ts create mode 100644 src/main/ssh/remote-install-previous-version.test.ts create mode 100644 src/main/ssh/remote-install-previous-version.ts create mode 100644 src/main/ssh/remote-node-runtime-store-gc.test.ts create mode 100644 src/main/ssh/remote-node-runtime-store-gc.ts create mode 100644 src/main/ssh/remote-node-runtime-store-inventory.ts create mode 100644 src/relay/relay-pid-publication.test.ts create mode 100644 src/relay/relay-pid-publication.ts diff --git a/src/main/ssh/orcad-remote-gc.test.ts b/src/main/ssh/orcad-remote-gc.test.ts index 9d365c631ef..6ef88795013 100644 --- a/src/main/ssh/orcad-remote-gc.test.ts +++ b/src/main/ssh/orcad-remote-gc.test.ts @@ -201,4 +201,24 @@ describe('orcad GC', () => { expect(removed).toEqual(['orcad-0.0.9+dead']) }) + + it('collects the runtime store only when the caller names its runtime pins', async () => { + const removed: string[] = [] + scriptHost({ listing: [], removed }) + const options = { + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb', + record: emptyOrcadActivationRecord() + } + const inventories = (): number => + mockExec.mock.calls.filter(([, command]) => String(command).includes('RUNTIME_STORE')).length + + await gcOldOrcadVersions(options) + expect(inventories()).toBe(0) + + await gcOldOrcadVersions({ ...options, nodeRuntimePins: ['a'.repeat(64)] }) + expect(inventories()).toBe(1) + }) }) diff --git a/src/main/ssh/orcad-remote-gc.ts b/src/main/ssh/orcad-remote-gc.ts index 09997dc40d3..2f1692201e4 100644 --- a/src/main/ssh/orcad-remote-gc.ts +++ b/src/main/ssh/orcad-remote-gc.ts @@ -1,13 +1,12 @@ /** - * orcad's garbage collection, and the half of §06 falsifier 1 that says who owns it. + * orcad's garbage collection, and who owns it (design D10; to be tracked in + * docs/reference/remote-server-install-model.md). * - * **Each model GCs only its own namespace, permanently.** orcad removes `orcad-/` - * directories; the relay removes `relay-/` directories; neither ever removes the other's, - * and no plan item makes one the winner. That is not a migration compromise — the two models - * serve different users on the same machine (SSH target vs paired peer), so there is no - * moment at which one of them is entitled to clean up after the other. A pass that deleted - * the sibling's tree would be reaching across the execution boundary the whole design exists - * to keep intact. + * **Each model GCs only its own namespace.** orcad removes `orcad-/` directories; the relay + * removes `relay-/` directories; neither ever removes the other's. The converged server's + * migration sweep takes over legacy directories only in the release after it has listed them + * as diagnostics, and only on an `exited` verdict. A pass that deleted the sibling's tree + * would be reaching across the execution boundary the whole design exists to keep intact. * * On top of the ownership rule, orcad pins three directories that are idle-looking but * load-bearing: the active version, the rollback target, and whichever version the LIVE @@ -25,6 +24,7 @@ import { parseOrcadLiveness } from './orcad-remote-launch' import type { RemoteHostPlatform } from './ssh-remote-platform' +import { gcRemoteNodeRuntimeStore } from './remote-node-runtime-store-gc' export type OrcadGcOptions = { conn: SshConnection @@ -41,6 +41,11 @@ export type OrcadGcOptions = { * would remove the tree under a running process. */ liveDaemonVersion?: string | null + /** + * executableSha256 of every runtime pin this client runs. Also the gate for the shared + * runtime store pass: without it this client cannot say which runtime is current. + */ + nodeRuntimePins?: readonly string[] signal?: AbortSignal } @@ -75,4 +80,11 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise } } ) + // Why after the version pass: removing version dirs is what drops their runtime references. + if (options.nodeRuntimePins?.length) { + await gcRemoteNodeRuntimeStore(options.conn, options.host, options.remoteHome, { + currentPins: options.nodeRuntimePins, + signal: options.signal + }) + } } diff --git a/src/main/ssh/relay-version-dir-liveness.test.ts b/src/main/ssh/relay-version-dir-liveness.test.ts new file mode 100644 index 00000000000..0d05ae6bc95 --- /dev/null +++ b/src/main/ssh/relay-version-dir-liveness.test.ts @@ -0,0 +1,131 @@ +/** Runs the generated POSIX liveness probe through a real `/bin/sh` against real sockets. */ +import { execFileSync, spawnSync } from 'node:child_process' +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type Server } from 'node:net' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { RELAY_PID_FILENAME } from '../../shared/relay-artifacts' +import { + parseRelayVersionDirLiveness, + relayVersionDirLivenessCommand +} from './relay-version-dir-liveness' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const host = getRemoteHostPlatform('linux-x64') +const posixOnly = process.platform === 'win32' ? describe.skip : describe + +function probe(dir: string, nodePath: string | undefined = process.execPath): string { + return execFileSync('/bin/sh', ['-c', relayVersionDirLivenessCommand(host, dir, { nodePath })], { + encoding: 'utf8' + }) +} + +/** A socket inode left by a SIGKILLed listener: connect is refused. Returns the dead PID. */ +function leaveStaleSocket(sockPath: string): number { + const child = spawnSync( + process.execPath, + [ + '-e', + 'require("net").createServer().listen(process.argv[1],()=>{' + + 'process.stdout.write(String(process.pid));process.kill(process.pid,"SIGKILL")})', + sockPath + ], + { encoding: 'utf8' } + ) + return Number.parseInt(child.stdout, 10) +} + +posixOnly('relayVersionDirLivenessCommand (real shell)', () => { + const dirs: string[] = [] + const servers: Server[] = [] + afterEach(async () => { + await Promise.all( + servers.splice(0).map((server) => new Promise((done) => server.close(() => done()))) + ) + for (const dir of dirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } + }) + + function versionDir(): string { + // Short base: sun_path caps socket paths near 104 bytes on macOS. + const dir = mkdtempSync(join('/tmp', 'rvl-')) + dirs.push(dir) + return dir + } + + it('is exited for a stale socket whose recorded PID is dead', () => { + const dir = versionDir() + const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock')) + writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`) + + expect(parseRelayVersionDirLiveness(probe(dir))).toBe('exited') + }) + + it('is live for a stale socket whose recorded PID is still running', () => { + const dir = versionDir() + leaveStaleSocket(join(dir, 'relay-a.sock')) + writeFileSync(join(dir, RELAY_PID_FILENAME), `${process.pid}\n`) + + expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live') + }) + + it('keeps the test -S rule for a refused socket without a PID file', () => { + const dir = versionDir() + leaveStaleSocket(join(dir, 'relay-a.sock')) + + expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live') + }) + + it('is exited without a PID file only when no socket is left', () => { + expect(parseRelayVersionDirLiveness(probe(versionDir()))).toBe('exited') + }) + + it('is live when another relay of this build still accepts on its socket', async () => { + const dir = versionDir() + const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock')) + writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`) + const server = createServer() + servers.push(server) + await new Promise((done) => server.listen(join(dir, 'relay-b.sock'), done)) + + expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live') + }) + + it('is unverifiable when the connect probe times out', () => { + const dir = versionDir() + const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock')) + writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`) + // Stands in for node: the connect probe prints `unknown` when its timer fires first. + const timedOutNode = join(dir, 'node') + writeFileSync(timedOutNode, '#!/bin/sh\nprintf unknown\n') + chmodSync(timedOutNode, 0o755) + + expect(parseRelayVersionDirLiveness(probe(dir, timedOutNode))).toBe('unverifiable') + }) + + it('is unverifiable when no Node is available to test a leftover socket', () => { + const dir = versionDir() + const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock')) + writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`) + + expect(parseRelayVersionDirLiveness(probe(dir, ''))).toBe('unverifiable') + }) + + it('is unverifiable for an unreadable PID record', () => { + const dir = versionDir() + writeFileSync(join(dir, RELAY_PID_FILENAME), 'not-a-pid\n') + + expect(parseRelayVersionDirLiveness(probe(dir))).toBe('unverifiable') + }) +}) + +describe('parseRelayVersionDirLiveness', () => { + it('maps the Windows pipe vocabulary and treats anything else as unverifiable', () => { + expect(parseRelayVersionDirLiveness('ALIVE')).toBe('live') + expect(parseRelayVersionDirLiveness('WAITING')).toBe('exited') + expect(parseRelayVersionDirLiveness('DEAD\n')).toBe('exited') + expect(parseRelayVersionDirLiveness('')).toBe('unverifiable') + expect(parseRelayVersionDirLiveness('UNKNOWN')).toBe('unverifiable') + }) +}) diff --git a/src/main/ssh/relay-version-dir-liveness.ts b/src/main/ssh/relay-version-dir-liveness.ts new file mode 100644 index 00000000000..1a41d16f4f6 --- /dev/null +++ b/src/main/ssh/relay-version-dir-liveness.ts @@ -0,0 +1,65 @@ +/** + * Whether a relay version directory is still in use, answered with the execution-boundary + * vocabulary (docs/reference/ssh-execution-boundary.md): `live` / `unverifiable` / `exited`. + * + * Design D5: a directory is `exited` only when its recorded `.relay-pid` is provably dead AND + * every `relay-*.sock` in it refuses a connection. The PID is checked first so a live daemon + * about to idle is never connected to (a connection would cancel its grace timer). A directory + * with no PID file was last used by a relay that predates it and keeps the `test -S` rule. + */ +import { RELAY_PID_FILENAME } from '../../shared/relay-artifacts' +import { shellEscape } from './ssh-connection-utils' +import { posixProcessAliveShellFunction } from './orcad-remote-host-support' +import { RELAY_CONNECT_PROBE_JS, type RelayEndpointVerdict } from './ssh-relay-endpoint-incumbent' +import { relayLivenessProbeCommand, type WindowsRelayLivenessOptions } from './ssh-remote-commands' +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +export function relayVersionDirLivenessCommand( + host: RemoteHostPlatform, + dir: string, + options: { nodePath?: string; windows?: WindowsRelayLivenessOptions } = {} +): string { + if (isWindowsRemoteHost(host)) { + return relayLivenessProbeCommand(host, dir, options.windows) + } + return [ + `dir=${shellEscape(dir)}`, + `node=${shellEscape(options.nodePath ?? '')}`, + `pid_file="$dir"/${RELAY_PID_FILENAME}`, + 'socks=', + 'for f in "$dir"/relay-*.sock "$dir"/relay.sock; do [ -S "$f" ] && socks=yes; done', + 'if [ ! -e "$pid_file" ]; then', + ' if [ -n "$socks" ]; then echo LIVE; else echo EXITED; fi', + ' exit 0', + 'fi', + // Prints UNKNOWN and exits when kill -0 fails for any reason but "No such process". + posixProcessAliveShellFunction({ refuseUnverifiable: true }), + 'pid=$(cat "$pid_file" 2>/dev/null) || { echo UNVERIFIABLE; exit 0; }', + 'case "$pid" in "" | *[!0-9]*) echo UNVERIFIABLE; exit 0;; esac', + 'if orcad_alive "$pid"; then echo LIVE; exit 0; fi', + 'for f in "$dir"/relay-*.sock "$dir"/relay.sock; do', + ' [ -S "$f" ] || continue', + ' [ -n "$node" ] || { echo UNVERIFIABLE; exit 0; }', + // Another relay of this build may share the dir under its own socket; only a refusal clears it. + ` r=$("$node" -e ${shellEscape(RELAY_CONNECT_PROBE_JS)} "$f" 2>/dev/null) || r=unknown`, + ' case "$r" in', + ' refused | absent) ;;', + ' accepted) echo LIVE; exit 0;;', + ' *) echo UNVERIFIABLE; exit 0;;', + ' esac', + 'done', + 'echo EXITED' + ].join('\n') +} + +export function parseRelayVersionDirLiveness(output: string): RelayEndpointVerdict { + const token = output.trim().split('\n').pop()?.trim() ?? '' + // ALIVE / DEAD / WAITING are the Windows pipe probe's vocabulary. + if (token === 'LIVE' || token === 'ALIVE') { + return 'live' + } + if (token === 'EXITED' || token === 'DEAD' || token === 'WAITING') { + return 'exited' + } + return 'unverifiable' +} diff --git a/src/main/ssh/remote-install-gc-termination.test.ts b/src/main/ssh/remote-install-gc-termination.test.ts index 73b50e7b6f1..7cf6ff3f208 100644 --- a/src/main/ssh/remote-install-gc-termination.test.ts +++ b/src/main/ssh/remote-install-gc-termination.test.ts @@ -11,6 +11,7 @@ import { gcOldRelayVersions } from './remote-install-gc' import { execCommand } from './ssh-relay-deploy-helpers' import { gcRelayNativeDepsCache } from './ssh-relay-native-deps-cache-gc' import { gcRemoteRipgrepCache } from './ssh-relay-ripgrep-cache-gc' +import { REMOTE_INSTALL_ORDER_OK } from './remote-install-previous-version' import { getRemoteHostPlatform } from './ssh-remote-platform' // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock. @@ -41,8 +42,17 @@ function collectRelayVersions(): Promise { return gcOldRelayVersions(conn, home, currentDir, host, { nativeDepsCacheKeys: [nativeKey] }) } +// ddd is the previous build, so it is pinned and never probed. +const installOrder = [ + `${home}/.orca-remote/relay-0.1.0+bbb/.install-complete`, + `${home}/.orca-remote/relay-0.1.0+ddd/.install-complete`, + `${home}/.orca-remote/relay-0.1.0+aaa/.install-complete`, + REMOTE_INSTALL_ORDER_OK +].join('\n') + const versionSteps = [ - ['listing', 'relay-0.1.0+aaa\nrelay-0.1.0+ccc'], + ['listing', 'relay-0.1.0+aaa\nrelay-0.1.0+ddd'], + ['previous install order', installOrder], ['install lock probe', 'OPEN'], ['completion probe', 'COMPLETE'], ['liveness probe', 'DEAD'], @@ -73,8 +83,8 @@ describe('version GC termination', () => { 'stops after an unconfirmed stale lock probe with claim held: %s', async (claimed) => { const replies = claimed - ? versionSteps.slice(0, 6).map(([, reply]) => String(reply)) - : [versionSteps[0][1]] + ? versionSteps.slice(0, 7).map(([, reply]) => String(reply)) + : versionSteps.slice(0, 2).map(([, reply]) => String(reply)) const error = failAfter([...replies, 'LOCKED']) await expect(collectRelayVersions()).rejects.toBe(error) diff --git a/src/main/ssh/remote-install-gc.ts b/src/main/ssh/remote-install-gc.ts index 986154e356e..532a350681a 100644 --- a/src/main/ssh/remote-install-gc.ts +++ b/src/main/ssh/remote-install-gc.ts @@ -28,7 +28,6 @@ import { MAX_RELAY_GC_LISTING_ENTRIES, moveRemoteTreeCommand, probeFileExistsCommand, - relayLivenessProbeCommand, removeRemoteTreeCommand } from './ssh-remote-commands' import { @@ -39,7 +38,13 @@ import { type RemoteHostPlatform } from './ssh-remote-platform' import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' +import type { RelayEndpointVerdict } from './ssh-relay-endpoint-incumbent' import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' +import { findPreviousRemoteInstall } from './remote-install-previous-version' +import { + parseRelayVersionDirLiveness, + relayVersionDirLivenessCommand +} from './relay-version-dir-liveness' // Legacy relay dirs predate `.install-complete`; they need a liveness-only GC check so they // eventually drain. There is no orcad equivalent — orcad has never shipped without one. @@ -68,13 +73,19 @@ export type RemoteInstallGcOptions = { * the rollback target, and GC'ing it turns a recoverable bad update into a re-deploy. */ pinnedDirNames?: readonly string[] + /** + * More pins, resolved only once a candidate exists. Null means the host could not say which + * directories to keep, so this pass deletes nothing. + */ + resolveExtraPinnedDirNames?: () => Promise } /** * Garbage-collect one model's old version directories. * - * **GC ownership (design §06 falsifier 1):** a pass only ever sees, and only ever deletes, - * directories belonging to `model`. The remote listing is scoped by prefix, and + * **GC ownership (design D10; to be tracked in docs/reference/remote-server-install-model.md):** + * a pass only ever sees, and only ever deletes, directories belonging to `model`. The remote + * listing is scoped by prefix, and * `remoteInstallGcPermits` re-checks every candidate locally, so neither a widened glob nor * a hand-rolled listing can make one model delete the other's live install. */ @@ -121,10 +132,17 @@ export async function gcOldRemoteInstallVersions( if (candidates.length === 0) { return } + const extraPins = options.resolveExtraPinnedDirNames + ? await options.resolveExtraPinnedDirNames() + : [] + if (!extraPins) { + return + } + const survivors = candidates.filter((name) => !extraPins.includes(name)) const removed: string[] = [] const kept: string[] = [] - for (const name of candidates) { + for (const name of survivors) { const dir = joinRemotePath(host, baseDir, name) try { const safe = await isCandidateSafeToRemove(conn, model, dir, name, host, options) @@ -249,8 +267,8 @@ async function isCandidateSafeToRemove( } /** - * The relay's GC, bound to its own namespace and its own liveness probe (a live unix socket - * or Windows pipe inside the version dir). + * The relay's GC, bound to its own namespace. A version dir goes only on an `exited` verdict + * (relay-version-dir-liveness.ts), and the previous completed build is pinned (design D5). */ export async function gcOldRelayVersions( conn: SshConnection, @@ -260,6 +278,8 @@ export async function gcOldRelayVersions( options?: { windowsNodePath?: string windowsSockNames?: string[] + /** Host Node that runs the connect probe once a recorded relay PID is dead. */ + nodePath?: string /** * Cache entries this connection depends on, whether or not it links to them. Also the gate: * a caller that could not compute a key is not using the shared-cache model on this host, and @@ -270,7 +290,22 @@ export async function gcOldRelayVersions( ): Promise { await gcOldRemoteInstallVersions(conn, RELAY_INSTALL_MODEL, remoteHome, currentDirAbsPath, host, { ...options, - isDirLive: (dir) => hasLiveRelaySocket(conn, dir, host, options) + resolveExtraPinnedDirNames: async () => { + const previous = await findPreviousRemoteInstall( + conn, + host, + joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR), + RELAY_INSTALL_MODEL, + remoteBasename(currentDirAbsPath, host) + ) + // Why null rather than a guess: without the order, any candidate could be the previous build. + if (previous.state !== 'ok') { + return null + } + return previous.dirName ? [previous.dirName] : [] + }, + isDirLive: async (dir) => + (await probeRelayVersionDirLiveness(conn, dir, host, options)) !== 'exited' }) // Why after and not before: version-dir removal is what turns a cache entry unreferenced, so // running it second lets one pass reclaim both instead of leaving the tree for the next connect. @@ -285,18 +320,19 @@ export async function gcOldRelayVersions( } } -async function hasLiveRelaySocket( +/** GC deletes a relay version dir only on `exited`; see relay-version-dir-liveness.ts. */ +export async function probeRelayVersionDirLiveness( conn: SshConnection, dir: string, host: RemoteHostPlatform = DEFAULT_REMOTE_HOST, options?: { windowsNodePath?: string windowsSockNames?: string[] + nodePath?: string } -): Promise { +): Promise { try { - // Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle. - const windowsOptions = + const windows = isWindowsRemoteHost(host) && options?.windowsNodePath ? { nodePath: options.windowsNodePath, @@ -308,15 +344,14 @@ async function hasLiveRelaySocket( const out = await execHostCommand( conn, host, - relayLivenessProbeCommand(host, dir, windowsOptions) + relayVersionDirLivenessCommand(host, dir, { nodePath: options?.nodePath, windows }) ) - const state = out.trim() - return state !== 'DEAD' && state !== 'WAITING' + return parseRelayVersionDirLiveness(out) } catch (err) { if (isUnconfirmedSshCommandTermination(err)) { throw err } - // Why: an inconclusive liveness probe must never authorize deletion. - return true + // Why: an unanswered probe observes nothing; it never authorizes deletion. + return 'unverifiable' } } diff --git a/src/main/ssh/remote-install-model.test.ts b/src/main/ssh/remote-install-model.test.ts index 4768a7bf0d0..20d2303aeb9 100644 --- a/src/main/ssh/remote-install-model.test.ts +++ b/src/main/ssh/remote-install-model.test.ts @@ -4,7 +4,10 @@ import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { describe, expect, it } from 'vitest' import { orcadRipgrepArtifact } from '../../shared/orcad-artifacts' -import { probeRemoteInstallCompleteCommand } from './ssh-remote-commands' +import { + listRemoteInstallBaseDirsCommand, + probeRemoteInstallCompleteCommand +} from './ssh-remote-commands' import { getRemoteHostPlatform } from './ssh-remote-platform' import { @@ -120,4 +123,35 @@ describe('GC ownership — each model collects only its own namespace', () => { expect(inventory.orcad).toEqual(ORCAD_DIRS) expect(inventory.unknown).toEqual(['something-else']) }) + + it('gives the shared runtime store its own owner that no version-dir GC may take', () => { + expect(remoteInstallDirOwner('runtimes')).toBe('runtimes') + expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, 'runtimes')).toBe(false) + expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'runtimes')).toBe(false) + expect(inventoryRemoteInstallDirs(['runtimes', ...RELAY_DIRS]).runtimes).toEqual(['runtimes']) + }) + + it.skipIf(process.platform === 'win32')( + 'keeps runtimes/ out of every model listing, including older clients’ prefix scans', + () => { + const base = mkdtempSync(join(tmpdir(), 'install-listing-')) + try { + for (const name of ['runtimes', 'relay-0.1.0+aa', 'orcad-0.1.0+aa']) { + mkdirSync(join(base, name)) + } + mkdirSync(join(base, 'runtimes', `node-${'a'.repeat(64)}`)) + const host = getRemoteHostPlatform('linux-x64') + for (const model of [RELAY_INSTALL_MODEL, ORCAD_INSTALL_MODEL]) { + const listed = execFileSync( + '/bin/sh', + ['-c', listRemoteInstallBaseDirsCommand(host, base, model)], + { encoding: 'utf8' } + ) + expect(listed.trim().split('\n')).toEqual([`${model.dirPrefix}-0.1.0+aa`]) + } + } finally { + rmSync(base, { recursive: true, force: true }) + } + } + ) }) diff --git a/src/main/ssh/remote-install-model.ts b/src/main/ssh/remote-install-model.ts index b98cfd258c1..f547c02dd10 100644 --- a/src/main/ssh/remote-install-model.ts +++ b/src/main/ssh/remote-install-model.ts @@ -1,11 +1,13 @@ /** - * The two things Orca installs into `~/.orca-remote/`, and the rules that keep them from + * The things Orca installs into `~/.orca-remote/`, and the rules that keep them from * touching each other. * - * `docs/design/shipping-orcad.html` §06 settles that on-disk coexistence is permanent: the - * relay is the dumb execution host for SSH-target users, orcad is the peer for paired - * environments, and no plan item retires either. So `relay-/` and `orcad-/` - * sit side by side forever, and the namespace has to be a parameter rather than a literal. + * Design D10 (to be tracked in docs/reference/remote-server-install-model.md): the relay and + * orcad converge into one server package; on the host that is `server-/` plus the + * shared `runtimes/node-/` store. Legacy `relay-*` / `orcad-*` directories belong to a + * migration sweep that deletes only on an `exited` verdict, handed over in two releases: this + * one lists them as diagnostics only. Until then each is its own namespace, so the prefix has + * to be a parameter rather than a literal. * * GC ownership is the trap that parameterization creates. Each model garbage-collects ONLY * its own directories — see `remoteInstallDirOwner`. Relay's regex happened to be narrow @@ -21,6 +23,7 @@ import { import { orcadArtifactFilenames, ORCAD_INSTALL_COMPLETE_FILENAME, + ORCAD_RUNTIMES_DIRNAME, ORCAD_VERSION_FILENAME } from '../../shared/orcad-artifacts' import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' @@ -104,15 +107,18 @@ export function remoteInstallListingRegexSource(model: RemoteInstallModel): stri } /** - * Which model owns a directory found in `~/.orca-remote/`, or null for anything neither - * model created. + * Which owner a directory found in `~/.orca-remote/` belongs to, or null for anything no + * owner created. * - * This is the answer to §06 falsifier 1's first half: **the model that created a directory - * owns it, and nothing else may delete it.** A relay GC pass that saw `orcad-0.1.0+abc` - * would be looking at the live install of a peer whose lifecycle it has no view into — the - * SSH-execution-boundary collapse in directory form. + * Design D10: **the model that created a directory owns it, and nothing else may delete it** + * (amended only in the release after the two-step hand-over). A relay GC pass that saw + * `orcad-0.1.0+abc` would be looking at the live install of a peer whose lifecycle it has no + * view into — the SSH-execution-boundary collapse in directory form. */ -export function remoteInstallDirOwner(dirName: string): RemoteInstallModelId | null { +export function remoteInstallDirOwner(dirName: string): RemoteInstallDirOwner | null { + if (dirName === ORCAD_RUNTIMES_DIRNAME) { + return 'runtimes' + } for (const model of REMOTE_INSTALL_MODELS) { if (new RegExp(remoteInstallListingRegexSource(model)).test(dirName)) { return model.id @@ -126,11 +132,17 @@ export function remoteInstallGcPermits(model: RemoteInstallModel, dirName: strin return remoteInstallDirOwner(dirName) === model.id } -export type RemoteInstallInventory = Record +/** + * `runtimes` is the shared Node store (design D5). No version-dir model owns it, so neither + * model's GC can list or delete it; only `remote-node-runtime-store-gc.ts` collects inside it. + */ +export type RemoteInstallDirOwner = RemoteInstallModelId | 'runtimes' + +export type RemoteInstallInventory = Record /** Group a raw `~/.orca-remote/` listing by owning model, for diagnostics and the client's choice. */ export function inventoryRemoteInstallDirs(dirNames: readonly string[]): RemoteInstallInventory { - const inventory: RemoteInstallInventory = { relay: [], orcad: [], unknown: [] } + const inventory: RemoteInstallInventory = { relay: [], orcad: [], runtimes: [], unknown: [] } for (const name of dirNames) { const owner = remoteInstallDirOwner(name) if (owner) { diff --git a/src/main/ssh/remote-install-previous-version.test.ts b/src/main/ssh/remote-install-previous-version.test.ts new file mode 100644 index 00000000000..e64fdb72ede --- /dev/null +++ b/src/main/ssh/remote-install-previous-version.test.ts @@ -0,0 +1,160 @@ +import { execFileSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) + +import type { SshConnection } from './ssh-connection' +import { gcOldRelayVersions } from './remote-install-gc' +import { ORCAD_INSTALL_MODEL, RELAY_INSTALL_MODEL } from './remote-install-model' +import { + listCompletedInstallsNewestFirstCommand, + parseCompletedInstallsNewestFirst, + REMOTE_INSTALL_ORDER_OK +} from './remote-install-previous-version' +import { execCommand } from './ssh-relay-deploy-helpers' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock. +const conn = {} as SshConnection +const host = getRemoteHostPlatform('linux-x64') +const mockExec = vi.mocked(execCommand) + +describe('parseCompletedInstallsNewestFirst', () => { + it('keeps only this model’s version dirs, in host order', () => { + const output = [ + '/h/.orca-remote/relay-0.2.0+bbb/.install-complete', + '/h/.orca-remote/orcad-0.2.0+bbb/.install-complete', + '/h/.orca-remote/relay-0.1.0+aaa/.install-complete', + REMOTE_INSTALL_ORDER_OK + ].join('\n') + expect(parseCompletedInstallsNewestFirst(output, RELAY_INSTALL_MODEL)).toEqual([ + 'relay-0.2.0+bbb', + 'relay-0.1.0+aaa' + ]) + }) + + it('is null when the host did not finish the listing', () => { + expect( + parseCompletedInstallsNewestFirst( + '/h/.orca-remote/relay-0.1.0+aaa/.install-complete', + RELAY_INSTALL_MODEL + ) + ).toBeNull() + }) +}) + +const posixOnly = process.platform === 'win32' ? describe.skip : describe + +posixOnly('listCompletedInstallsNewestFirstCommand (real shell)', () => { + let base: string + beforeEach(() => { + base = mkdtempSync(join(tmpdir(), 'install-order-')) + }) + afterEach(() => { + rmSync(base, { recursive: true, force: true }) + }) + + function install(name: string, mtimeSeconds: number, complete = true): void { + mkdirSync(join(base, name)) + if (complete) { + const marker = join(base, name, '.install-complete') + writeFileSync(marker, '') + utimesSync(marker, mtimeSeconds, mtimeSeconds) + } + } + + function run(): string[] | null { + const out = execFileSync( + '/bin/sh', + ['-c', listCompletedInstallsNewestFirstCommand(host, base, RELAY_INSTALL_MODEL)], + { encoding: 'utf8' } + ) + return parseCompletedInstallsNewestFirst(out, RELAY_INSTALL_MODEL) + } + + it('orders completed installs by marker mtime and skips torn ones', () => { + install('relay-0.1.0+aaa', 1_000) + install('relay-0.3.0+ccc', 3_000) + install('relay-0.2.0+bbb', 2_000) + install('relay-0.4.0+ddd', 4_000, false) + install('orcad-0.9.0+eee', 9_000) + + expect(run()).toEqual(['relay-0.3.0+ccc', 'relay-0.2.0+bbb', 'relay-0.1.0+aaa']) + }) + + it('answers an empty order for a base without completed installs', () => { + expect(run()).toEqual([]) + }) + + it('is scoped to the model prefix', () => { + install('orcad-0.9.0+eee', 9_000) + const out = execFileSync( + '/bin/sh', + ['-c', listCompletedInstallsNewestFirstCommand(host, base, ORCAD_INSTALL_MODEL)], + { encoding: 'utf8' } + ) + expect(parseCompletedInstallsNewestFirst(out, ORCAD_INSTALL_MODEL)).toEqual(['orcad-0.9.0+eee']) + }) +}) + +describe('relay GC keeps the previous build', () => { + beforeEach(() => { + mockExec.mockReset() + mockExec.mockResolvedValue('') + }) + + it('never probes or removes the most recent other completed install', async () => { + mockExec + .mockResolvedValueOnce('relay-0.1.0+aaa\n') + .mockResolvedValueOnce( + [ + '/home/u/.orca-remote/relay-0.2.0+bbb/.install-complete', + '/home/u/.orca-remote/relay-0.1.0+aaa/.install-complete', + REMOTE_INSTALL_ORDER_OK + ].join('\n') + ) + + await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host) + + expect(mockExec).toHaveBeenCalledTimes(2) + }) + + it('deletes nothing when the host cannot report install order', async () => { + mockExec + .mockResolvedValueOnce('relay-0.1.0+aaa\nrelay-0.0.9+zzz\n') + .mockRejectedValueOnce( + Object.assign(new Error('ls failed'), { sshChannelCloseConfirmed: true }) + ) + + await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host) + + expect(mockExec).toHaveBeenCalledTimes(2) + }) + + it('keeps an old build whose liveness probe times out', async () => { + mockExec + .mockResolvedValueOnce('relay-0.1.0+aaa\n') + .mockResolvedValueOnce(`relay-0.1.5+fff\n${REMOTE_INSTALL_ORDER_OK}`) + .mockResolvedValueOnce('OPEN') + .mockResolvedValueOnce('COMPLETE') + .mockRejectedValueOnce( + Object.assign(new Error('SSH command timed out'), { sshChannelCloseConfirmed: true }) + ) + + await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host, { + nodePath: '/usr/bin/node' + }) + + const commands = mockExec.mock.calls.map(([, command]) => command) + expect(commands[4]).toContain('.relay-pid') + expect(commands.some((command) => command.includes('gc-claim'))).toBe(false) + expect(mockExec).toHaveBeenCalledTimes(5) + }) +}) diff --git a/src/main/ssh/remote-install-previous-version.ts b/src/main/ssh/remote-install-previous-version.ts new file mode 100644 index 00000000000..e9b2b92f683 --- /dev/null +++ b/src/main/ssh/remote-install-previous-version.ts @@ -0,0 +1,106 @@ +/** + * The version dir a model ran before the current one, pinned against GC like orcad's + * rollback target (design D5 "plus the relay's previous version as a pin"). + * + * The relay keeps no activation record, so "previous" is the most recently completed install + * of the same model other than the current one, by `.install-complete` mtime. + */ +import type { SshConnection } from './ssh-connection' +import { shellEscape } from './ssh-connection-utils' +import { execCommand } from './ssh-relay-deploy-helpers' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' +import { + remoteInstallGcPermits, + remoteInstallVersionDirRegex, + type RemoteInstallModel +} from './remote-install-model' +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' +import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell' + +export const REMOTE_INSTALL_ORDER_OK = 'ORCA_INSTALL_ORDER_OK' + +export function listCompletedInstallsNewestFirstCommand( + host: RemoteHostPlatform, + baseDir: string, + model: RemoteInstallModel +): string { + const prefix = `${model.dirPrefix}-` + if (isWindowsRemoteHost(host)) { + return powerShellCommand( + [ + "$ErrorActionPreference = 'Stop'", + `$base = ${powerShellLiteral(baseDir)}`, + 'if (Test-Path -LiteralPath $base -PathType Container) {', + `Get-ChildItem -LiteralPath $base -Directory -Filter '${prefix}*' | ForEach-Object { ` + + `$marker = Join-Path $_.FullName ${powerShellLiteral(model.installCompleteFilename)}; ` + + 'if (Test-Path -LiteralPath $marker -PathType Leaf) { Get-Item -LiteralPath $marker } ' + + '} | Sort-Object LastWriteTimeUtc -Descending | ForEach-Object { $_.Directory.Name }', + '}', + `'${REMOTE_INSTALL_ORDER_OK}'` + ].join('\n') + ) + } + return [ + `base=${shellEscape(baseDir)}`, + `set -- "$base"/${prefix}*/${shellEscape(model.installCompleteFilename)}`, + `[ -e "$1" ] || { echo ${REMOTE_INSTALL_ORDER_OK}; exit 0; }`, + 'ls -1t -- "$@" || exit 1', + `echo ${REMOTE_INSTALL_ORDER_OK}` + ].join('\n') +} + +/** Dir names newest first, or null when the host could not answer. */ +export function parseCompletedInstallsNewestFirst( + output: string, + model: RemoteInstallModel +): string[] | null { + const lines = output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + if (!lines.includes(REMOTE_INSTALL_ORDER_OK)) { + return null + } + const versionDirRegex = remoteInstallVersionDirRegex(model) + const names: string[] = [] + for (const line of lines) { + // POSIX prints `//.install-complete`; PowerShell prints ``. + const segments = line.split(/[\\/]/) + const name = segments.length > 1 ? segments.at(-2) : segments[0] + if (name && versionDirRegex.test(name) && remoteInstallGcPermits(model, name)) { + names.push(name) + } + } + return names +} + +export type PreviousInstallResult = { state: 'ok'; dirName: string | null } | { state: 'unknown' } + +export async function findPreviousRemoteInstall( + conn: SshConnection, + host: RemoteHostPlatform, + baseDir: string, + model: RemoteInstallModel, + currentDirName: string +): Promise { + let output: string + try { + output = await execCommand( + conn, + listCompletedInstallsNewestFirstCommand(host, baseDir, model), + { + wrapCommand: host.commandDialect !== 'powershell' + } + ) + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + throw err + } + return { state: 'unknown' } + } + const ordered = parseCompletedInstallsNewestFirst(output, model) + if (!ordered) { + return { state: 'unknown' } + } + return { state: 'ok', dirName: ordered.find((name) => name !== currentDirName) ?? null } +} diff --git a/src/main/ssh/remote-node-runtime-store-gc.test.ts b/src/main/ssh/remote-node-runtime-store-gc.test.ts new file mode 100644 index 00000000000..66f05811e53 --- /dev/null +++ b/src/main/ssh/remote-node-runtime-store-gc.test.ts @@ -0,0 +1,279 @@ +import { execFileSync, spawn, type ChildProcess } from 'node:child_process' +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { once } from 'node:events' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) + +import type { SshConnection } from './ssh-connection' +import { gcRemoteNodeRuntimeStore, planRuntimeStoreGc } from './remote-node-runtime-store-gc' +import { + parseRuntimeStoreInventory, + RUNTIME_REF_NODE_PREFIX, + runtimeStoreInventoryCommand, + type RuntimeStoreInventory +} from './remote-node-runtime-store-inventory' +import { execCommand } from './ssh-relay-deploy-helpers' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock. +const conn = {} as SshConnection +const host = getRemoteHostPlatform('linux-x64') +const mockExec = vi.mocked(execCommand) +const sha = (c: string): string => c.repeat(64) + +function inventory(overrides: Partial = {}): RuntimeStoreInventory { + return { + entries: [], + verifiedNewestFirst: [], + referenced: new Set(), + held: new Set(), + processCheckRan: true, + dirNames: [], + ...overrides + } +} + +describe('planRuntimeStoreGc', () => { + const all = ['a', 'b', 'c', 'd'].map((c) => `node-${sha(c)}`) + + it('keeps the current pin and the newest other verified runtime', () => { + const plan = planRuntimeStoreGc( + inventory({ entries: all, verifiedNewestFirst: [all[0], all[1], all[2], all[3]] }), + [sha('b')] + ) + expect(plan.remove).toEqual([all[2], all[3]]) + expect(plan.kept).toEqual([all[0], all[1]]) + }) + + it('keeps referenced, process-held and unverified runtimes', () => { + const plan = planRuntimeStoreGc( + inventory({ + entries: all, + verifiedNewestFirst: [all[0], all[1], all[2]], + referenced: new Set([sha('b')]), + held: new Set([sha('c')]) + }), + [sha('a')] + ) + expect(plan.remove).toEqual([]) + }) + + it('keeps everything when no process check could run', () => { + const plan = planRuntimeStoreGc( + inventory({ entries: all, verifiedNewestFirst: all, processCheckRan: false }), + [sha('a')] + ) + expect(plan.remove).toEqual([]) + }) + + it('purges only abandoned tombstones of unwanted runtimes', () => { + const now = 10 * 60 * 60_000 + const old = `.gc-tombstone-node-${sha('c')}.7.${now - 31 * 60_000}` + const fresh = `.gc-tombstone-node-${sha('d')}.7.${now - 60_000}` + const referenced = `.gc-tombstone-node-${sha('e')}.7.${now - 31 * 60_000}` + const plan = planRuntimeStoreGc( + inventory({ entries: [old, fresh, referenced], referenced: new Set([sha('e')]) }), + [sha('a')], + now + ) + expect(plan.purgeTombstones).toEqual([old]) + }) +}) + +describe('parseRuntimeStoreInventory', () => { + it('rejects a partial listing, a reference error, and an unattributable reference', () => { + expect(parseRuntimeStoreInventory('ENTRY node-x')).toBeNull() + expect( + parseRuntimeStoreInventory('__ORCA_RUNTIME_STORE__REFS_ERR\n__ORCA_RUNTIME_STORE__OK') + ).toBeNull() + expect(parseRuntimeStoreInventory('REF garbage\n__ORCA_RUNTIME_STORE__OK')).toBeNull() + }) + + it('attributes process holds by runtime path, including renamed tombstones', () => { + const parsed = parseRuntimeStoreInventory( + [ + 'PROCESS_CHECK ps', + `HOLD /h/.orca-remote/runtimes/node-${sha('a')}/bin/node server.js`, + `HOLD /h/.orca-remote/runtimes/.gc-tombstone-node-${sha('b')}.1.2/bin/node`, + 'HOLD grep -F -- /h/.orca-remote/runtimes/', + '__ORCA_RUNTIME_STORE__OK' + ].join('\n') + ) + expect(parsed?.held).toEqual(new Set([sha('a'), sha('b')])) + }) +}) + +const posixOnly = process.platform === 'win32' ? describe.skip : describe + +posixOnly('gcRemoteNodeRuntimeStore (real shell)', () => { + let home: string + let root: string + let running: ChildProcess | null = null + + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'runtime-store-')) + root = join(home, '.orca-remote') + mkdirSync(join(root, 'runtimes'), { recursive: true }) + mockExec.mockReset() + mockExec.mockImplementation(async (_conn, command) => + execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' }) + ) + }) + afterEach(() => { + running?.kill('SIGKILL') + running = null + rmSync(home, { recursive: true, force: true }) + }) + + // `b` is always verified last, so it is the previous pin the store keeps. + function runtime(c: string, verified = true): string { + const dir = join(root, 'runtimes', `node-${sha(c)}`) + mkdirSync(join(dir, 'bin'), { recursive: true }) + writeFileSync(join(dir, 'bin', 'node'), '#!/bin/sh\nsleep 30\n') + chmodSync(join(dir, 'bin', 'node'), 0o755) + if (verified) { + writeFileSync(join(dir, '.verified'), '') + const at = c === 'b' ? 2_000_000 : 1_000_000 + utimesSync(join(dir, '.verified'), at, at) + } + return dir + } + + function versionDir(name: string): string { + const dir = join(root, name) + mkdirSync(dir, { recursive: true }) + return dir + } + + it('removes only unreferenced, unpinned, idle, verified runtimes', async () => { + runtime('a') // current pin + runtime('c') // referenced by an orcad slot marker + runtime('d') // referenced by a relay ref file + runtime('e') // unreferenced: collected + const running_ = runtime('f') // executing + runtime('9', false) // mid-promotion + runtime('b') // newest other verified: the previous pin + writeFileSync(join(versionDir('orcad-0.1.0+aaa'), '.runtime-node'), `${sha('c')}\n`) + writeFileSync(join(versionDir('relay-0.1.0+aaa'), `${RUNTIME_REF_NODE_PREFIX}${sha('d')}`), '') + running = spawn(join(running_, 'bin', 'node'), [], { stdio: 'ignore' }) + await once(running, 'spawn') + + const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] }) + + expect(result).toMatchObject({ + state: 'collected', + removed: [`node-${sha('e')}`], + legacyDirs: expect.arrayContaining(['orcad-0.1.0+aaa', 'relay-0.1.0+aaa']) + }) + for (const kept of ['a', 'b', 'c', 'd', 'f', '9']) { + expect(existsSync(join(root, 'runtimes', `node-${sha(kept)}`))).toBe(true) + } + expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`))).toBe(false) + // D10 two-step: legacy dirs are reported, never deleted by this pass. + expect(existsSync(join(root, 'orcad-0.1.0+aaa'))).toBe(true) + expect(existsSync(join(root, 'relay-0.1.0+aaa'))).toBe(true) + }) + + it('holds a runtime a process runs through another spelling of a symlinked home', async () => { + runtime('a') + runtime('b') + const held = runtime('e') + const alias = mkdtempSync(join(tmpdir(), 'runtime-store-alias-')) + rmSync(alias, { recursive: true }) + symlinkSync(home, alias) + try { + running = spawn(join(held, 'bin', 'node'), [], { stdio: 'ignore' }) + await once(running, 'spawn') + + const result = await gcRemoteNodeRuntimeStore(conn, host, alias, { currentPins: [sha('a')] }) + + expect(result).toMatchObject({ state: 'collected', removed: [] }) + expect(existsSync(join(held, 'bin', 'node'))).toBe(true) + } finally { + rmSync(alias, { force: true }) + } + }) + + it('keeps every runtime when a reference marker cannot be attributed', async () => { + runtime('a') + runtime('b') + runtime('e') + writeFileSync(join(versionDir('server-0.2.0+ccc'), '.runtime-node'), 'not-a-sha\n') + + const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] }) + + expect(result.state).toBe('skipped') + expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`))).toBe(true) + }) + + it('restores a runtime that gained a reference after the rename', async () => { + runtime('a') + runtime('b') + runtime('e') + let inventories = 0 + mockExec.mockImplementation(async (_conn, command) => { + if (command.includes('__ORCA_RUNTIME_STORE__OK') && ++inventories === 2) { + writeFileSync(join(versionDir('orcad-0.3.0+ddd'), '.runtime-node'), `${sha('e')}\n`) + } + return execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' }) + }) + + const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] }) + + expect(result).toMatchObject({ state: 'collected', removed: [] }) + expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`, 'bin', 'node'))).toBe(true) + }) + + it('is inert when the store does not exist', async () => { + rmSync(join(root, 'runtimes'), { recursive: true }) + const out = execFileSync('/bin/sh', ['-c', runtimeStoreInventoryCommand(host, home)], { + encoding: 'utf8' + }) + expect(parseRuntimeStoreInventory(out)?.entries).toEqual([]) + }) +}) + +describe('gcRemoteNodeRuntimeStore termination', () => { + beforeEach(() => { + mockExec.mockReset() + }) + + it('rethrows an unconfirmed inventory termination', async () => { + const error = Object.assign(new Error('SSH command timed out'), { + sshChannelCloseConfirmed: false + }) + mockExec.mockRejectedValueOnce(error) + await expect( + gcRemoteNodeRuntimeStore(conn, host, '/home/u', { currentPins: [sha('a')] }) + ).rejects.toBe(error) + }) + + it('skips Windows hosts without running anything', async () => { + const result = await gcRemoteNodeRuntimeStore( + conn, + getRemoteHostPlatform('win32-x64'), + 'C:/Users/u', + { + currentPins: [sha('a')] + } + ) + expect(result.state).toBe('skipped') + expect(mockExec).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/remote-node-runtime-store-gc.ts b/src/main/ssh/remote-node-runtime-store-gc.ts new file mode 100644 index 00000000000..24ec70c12ef --- /dev/null +++ b/src/main/ssh/remote-node-runtime-store-gc.ts @@ -0,0 +1,249 @@ +/** + * Collects the shared `~/.orca-remote/runtimes/node-/` store (design D5 GC). + * + * A runtime is deleted only when all of these hold: no retained directory references it + * (`.runtime-node` or `.runtime-ref-node-`), it is neither a pin this client runs nor the + * newest other verified runtime (keep two), and a process check ran and found nothing executing + * from it. Process evidence can only add holds; a scan that could not run keeps everything. + * + * Legacy `relay-*` / `orcad-*` directories are read for references and reported as + * diagnostics, never deleted here (design D10 two-step hand-over). + */ +import { randomInt } from 'node:crypto' +import { ORCAD_RUNTIMES_DIRNAME } from '../../shared/orcad-artifacts' +import type { SshConnection } from './ssh-connection' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { inventoryRemoteInstallDirs } from './remote-install-model' +import { + parseRuntimeStoreInventory, + RUNTIME_STORE_ENTRY_NAME, + RUNTIME_STORE_TOMBSTONE_NAME, + RUNTIME_STORE_TOMBSTONE_PREFIX, + runtimeStoreInventoryCommand, + type RuntimeStoreInventory +} from './remote-node-runtime-store-inventory' +import { execCommand } from './ssh-relay-deploy-helpers' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' +import { + moveRemoteTreeCommand, + removeRemoteTreeCommand, + restoreRemoteTreeCommand +} from './ssh-remote-commands' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +const MAX_REMOVALS_PER_PASS = 8 +const ABANDONED_TOMBSTONE_MS = 30 * 60_000 + +export type RuntimeStoreGcPlan = { + /** `node-` entries to rename away and delete. */ + remove: string[] + /** Abandoned tombstones whose runtime is still unwanted. */ + purgeTombstones: string[] + kept: string[] +} + +/** Why a sha must stay, or null when nothing holds it. */ +function holdReason( + sha: string, + inventory: RuntimeStoreInventory, + pins: ReadonlySet +): string | null { + if (!inventory.processCheckRan) { + return 'process check unavailable' + } + if (pins.has(sha)) { + return 'pinned' + } + if (inventory.referenced.has(sha)) { + return 'referenced' + } + if (inventory.held.has(sha)) { + return 'in use by a process' + } + return null +} + +export function planRuntimeStoreGc( + inventory: RuntimeStoreInventory, + currentPins: readonly string[], + now: number = Date.now() +): RuntimeStoreGcPlan { + const pins = new Set(currentPins) + // Keep two: the pin this client runs and the newest other verified runtime (the previous pin). + const previous = inventory.verifiedNewestFirst + .map((name) => RUNTIME_STORE_ENTRY_NAME.exec(name)?.[1]) + .find((sha): sha is string => !!sha && !pins.has(sha)) + if (previous) { + pins.add(previous) + } + const verified = new Set(inventory.verifiedNewestFirst) + const plan: RuntimeStoreGcPlan = { remove: [], purgeTombstones: [], kept: [] } + for (const name of inventory.entries) { + const entry = RUNTIME_STORE_ENTRY_NAME.exec(name) + if (entry) { + // Unverified means mid-promotion or torn; the installer, not GC, owns that state. + const idle = verified.has(name) && holdReason(entry[1], inventory, pins) === null + if (idle && plan.remove.length < MAX_REMOVALS_PER_PASS) { + plan.remove.push(name) + } else { + plan.kept.push(name) + } + continue + } + const tombstone = RUNTIME_STORE_TOMBSTONE_NAME.exec(name) + if ( + tombstone && + now - Number(tombstone[2]) >= ABANDONED_TOMBSTONE_MS && + holdReason(tombstone[1], inventory, pins) === null + ) { + plan.purgeTombstones.push(name) + } + } + return plan +} + +export type RuntimeStoreGcResult = + | { state: 'skipped'; reason: string } + | { state: 'collected'; removed: string[]; kept: string[]; legacyDirs: string[] } + +function exec(conn: SshConnection, command: string, signal?: AbortSignal): Promise { + return execCommand(conn, command, { wrapCommand: true, signal }) +} + +async function readInventory( + conn: SshConnection, + host: RemoteHostPlatform, + remoteHome: string, + signal?: AbortSignal +): Promise { + try { + return parseRuntimeStoreInventory( + await exec(conn, runtimeStoreInventoryCommand(host, remoteHome), signal) + ) + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + throw err + } + return null + } +} + +/** + * One pass over the runtime store. Confirmed failures keep the runtime and end quietly; an + * unconfirmed SSH termination is rethrown so the caller stops its cleanup chain. + */ +export async function gcRemoteNodeRuntimeStore( + conn: SshConnection, + host: RemoteHostPlatform, + remoteHome: string, + options: { currentPins: readonly string[]; signal?: AbortSignal } +): Promise { + if (isWindowsRemoteHost(host)) { + return { state: 'skipped', reason: 'Windows hosts have no managed runtime store yet' } + } + const inventory = await readInventory(conn, host, remoteHome, options.signal) + if (!inventory) { + return { state: 'skipped', reason: 'runtime store inventory was unverifiable' } + } + const legacy = inventoryRemoteInstallDirs(inventory.dirNames) + const legacyDirs = [...legacy.relay, ...legacy.orcad] + const plan = planRuntimeStoreGc(inventory, options.currentPins) + const store = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_RUNTIMES_DIRNAME) + const removed: string[] = [] + const kept = [...plan.kept] + for (const name of plan.purgeTombstones) { + if (await removeTree(conn, host, joinRemotePath(host, store, name), options.signal)) { + removed.push(name) + } + } + for (const name of plan.remove) { + const sha = RUNTIME_STORE_ENTRY_NAME.exec(name)?.[1] ?? '' + const entryDir = joinRemotePath(host, store, name) + const tombstone = joinRemotePath( + host, + store, + `${RUNTIME_STORE_TOMBSTONE_PREFIX}${name}.${randomInt(1, 2 ** 47)}.${Date.now()}` + ) + if (!(await moveTree(conn, host, entryDir, tombstone, options.signal))) { + kept.push(name) + continue + } + // Why recheck after the rename: an installer that saw this runtime present may be writing + // its reference now; restoring is the only outcome that leaves its slot launchable. + const recheck = await readInventory(conn, host, remoteHome, options.signal).catch( + async (err: unknown) => { + await restoreTree(conn, host, tombstone, entryDir, options.signal).catch(() => {}) + throw err + } + ) + if (!recheck || holdReason(sha, recheck, new Set(options.currentPins)) !== null) { + await restoreTree(conn, host, tombstone, entryDir, options.signal) + kept.push(name) + continue + } + if (await removeTree(conn, host, tombstone, options.signal)) { + removed.push(name) + } else { + kept.push(name) + } + } + if (removed.length > 0) { + const legacyNote = + legacyDirs.length > 0 + ? `; legacy install dirs left for the migration sweep: ${legacyDirs.join(', ')}` + : '' + console.log(`[runtime-store] GC: removed ${removed.join(', ')}${legacyNote}`) + } + return { state: 'collected', removed, kept, legacyDirs } +} + +async function moveTree( + conn: SshConnection, + host: RemoteHostPlatform, + source: string, + destination: string, + signal?: AbortSignal +): Promise { + try { + return ( + (await exec(conn, moveRemoteTreeCommand(host, source, destination), signal)).trim() === + 'MOVED' + ) + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + throw err + } + return false + } +} + +async function restoreTree( + conn: SshConnection, + host: RemoteHostPlatform, + tombstone: string, + entryDir: string, + signal?: AbortSignal +): Promise { + await exec(conn, restoreRemoteTreeCommand(host, tombstone, entryDir), signal).catch((err) => { + if (isUnconfirmedSshCommandTermination(err)) { + throw err + } + }) +} + +async function removeTree( + conn: SshConnection, + host: RemoteHostPlatform, + path: string, + signal?: AbortSignal +): Promise { + try { + await exec(conn, removeRemoteTreeCommand(host, path), signal) + return true + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + throw err + } + return false + } +} diff --git a/src/main/ssh/remote-node-runtime-store-inventory.ts b/src/main/ssh/remote-node-runtime-store-inventory.ts new file mode 100644 index 00000000000..583ad230139 --- /dev/null +++ b/src/main/ssh/remote-node-runtime-store-inventory.ts @@ -0,0 +1,140 @@ +/** + * One read-only pass over `~/.orca-remote/` answering what the runtime store GC needs: store + * entries, which runtimes are referenced, verified order, and which a running process holds. + */ +import { + ORCAD_NODE_RUNTIME_DIR_PREFIX, + ORCAD_NODE_RUNTIME_MARKER_FILENAME, + ORCAD_RUNTIMES_DIRNAME +} from '../../shared/orcad-artifacts' +import { shellEscape } from './ssh-connection-utils' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +/** A version dir names a runtime it needs with an empty file of this prefix + sha (design D5). */ +export const RUNTIME_REF_NODE_PREFIX = '.runtime-ref-node-' +export const RUNTIME_STORE_TOMBSTONE_PREFIX = '.gc-tombstone-' +const INVENTORY_OK = '__ORCA_RUNTIME_STORE__OK' +const REFS_ERR = '__ORCA_RUNTIME_STORE__REFS_ERR' +const MAX_DIRS = 512 +const SHA256 = /^[0-9a-f]{64}$/ +export const RUNTIME_STORE_ENTRY_NAME = new RegExp( + `^${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})$` +) +export const RUNTIME_STORE_TOMBSTONE_NAME = new RegExp( + `^${RUNTIME_STORE_TOMBSTONE_PREFIX.replace(/\./g, '\\.')}${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})\\.[0-9]+\\.([0-9]+)$` +) +const HELD_PATH = new RegExp( + `/${ORCAD_RUNTIMES_DIRNAME}/(?:${RUNTIME_STORE_TOMBSTONE_PREFIX.replace(/\./g, '\\.')})?${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})[./]` +) + +export type RuntimeStoreInventory = { + /** Store entry names: `node-` and this GC's tombstones. */ + entries: string[] + /** `node-` names with `.verified`, newest first. */ + verifiedNewestFirst: string[] + referenced: Set + held: Set + /** False when neither `ps` nor `/proc` answered; nothing may then be called idle. */ + processCheckRan: boolean + /** Other `~/.orca-remote/` directory names, for legacy diagnostics. */ + dirNames: string[] +} + +export function runtimeStoreInventoryCommand(host: RemoteHostPlatform, remoteHome: string): string { + const root = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) + const refPrefix = RUNTIME_REF_NODE_PREFIX + return [ + `root=${shellEscape(root)}`, + `rt="$root"/${ORCAD_RUNTIMES_DIRNAME}`, + `[ -d "$rt" ] || { printf '%s\\n' ${INVENTORY_OK}; exit 0; }`, + `[ -r "$root" ] && [ -x "$root" ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`, + 'n=0', + // Why every sibling and not a prefix: a directory this client does not recognise may still + // be a newer Orca's install that names a runtime. + 'for d in "$root"/* "$root"/.[!.]*; do', + ' [ -d "$d" ] || continue', + ' name=${d##*/}', + ` [ "$name" = ${ORCAD_RUNTIMES_DIRNAME} ] && continue`, + ` [ -r "$d" ] && [ -x "$d" ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`, + ' n=$((n+1))', + ` [ "$n" -le ${MAX_DIRS} ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`, + ` printf 'DIR %s\\n' "$name"`, + ` if [ -e "$d"/${ORCAD_NODE_RUNTIME_MARKER_FILENAME} ]; then`, + ` sha=$(cat "$d"/${ORCAD_NODE_RUNTIME_MARKER_FILENAME}) || { printf '%s\\n' ${REFS_ERR}; exit 0; }`, + ` printf 'REF %s\\n' "$sha"`, + ' fi', + ` for f in "$d"/${refPrefix}*; do`, + ' [ -e "$f" ] || continue', + ` printf 'REF %s\\n' "\${f##*/${refPrefix}}"`, + ' done', + 'done', + `for e in "$rt"/${ORCAD_NODE_RUNTIME_DIR_PREFIX}* "$rt"/${RUNTIME_STORE_TOMBSTONE_PREFIX}${ORCAD_NODE_RUNTIME_DIR_PREFIX}*; do`, + ` [ -d "$e" ] && printf 'ENTRY %s\\n' "\${e##*/}"`, + 'done', + `set -- "$rt"/${ORCAD_NODE_RUNTIME_DIR_PREFIX}*/.verified`, + 'if [ -e "$1" ]; then', + ` order=$(ls -1t -- "$@") || { printf '%s\\n' ${REFS_ERR}; exit 0; }`, + ` printf '%s\\n' "$order" | while IFS= read -r v; do v=\${v%/.verified}; printf 'VERIFIED %s\\n' "\${v##*/}"; done`, + 'fi', + // Process checks only add holds, so an unmatched `grep` is not an error. Why not "$rt/": + // /proc exe and argv may name the store through another spelling of a symlinked home. + 'if ps_out=$(ps -e -o args= 2>/dev/null); then', + " printf 'PROCESS_CHECK ps\\n'", + ` printf '%s\\n' "$ps_out" | grep -F -- /${ORCAD_RUNTIMES_DIRNAME}/ | sed 's/^/HOLD /'`, + 'fi', + 'if [ -n "$(readlink /proc/self/exe 2>/dev/null)" ]; then', + " printf 'PROCESS_CHECK proc\\n'", + ' for p in /proc/[0-9]*/exe; do', + ' t=$(readlink "$p" 2>/dev/null) || continue', + ` case "$t" in */${ORCAD_RUNTIMES_DIRNAME}/*) printf 'HOLD %s\\n' "$t";; esac`, + ' done', + 'fi', + `printf '%s\\n' ${INVENTORY_OK}` + ].join('\n') +} + +/** Null when the host could not produce a complete inventory; that keeps every runtime. */ +export function parseRuntimeStoreInventory(output: string): RuntimeStoreInventory | null { + const lines = output.split(/\r?\n/).map((line) => line.trim()) + if (!lines.includes(INVENTORY_OK) || lines.includes(REFS_ERR)) { + return null + } + const inventory: RuntimeStoreInventory = { + entries: [], + verifiedNewestFirst: [], + referenced: new Set(), + held: new Set(), + processCheckRan: false, + dirNames: [] + } + for (const line of lines) { + const space = line.indexOf(' ') + const tag = space === -1 ? line : line.slice(0, space) + const value = space === -1 ? '' : line.slice(space + 1).trim() + if (tag === 'REF') { + // An unattributable reference could name any runtime, so it stops the pass. + if (!SHA256.test(value)) { + return null + } + inventory.referenced.add(value) + } else if ( + tag === 'ENTRY' && + (RUNTIME_STORE_ENTRY_NAME.test(value) || RUNTIME_STORE_TOMBSTONE_NAME.test(value)) + ) { + inventory.entries.push(value) + } else if (tag === 'VERIFIED' && RUNTIME_STORE_ENTRY_NAME.test(value)) { + inventory.verifiedNewestFirst.push(value) + } else if (tag === 'HOLD') { + const sha = HELD_PATH.exec(value)?.[1] + if (sha) { + inventory.held.add(sha) + } + } else if (tag === 'PROCESS_CHECK') { + inventory.processCheckRan = true + } else if (tag === 'DIR' && value) { + inventory.dirNames.push(value) + } + } + return inventory +} diff --git a/src/main/ssh/ssh-relay-cross-version-isolation.test.ts b/src/main/ssh/ssh-relay-cross-version-isolation.test.ts index be66463206a..f519dbab4fb 100644 --- a/src/main/ssh/ssh-relay-cross-version-isolation.test.ts +++ b/src/main/ssh/ssh-relay-cross-version-isolation.test.ts @@ -61,6 +61,7 @@ vi.mock('./ssh-connection-utils', () => ({ import { deployAndLaunchRelay } from './ssh-relay-deploy' import { execCommand } from './ssh-relay-deploy-helpers' import type { SshConnection } from './ssh-connection' +import { REMOTE_INSTALL_ORDER_OK } from './remote-install-previous-version' function makeMockConnection(): SshConnection { return { @@ -150,6 +151,12 @@ describe('cross-version isolation', () => { if (command.includes('test -S') && command.includes('echo ALIVE || echo DEAD')) { return Promise.resolve('DEAD') } + if (command.includes(REMOTE_INSTALL_ORDER_OK)) { + // A newer v0 is the previous build, so v1 is kept only by its live socket. + return Promise.resolve( + `relay-0.1.0+222222222222\nrelay-0.1.0+000000000000\nrelay-0.1.0+111111111111\n${REMOTE_INSTALL_ORDER_OK}` + ) + } if (command.includes('__ORCA_RELAY_GC_FIND_STATUS__')) { return Promise.resolve('relay-0.1.0+111111111111\nrelay-0.1.0+222222222222\n') } diff --git a/src/main/ssh/ssh-relay-deploy.ts b/src/main/ssh/ssh-relay-deploy.ts index 4424adc0c01..84e5f4cd0c4 100644 --- a/src/main/ssh/ssh-relay-deploy.ts +++ b/src/main/ssh/ssh-relay-deploy.ts @@ -693,6 +693,7 @@ async function deployAndLaunchRelayAttempt( gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, { windowsNodePath: launched.nodePath, windowsSockNames: [relaySocketNameForInstanceId(relayInstanceId)], + nodePath: launched.nodePath, // Why pin rather than rely on the symlink alone: a deploy that fell back to a // per-directory install has no reference to show, and its key must still survive. nativeDepsCacheKeys: [ diff --git a/src/main/ssh/ssh-relay-endpoint-incumbent.ts b/src/main/ssh/ssh-relay-endpoint-incumbent.ts index 343f92cadec..ee4d7d59eea 100644 --- a/src/main/ssh/ssh-relay-endpoint-incumbent.ts +++ b/src/main/ssh/ssh-relay-endpoint-incumbent.ts @@ -69,7 +69,7 @@ const PROBE_END = 'ORCA-INCUMBENT-END' const CONNECT_PROBE_TIMEOUT_MS = 1000 // Why ES5 syntax: nodePath may be a host-resolved system node, not the bundled one. -const CONNECT_PROBE_JS = [ +export const RELAY_CONNECT_PROBE_JS = [ 'var s=require("net").connect(process.argv[1]);', 'var done=false;', 'function say(v){if(done)return;done=true;try{s.destroy()}catch(e){};', @@ -100,7 +100,7 @@ export function relayEndpointIncumbentProbeCommand(nodePath: string, sockPath: s `printf '%s\\n' ${shellEscape(PROBE_BEGIN)}`, 'if [ -S "$sock" ]; then', " printf 'PRESENT=yes\\n'", - ` listen=$("$node" -e ${shellEscape(CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`, + ` listen=$("$node" -e ${shellEscape(RELAY_CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`, ' [ -n "$listen" ] || listen=unknown', 'else', " printf 'PRESENT=no\\n'", diff --git a/src/main/ssh/ssh-relay-versioned-install.test.ts b/src/main/ssh/ssh-relay-versioned-install.test.ts index 1778ac1affc..0dc2f428df9 100644 --- a/src/main/ssh/ssh-relay-versioned-install.test.ts +++ b/src/main/ssh/ssh-relay-versioned-install.test.ts @@ -15,6 +15,11 @@ vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +// The previous-build pin has its own tests; here it names a build that is never a candidate. +vi.mock('./remote-install-previous-version', () => ({ + findPreviousRemoteInstall: vi.fn().mockResolvedValue({ state: 'ok', dirName: 'relay-0.1.0+fff' }) +})) + import { existsSync, readFileSync } from 'node:fs' import { readLocalFullVersion, diff --git a/src/main/ssh/ssh-remote-commands.ts b/src/main/ssh/ssh-remote-commands.ts index 11e39921fc2..daa8715e260 100644 --- a/src/main/ssh/ssh-remote-commands.ts +++ b/src/main/ssh/ssh-remote-commands.ts @@ -201,7 +201,7 @@ export function probeFileExistsCommand(host: RemoteHostPlatform, remotePath: str ) } -type WindowsRelayLivenessOptions = { +export type WindowsRelayLivenessOptions = { nodePath: string pipePaths: string[] } diff --git a/src/relay/relay-daemon.ts b/src/relay/relay-daemon.ts index 27983176abf..649fde3427a 100644 --- a/src/relay/relay-daemon.ts +++ b/src/relay/relay-daemon.ts @@ -14,6 +14,7 @@ import { restrictWindowsRelayEndpointCredential } from './relay-endpoint-credential-publication' import { SKILL_RELAY_CAPABILITIES } from './skill-install-handler' +import { publishRelayPid } from './relay-pid-publication' export async function runRelayDaemon(options: RelayLaunchOptions): Promise { if (options.detached && options.logFile) { @@ -105,6 +106,7 @@ export async function runRelayDaemon(options: RelayLaunchOptions): Promise // exits inside start() and never reaches the credential file, so racing starters cannot // rotate the secret a surviving daemon enforces. await reconnectListener.start() + publishRelayPid() reconnectListener.setEndpointCredential(publishRelayEndpointCredential(options.credentialFile)) agentHooks.publishEndpointFile() } catch (error) { diff --git a/src/relay/relay-pid-publication.test.ts b/src/relay/relay-pid-publication.test.ts new file mode 100644 index 00000000000..f06640e1460 --- /dev/null +++ b/src/relay/relay-pid-publication.test.ts @@ -0,0 +1,41 @@ +import { existsSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { RELAY_PID_FILENAME, RELAY_VERSION_FILENAME } from '../shared/relay-artifacts' +import { publishRelayPid } from './relay-pid-publication' + +describe('publishRelayPid', () => { + const directories: string[] = [] + afterEach(async () => { + await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) + }) + + function versionDir(installed: boolean): string { + const dir = mkdtempSync(join(tmpdir(), 'relay-pid-')) + directories.push(dir) + writeFileSync(join(dir, 'relay.js'), '') + if (installed) { + writeFileSync(join(dir, RELAY_VERSION_FILENAME), '0.1.0+abc\n') + } + return dir + } + + it('records the daemon PID beside an installed relay.js', () => { + const dir = versionDir(true) + publishRelayPid(join(dir, 'relay.js')) + expect(readFileSync(join(dir, RELAY_PID_FILENAME), 'utf8')).toBe(`${process.pid}\n`) + expect(readdirSync(dir).filter((name) => name.endsWith('.tmp'))).toEqual([]) + }) + + it('leaves a build dir without .version untouched', () => { + const dir = versionDir(false) + publishRelayPid(join(dir, 'relay.js')) + expect(existsSync(join(dir, RELAY_PID_FILENAME))).toBe(false) + }) + + it('does not throw when the entry cannot be resolved', () => { + expect(() => publishRelayPid(join(tmpdir(), 'missing-relay-dir', 'relay.js'))).not.toThrow() + }) +}) diff --git a/src/relay/relay-pid-publication.ts b/src/relay/relay-pid-publication.ts new file mode 100644 index 00000000000..40152def587 --- /dev/null +++ b/src/relay/relay-pid-publication.ts @@ -0,0 +1,39 @@ +import { existsSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { RELAY_PID_FILENAME, RELAY_VERSION_FILENAME } from '../shared/relay-artifacts' +import { relayLogLine } from './relay-diagnostic-log' + +/** + * Record this daemon's PID in its version dir once it owns its socket, so version GC can + * tell a stale socket from a live daemon without connecting to one that is about to idle. + */ +export function publishRelayPid(entry: string | undefined = process.argv[1]): void { + if (!entry) { + return + } + let temporary: string | null = null + try { + const versionDir = dirname(realpathSync(entry)) + // Why: only an installed version dir carries `.version`; a dev or test build dir is not GC'd. + if (!existsSync(join(versionDir, RELAY_VERSION_FILENAME))) { + return + } + temporary = join(versionDir, `${RELAY_PID_FILENAME}.${process.pid}.tmp`) + writeFileSync(temporary, `${process.pid}\n`, { mode: 0o600 }) + // Why rename: GC must never read a half-written PID as a different, dead process. + renameSync(temporary, join(versionDir, RELAY_PID_FILENAME)) + temporary = null + } catch (error) { + relayLogLine( + `[relay] Could not record relay PID: ${error instanceof Error ? error.message : String(error)}` + ) + } finally { + if (temporary) { + try { + rmSync(temporary, { force: true }) + } catch { + // A leftover temp file is inert; GC reads only the renamed name. + } + } + } +} diff --git a/src/shared/relay-artifacts.ts b/src/shared/relay-artifacts.ts index 4e4c834f220..8a4633fb81c 100644 --- a/src/shared/relay-artifacts.ts +++ b/src/shared/relay-artifacts.ts @@ -88,6 +88,9 @@ export const RELAY_VERSION_FILENAME = '.version' /** Written last by the installer; its absence means a torn install. */ export const RELAY_INSTALL_COMPLETE_FILENAME = '.install-complete' +/** PID of the last relay daemon that bound a socket from this version dir; GC liveness evidence. */ +export const RELAY_PID_FILENAME = '.relay-pid' + /** Artifacts every relay must have; the remote install probe requires each one. */ export function relayArtifactFilenames(isWindows: boolean): string[] { return RELAY_ARTIFACTS.filter(