fix(ssh): collect relay versions only when provably exited; runtimes/ store GC (#24130)

* fix(ssh): relay version GC deletes only on an exited verdict and keeps the previous build

The relay records .relay-pid in its version dir once it owns its socket. GC calls a
relay version dir exited only when that PID is provably dead and every relay-*.sock
refuses a connection; a dir without a PID file keeps the test -S rule. The most
recently completed other relay build is pinned like orcad's rollback target.
Design D5 GC liveness.

* feat(ssh): collect the shared runtimes/ Node store and give it its own owner

runtimes/ gets its own owner in the install model, so no version-dir GC (new or old
clients, whose listings are prefix-scoped) can list or delete it. A store pass
removes node-<sha> only when no retained dir references it, it is neither a current
pin nor the newest other verified runtime, and a ps or /proc check ran and found no
process using it. Legacy relay-*/orcad-* dirs are read for references and reported
as diagnostics only (design D10 two-step). Wired behind orcad GC's nodeRuntimePins.

* fix(ssh): runtime store process check holds runtimes reached through a symlinked home

/proc exe resolves symlinks and argv keeps whatever spelling launched the runtime, so
filtering on the exact $root path missed in-use runtimes on hosts like /home -> /var/home.
Filter on the store segment instead; the parser already attributes holds root-agnostically.

* test(ssh): wait for the holder process to spawn instead of a fixed delay

---------

Co-authored-by: m4air <m4air@m4airs-Air.localdomain>
This commit is contained in:
OrcaWin
2026-10-01 01:10:47 -07:00
committed by GitHub
co-authored by m4air
parent 8c2cd7d331
commit 9ddcc9b9f0
22 changed files with 1396 additions and 45 deletions
+20
View File
@@ -201,4 +201,24 @@ describe('orcad GC', () => {
expect(removed).toEqual(['orcad-0.0.9+dead'])
})
it('collects the runtime store only when the caller names its runtime pins', async () => {
const removed: string[] = []
scriptHost({ listing: [], removed })
const options = {
conn,
host,
remoteHome: '/home/u',
currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb',
record: emptyOrcadActivationRecord()
}
const inventories = (): number =>
mockExec.mock.calls.filter(([, command]) => String(command).includes('RUNTIME_STORE')).length
await gcOldOrcadVersions(options)
expect(inventories()).toBe(0)
await gcOldOrcadVersions({ ...options, nodeRuntimePins: ['a'.repeat(64)] })
expect(inventories()).toBe(1)
})
})
+20 -8
View File
@@ -1,13 +1,12 @@
/**
* orcad's garbage collection, and the half of §06 falsifier 1 that says who owns it.
* orcad's garbage collection, and who owns it (design D10; to be tracked in
* docs/reference/remote-server-install-model.md).
*
* **Each model GCs only its own namespace, permanently.** orcad removes `orcad-<v>/`
* directories; the relay removes `relay-<v>/` directories; neither ever removes the other's,
* and no plan item makes one the winner. That is not a migration compromise — the two models
* serve different users on the same machine (SSH target vs paired peer), so there is no
* moment at which one of them is entitled to clean up after the other. A pass that deleted
* the sibling's tree would be reaching across the execution boundary the whole design exists
* to keep intact.
* **Each model GCs only its own namespace.** orcad removes `orcad-<v>/` directories; the relay
* removes `relay-<v>/` directories; neither ever removes the other's. The converged server's
* migration sweep takes over legacy directories only in the release after it has listed them
* as diagnostics, and only on an `exited` verdict. A pass that deleted the sibling's tree
* would be reaching across the execution boundary the whole design exists to keep intact.
*
* On top of the ownership rule, orcad pins three directories that are idle-looking but
* load-bearing: the active version, the rollback target, and whichever version the LIVE
@@ -25,6 +24,7 @@ import {
parseOrcadLiveness
} from './orcad-remote-launch'
import type { RemoteHostPlatform } from './ssh-remote-platform'
import { gcRemoteNodeRuntimeStore } from './remote-node-runtime-store-gc'
export type OrcadGcOptions = {
conn: SshConnection
@@ -41,6 +41,11 @@ export type OrcadGcOptions = {
* would remove the tree under a running process.
*/
liveDaemonVersion?: string | null
/**
* executableSha256 of every runtime pin this client runs. Also the gate for the shared
* runtime store pass: without it this client cannot say which runtime is current.
*/
nodeRuntimePins?: readonly string[]
signal?: AbortSignal
}
@@ -75,4 +80,11 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise<void>
}
}
)
// Why after the version pass: removing version dirs is what drops their runtime references.
if (options.nodeRuntimePins?.length) {
await gcRemoteNodeRuntimeStore(options.conn, options.host, options.remoteHome, {
currentPins: options.nodeRuntimePins,
signal: options.signal
})
}
}
@@ -0,0 +1,131 @@
/** Runs the generated POSIX liveness probe through a real `/bin/sh` against real sockets. */
import { execFileSync, spawnSync } from 'node:child_process'
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { createServer, type Server } from 'node:net'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { RELAY_PID_FILENAME } from '../../shared/relay-artifacts'
import {
parseRelayVersionDirLiveness,
relayVersionDirLivenessCommand
} from './relay-version-dir-liveness'
import { getRemoteHostPlatform } from './ssh-remote-platform'
const host = getRemoteHostPlatform('linux-x64')
const posixOnly = process.platform === 'win32' ? describe.skip : describe
function probe(dir: string, nodePath: string | undefined = process.execPath): string {
return execFileSync('/bin/sh', ['-c', relayVersionDirLivenessCommand(host, dir, { nodePath })], {
encoding: 'utf8'
})
}
/** A socket inode left by a SIGKILLed listener: connect is refused. Returns the dead PID. */
function leaveStaleSocket(sockPath: string): number {
const child = spawnSync(
process.execPath,
[
'-e',
'require("net").createServer().listen(process.argv[1],()=>{' +
'process.stdout.write(String(process.pid));process.kill(process.pid,"SIGKILL")})',
sockPath
],
{ encoding: 'utf8' }
)
return Number.parseInt(child.stdout, 10)
}
posixOnly('relayVersionDirLivenessCommand (real shell)', () => {
const dirs: string[] = []
const servers: Server[] = []
afterEach(async () => {
await Promise.all(
servers.splice(0).map((server) => new Promise<void>((done) => server.close(() => done())))
)
for (const dir of dirs.splice(0)) {
rmSync(dir, { recursive: true, force: true })
}
})
function versionDir(): string {
// Short base: sun_path caps socket paths near 104 bytes on macOS.
const dir = mkdtempSync(join('/tmp', 'rvl-'))
dirs.push(dir)
return dir
}
it('is exited for a stale socket whose recorded PID is dead', () => {
const dir = versionDir()
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('exited')
})
it('is live for a stale socket whose recorded PID is still running', () => {
const dir = versionDir()
leaveStaleSocket(join(dir, 'relay-a.sock'))
writeFileSync(join(dir, RELAY_PID_FILENAME), `${process.pid}\n`)
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live')
})
it('keeps the test -S rule for a refused socket without a PID file', () => {
const dir = versionDir()
leaveStaleSocket(join(dir, 'relay-a.sock'))
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live')
})
it('is exited without a PID file only when no socket is left', () => {
expect(parseRelayVersionDirLiveness(probe(versionDir()))).toBe('exited')
})
it('is live when another relay of this build still accepts on its socket', async () => {
const dir = versionDir()
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
const server = createServer()
servers.push(server)
await new Promise<void>((done) => server.listen(join(dir, 'relay-b.sock'), done))
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live')
})
it('is unverifiable when the connect probe times out', () => {
const dir = versionDir()
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
// Stands in for node: the connect probe prints `unknown` when its timer fires first.
const timedOutNode = join(dir, 'node')
writeFileSync(timedOutNode, '#!/bin/sh\nprintf unknown\n')
chmodSync(timedOutNode, 0o755)
expect(parseRelayVersionDirLiveness(probe(dir, timedOutNode))).toBe('unverifiable')
})
it('is unverifiable when no Node is available to test a leftover socket', () => {
const dir = versionDir()
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
expect(parseRelayVersionDirLiveness(probe(dir, ''))).toBe('unverifiable')
})
it('is unverifiable for an unreadable PID record', () => {
const dir = versionDir()
writeFileSync(join(dir, RELAY_PID_FILENAME), 'not-a-pid\n')
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('unverifiable')
})
})
describe('parseRelayVersionDirLiveness', () => {
it('maps the Windows pipe vocabulary and treats anything else as unverifiable', () => {
expect(parseRelayVersionDirLiveness('ALIVE')).toBe('live')
expect(parseRelayVersionDirLiveness('WAITING')).toBe('exited')
expect(parseRelayVersionDirLiveness('DEAD\n')).toBe('exited')
expect(parseRelayVersionDirLiveness('')).toBe('unverifiable')
expect(parseRelayVersionDirLiveness('UNKNOWN')).toBe('unverifiable')
})
})
@@ -0,0 +1,65 @@
/**
* Whether a relay version directory is still in use, answered with the execution-boundary
* vocabulary (docs/reference/ssh-execution-boundary.md): `live` / `unverifiable` / `exited`.
*
* Design D5: a directory is `exited` only when its recorded `.relay-pid` is provably dead AND
* every `relay-*.sock` in it refuses a connection. The PID is checked first so a live daemon
* about to idle is never connected to (a connection would cancel its grace timer). A directory
* with no PID file was last used by a relay that predates it and keeps the `test -S` rule.
*/
import { RELAY_PID_FILENAME } from '../../shared/relay-artifacts'
import { shellEscape } from './ssh-connection-utils'
import { posixProcessAliveShellFunction } from './orcad-remote-host-support'
import { RELAY_CONNECT_PROBE_JS, type RelayEndpointVerdict } from './ssh-relay-endpoint-incumbent'
import { relayLivenessProbeCommand, type WindowsRelayLivenessOptions } from './ssh-remote-commands'
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
export function relayVersionDirLivenessCommand(
host: RemoteHostPlatform,
dir: string,
options: { nodePath?: string; windows?: WindowsRelayLivenessOptions } = {}
): string {
if (isWindowsRemoteHost(host)) {
return relayLivenessProbeCommand(host, dir, options.windows)
}
return [
`dir=${shellEscape(dir)}`,
`node=${shellEscape(options.nodePath ?? '')}`,
`pid_file="$dir"/${RELAY_PID_FILENAME}`,
'socks=',
'for f in "$dir"/relay-*.sock "$dir"/relay.sock; do [ -S "$f" ] && socks=yes; done',
'if [ ! -e "$pid_file" ]; then',
' if [ -n "$socks" ]; then echo LIVE; else echo EXITED; fi',
' exit 0',
'fi',
// Prints UNKNOWN and exits when kill -0 fails for any reason but "No such process".
posixProcessAliveShellFunction({ refuseUnverifiable: true }),
'pid=$(cat "$pid_file" 2>/dev/null) || { echo UNVERIFIABLE; exit 0; }',
'case "$pid" in "" | *[!0-9]*) echo UNVERIFIABLE; exit 0;; esac',
'if orcad_alive "$pid"; then echo LIVE; exit 0; fi',
'for f in "$dir"/relay-*.sock "$dir"/relay.sock; do',
' [ -S "$f" ] || continue',
' [ -n "$node" ] || { echo UNVERIFIABLE; exit 0; }',
// Another relay of this build may share the dir under its own socket; only a refusal clears it.
` r=$("$node" -e ${shellEscape(RELAY_CONNECT_PROBE_JS)} "$f" 2>/dev/null) || r=unknown`,
' case "$r" in',
' refused | absent) ;;',
' accepted) echo LIVE; exit 0;;',
' *) echo UNVERIFIABLE; exit 0;;',
' esac',
'done',
'echo EXITED'
].join('\n')
}
export function parseRelayVersionDirLiveness(output: string): RelayEndpointVerdict {
const token = output.trim().split('\n').pop()?.trim() ?? ''
// ALIVE / DEAD / WAITING are the Windows pipe probe's vocabulary.
if (token === 'LIVE' || token === 'ALIVE') {
return 'live'
}
if (token === 'EXITED' || token === 'DEAD' || token === 'WAITING') {
return 'exited'
}
return 'unverifiable'
}
@@ -11,6 +11,7 @@ import { gcOldRelayVersions } from './remote-install-gc'
import { execCommand } from './ssh-relay-deploy-helpers'
import { gcRelayNativeDepsCache } from './ssh-relay-native-deps-cache-gc'
import { gcRemoteRipgrepCache } from './ssh-relay-ripgrep-cache-gc'
import { REMOTE_INSTALL_ORDER_OK } from './remote-install-previous-version'
import { getRemoteHostPlatform } from './ssh-remote-platform'
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock.
@@ -41,8 +42,17 @@ function collectRelayVersions(): Promise<void> {
return gcOldRelayVersions(conn, home, currentDir, host, { nativeDepsCacheKeys: [nativeKey] })
}
// ddd is the previous build, so it is pinned and never probed.
const installOrder = [
`${home}/.orca-remote/relay-0.1.0+bbb/.install-complete`,
`${home}/.orca-remote/relay-0.1.0+ddd/.install-complete`,
`${home}/.orca-remote/relay-0.1.0+aaa/.install-complete`,
REMOTE_INSTALL_ORDER_OK
].join('\n')
const versionSteps = [
['listing', 'relay-0.1.0+aaa\nrelay-0.1.0+ccc'],
['listing', 'relay-0.1.0+aaa\nrelay-0.1.0+ddd'],
['previous install order', installOrder],
['install lock probe', 'OPEN'],
['completion probe', 'COMPLETE'],
['liveness probe', 'DEAD'],
@@ -73,8 +83,8 @@ describe('version GC termination', () => {
'stops after an unconfirmed stale lock probe with claim held: %s',
async (claimed) => {
const replies = claimed
? versionSteps.slice(0, 6).map(([, reply]) => String(reply))
: [versionSteps[0][1]]
? versionSteps.slice(0, 7).map(([, reply]) => String(reply))
: versionSteps.slice(0, 2).map(([, reply]) => String(reply))
const error = failAfter([...replies, 'LOCKED'])
await expect(collectRelayVersions()).rejects.toBe(error)
+51 -16
View File
@@ -28,7 +28,6 @@ import {
MAX_RELAY_GC_LISTING_ENTRIES,
moveRemoteTreeCommand,
probeFileExistsCommand,
relayLivenessProbeCommand,
removeRemoteTreeCommand
} from './ssh-remote-commands'
import {
@@ -39,7 +38,13 @@ import {
type RemoteHostPlatform
} from './ssh-remote-platform'
import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints'
import type { RelayEndpointVerdict } from './ssh-relay-endpoint-incumbent'
import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
import { findPreviousRemoteInstall } from './remote-install-previous-version'
import {
parseRelayVersionDirLiveness,
relayVersionDirLivenessCommand
} from './relay-version-dir-liveness'
// Legacy relay dirs predate `.install-complete`; they need a liveness-only GC check so they
// eventually drain. There is no orcad equivalent — orcad has never shipped without one.
@@ -68,13 +73,19 @@ export type RemoteInstallGcOptions = {
* the rollback target, and GC'ing it turns a recoverable bad update into a re-deploy.
*/
pinnedDirNames?: readonly string[]
/**
* More pins, resolved only once a candidate exists. Null means the host could not say which
* directories to keep, so this pass deletes nothing.
*/
resolveExtraPinnedDirNames?: () => Promise<readonly string[] | null>
}
/**
* Garbage-collect one model's old version directories.
*
* **GC ownership (design §06 falsifier 1):** a pass only ever sees, and only ever deletes,
* directories belonging to `model`. The remote listing is scoped by prefix, and
* **GC ownership (design D10; to be tracked in docs/reference/remote-server-install-model.md):**
* a pass only ever sees, and only ever deletes, directories belonging to `model`. The remote
* listing is scoped by prefix, and
* `remoteInstallGcPermits` re-checks every candidate locally, so neither a widened glob nor
* a hand-rolled listing can make one model delete the other's live install.
*/
@@ -121,10 +132,17 @@ export async function gcOldRemoteInstallVersions(
if (candidates.length === 0) {
return
}
const extraPins = options.resolveExtraPinnedDirNames
? await options.resolveExtraPinnedDirNames()
: []
if (!extraPins) {
return
}
const survivors = candidates.filter((name) => !extraPins.includes(name))
const removed: string[] = []
const kept: string[] = []
for (const name of candidates) {
for (const name of survivors) {
const dir = joinRemotePath(host, baseDir, name)
try {
const safe = await isCandidateSafeToRemove(conn, model, dir, name, host, options)
@@ -249,8 +267,8 @@ async function isCandidateSafeToRemove(
}
/**
* The relay's GC, bound to its own namespace and its own liveness probe (a live unix socket
* or Windows pipe inside the version dir).
* The relay's GC, bound to its own namespace. A version dir goes only on an `exited` verdict
* (relay-version-dir-liveness.ts), and the previous completed build is pinned (design D5).
*/
export async function gcOldRelayVersions(
conn: SshConnection,
@@ -260,6 +278,8 @@ export async function gcOldRelayVersions(
options?: {
windowsNodePath?: string
windowsSockNames?: string[]
/** Host Node that runs the connect probe once a recorded relay PID is dead. */
nodePath?: string
/**
* Cache entries this connection depends on, whether or not it links to them. Also the gate:
* a caller that could not compute a key is not using the shared-cache model on this host, and
@@ -270,7 +290,22 @@ export async function gcOldRelayVersions(
): Promise<void> {
await gcOldRemoteInstallVersions(conn, RELAY_INSTALL_MODEL, remoteHome, currentDirAbsPath, host, {
...options,
isDirLive: (dir) => hasLiveRelaySocket(conn, dir, host, options)
resolveExtraPinnedDirNames: async () => {
const previous = await findPreviousRemoteInstall(
conn,
host,
joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR),
RELAY_INSTALL_MODEL,
remoteBasename(currentDirAbsPath, host)
)
// Why null rather than a guess: without the order, any candidate could be the previous build.
if (previous.state !== 'ok') {
return null
}
return previous.dirName ? [previous.dirName] : []
},
isDirLive: async (dir) =>
(await probeRelayVersionDirLiveness(conn, dir, host, options)) !== 'exited'
})
// Why after and not before: version-dir removal is what turns a cache entry unreferenced, so
// running it second lets one pass reclaim both instead of leaving the tree for the next connect.
@@ -285,18 +320,19 @@ export async function gcOldRelayVersions(
}
}
async function hasLiveRelaySocket(
/** GC deletes a relay version dir only on `exited`; see relay-version-dir-liveness.ts. */
export async function probeRelayVersionDirLiveness(
conn: SshConnection,
dir: string,
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
options?: {
windowsNodePath?: string
windowsSockNames?: string[]
nodePath?: string
}
): Promise<boolean> {
): Promise<RelayEndpointVerdict> {
try {
// Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle.
const windowsOptions =
const windows =
isWindowsRemoteHost(host) && options?.windowsNodePath
? {
nodePath: options.windowsNodePath,
@@ -308,15 +344,14 @@ async function hasLiveRelaySocket(
const out = await execHostCommand(
conn,
host,
relayLivenessProbeCommand(host, dir, windowsOptions)
relayVersionDirLivenessCommand(host, dir, { nodePath: options?.nodePath, windows })
)
const state = out.trim()
return state !== 'DEAD' && state !== 'WAITING'
return parseRelayVersionDirLiveness(out)
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
// Why: an inconclusive liveness probe must never authorize deletion.
return true
// Why: an unanswered probe observes nothing; it never authorizes deletion.
return 'unverifiable'
}
}
+35 -1
View File
@@ -4,7 +4,10 @@ import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { orcadRipgrepArtifact } from '../../shared/orcad-artifacts'
import { probeRemoteInstallCompleteCommand } from './ssh-remote-commands'
import {
listRemoteInstallBaseDirsCommand,
probeRemoteInstallCompleteCommand
} from './ssh-remote-commands'
import { getRemoteHostPlatform } from './ssh-remote-platform'
import {
@@ -120,4 +123,35 @@ describe('GC ownership — each model collects only its own namespace', () => {
expect(inventory.orcad).toEqual(ORCAD_DIRS)
expect(inventory.unknown).toEqual(['something-else'])
})
it('gives the shared runtime store its own owner that no version-dir GC may take', () => {
expect(remoteInstallDirOwner('runtimes')).toBe('runtimes')
expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, 'runtimes')).toBe(false)
expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'runtimes')).toBe(false)
expect(inventoryRemoteInstallDirs(['runtimes', ...RELAY_DIRS]).runtimes).toEqual(['runtimes'])
})
it.skipIf(process.platform === 'win32')(
'keeps runtimes/ out of every model listing, including older clients’ prefix scans',
() => {
const base = mkdtempSync(join(tmpdir(), 'install-listing-'))
try {
for (const name of ['runtimes', 'relay-0.1.0+aa', 'orcad-0.1.0+aa']) {
mkdirSync(join(base, name))
}
mkdirSync(join(base, 'runtimes', `node-${'a'.repeat(64)}`))
const host = getRemoteHostPlatform('linux-x64')
for (const model of [RELAY_INSTALL_MODEL, ORCAD_INSTALL_MODEL]) {
const listed = execFileSync(
'/bin/sh',
['-c', listRemoteInstallBaseDirsCommand(host, base, model)],
{ encoding: 'utf8' }
)
expect(listed.trim().split('\n')).toEqual([`${model.dirPrefix}-0.1.0+aa`])
}
} finally {
rmSync(base, { recursive: true, force: true })
}
}
)
})
+26 -14
View File
@@ -1,11 +1,13 @@
/**
* The two things Orca installs into `~/.orca-remote/`, and the rules that keep them from
* The things Orca installs into `~/.orca-remote/`, and the rules that keep them from
* touching each other.
*
* `docs/design/shipping-orcad.html` §06 settles that on-disk coexistence is permanent: the
* relay is the dumb execution host for SSH-target users, orcad is the peer for paired
* environments, and no plan item retires either. So `relay-<version>/` and `orcad-<version>/`
* sit side by side forever, and the namespace has to be a parameter rather than a literal.
* Design D10 (to be tracked in docs/reference/remote-server-install-model.md): the relay and
* orcad converge into one server package; on the host that is `server-<version>/` plus the
* shared `runtimes/node-<sha256>/` store. Legacy `relay-*` / `orcad-*` directories belong to a
* migration sweep that deletes only on an `exited` verdict, handed over in two releases: this
* one lists them as diagnostics only. Until then each is its own namespace, so the prefix has
* to be a parameter rather than a literal.
*
* GC ownership is the trap that parameterization creates. Each model garbage-collects ONLY
* its own directories — see `remoteInstallDirOwner`. Relay's regex happened to be narrow
@@ -21,6 +23,7 @@ import {
import {
orcadArtifactFilenames,
ORCAD_INSTALL_COMPLETE_FILENAME,
ORCAD_RUNTIMES_DIRNAME,
ORCAD_VERSION_FILENAME
} from '../../shared/orcad-artifacts'
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
@@ -104,15 +107,18 @@ export function remoteInstallListingRegexSource(model: RemoteInstallModel): stri
}
/**
* Which model owns a directory found in `~/.orca-remote/`, or null for anything neither
* model created.
* Which owner a directory found in `~/.orca-remote/` belongs to, or null for anything no
* owner created.
*
* This is the answer to §06 falsifier 1's first half: **the model that created a directory
* owns it, and nothing else may delete it.** A relay GC pass that saw `orcad-0.1.0+abc`
* would be looking at the live install of a peer whose lifecycle it has no view into — the
* SSH-execution-boundary collapse in directory form.
* Design D10: **the model that created a directory owns it, and nothing else may delete it**
* (amended only in the release after the two-step hand-over). A relay GC pass that saw
* `orcad-0.1.0+abc` would be looking at the live install of a peer whose lifecycle it has no
* view into — the SSH-execution-boundary collapse in directory form.
*/
export function remoteInstallDirOwner(dirName: string): RemoteInstallModelId | null {
export function remoteInstallDirOwner(dirName: string): RemoteInstallDirOwner | null {
if (dirName === ORCAD_RUNTIMES_DIRNAME) {
return 'runtimes'
}
for (const model of REMOTE_INSTALL_MODELS) {
if (new RegExp(remoteInstallListingRegexSource(model)).test(dirName)) {
return model.id
@@ -126,11 +132,17 @@ export function remoteInstallGcPermits(model: RemoteInstallModel, dirName: strin
return remoteInstallDirOwner(dirName) === model.id
}
export type RemoteInstallInventory = Record<RemoteInstallModelId | 'unknown', string[]>
/**
* `runtimes` is the shared Node store (design D5). No version-dir model owns it, so neither
* model's GC can list or delete it; only `remote-node-runtime-store-gc.ts` collects inside it.
*/
export type RemoteInstallDirOwner = RemoteInstallModelId | 'runtimes'
export type RemoteInstallInventory = Record<RemoteInstallDirOwner | 'unknown', string[]>
/** Group a raw `~/.orca-remote/` listing by owning model, for diagnostics and the client's choice. */
export function inventoryRemoteInstallDirs(dirNames: readonly string[]): RemoteInstallInventory {
const inventory: RemoteInstallInventory = { relay: [], orcad: [], unknown: [] }
const inventory: RemoteInstallInventory = { relay: [], orcad: [], runtimes: [], unknown: [] }
for (const name of dirNames) {
const owner = remoteInstallDirOwner(name)
if (owner) {
@@ -0,0 +1,160 @@
import { execFileSync } from 'node:child_process'
import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as DeployHelpers from './ssh-relay-deploy-helpers'
vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({
...(await importOriginal<typeof DeployHelpers>()),
execCommand: vi.fn()
}))
import type { SshConnection } from './ssh-connection'
import { gcOldRelayVersions } from './remote-install-gc'
import { ORCAD_INSTALL_MODEL, RELAY_INSTALL_MODEL } from './remote-install-model'
import {
listCompletedInstallsNewestFirstCommand,
parseCompletedInstallsNewestFirst,
REMOTE_INSTALL_ORDER_OK
} from './remote-install-previous-version'
import { execCommand } from './ssh-relay-deploy-helpers'
import { getRemoteHostPlatform } from './ssh-remote-platform'
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock.
const conn = {} as SshConnection
const host = getRemoteHostPlatform('linux-x64')
const mockExec = vi.mocked(execCommand)
describe('parseCompletedInstallsNewestFirst', () => {
it('keeps only this model’s version dirs, in host order', () => {
const output = [
'/h/.orca-remote/relay-0.2.0+bbb/.install-complete',
'/h/.orca-remote/orcad-0.2.0+bbb/.install-complete',
'/h/.orca-remote/relay-0.1.0+aaa/.install-complete',
REMOTE_INSTALL_ORDER_OK
].join('\n')
expect(parseCompletedInstallsNewestFirst(output, RELAY_INSTALL_MODEL)).toEqual([
'relay-0.2.0+bbb',
'relay-0.1.0+aaa'
])
})
it('is null when the host did not finish the listing', () => {
expect(
parseCompletedInstallsNewestFirst(
'/h/.orca-remote/relay-0.1.0+aaa/.install-complete',
RELAY_INSTALL_MODEL
)
).toBeNull()
})
})
const posixOnly = process.platform === 'win32' ? describe.skip : describe
posixOnly('listCompletedInstallsNewestFirstCommand (real shell)', () => {
let base: string
beforeEach(() => {
base = mkdtempSync(join(tmpdir(), 'install-order-'))
})
afterEach(() => {
rmSync(base, { recursive: true, force: true })
})
function install(name: string, mtimeSeconds: number, complete = true): void {
mkdirSync(join(base, name))
if (complete) {
const marker = join(base, name, '.install-complete')
writeFileSync(marker, '')
utimesSync(marker, mtimeSeconds, mtimeSeconds)
}
}
function run(): string[] | null {
const out = execFileSync(
'/bin/sh',
['-c', listCompletedInstallsNewestFirstCommand(host, base, RELAY_INSTALL_MODEL)],
{ encoding: 'utf8' }
)
return parseCompletedInstallsNewestFirst(out, RELAY_INSTALL_MODEL)
}
it('orders completed installs by marker mtime and skips torn ones', () => {
install('relay-0.1.0+aaa', 1_000)
install('relay-0.3.0+ccc', 3_000)
install('relay-0.2.0+bbb', 2_000)
install('relay-0.4.0+ddd', 4_000, false)
install('orcad-0.9.0+eee', 9_000)
expect(run()).toEqual(['relay-0.3.0+ccc', 'relay-0.2.0+bbb', 'relay-0.1.0+aaa'])
})
it('answers an empty order for a base without completed installs', () => {
expect(run()).toEqual([])
})
it('is scoped to the model prefix', () => {
install('orcad-0.9.0+eee', 9_000)
const out = execFileSync(
'/bin/sh',
['-c', listCompletedInstallsNewestFirstCommand(host, base, ORCAD_INSTALL_MODEL)],
{ encoding: 'utf8' }
)
expect(parseCompletedInstallsNewestFirst(out, ORCAD_INSTALL_MODEL)).toEqual(['orcad-0.9.0+eee'])
})
})
describe('relay GC keeps the previous build', () => {
beforeEach(() => {
mockExec.mockReset()
mockExec.mockResolvedValue('')
})
it('never probes or removes the most recent other completed install', async () => {
mockExec
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
.mockResolvedValueOnce(
[
'/home/u/.orca-remote/relay-0.2.0+bbb/.install-complete',
'/home/u/.orca-remote/relay-0.1.0+aaa/.install-complete',
REMOTE_INSTALL_ORDER_OK
].join('\n')
)
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host)
expect(mockExec).toHaveBeenCalledTimes(2)
})
it('deletes nothing when the host cannot report install order', async () => {
mockExec
.mockResolvedValueOnce('relay-0.1.0+aaa\nrelay-0.0.9+zzz\n')
.mockRejectedValueOnce(
Object.assign(new Error('ls failed'), { sshChannelCloseConfirmed: true })
)
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host)
expect(mockExec).toHaveBeenCalledTimes(2)
})
it('keeps an old build whose liveness probe times out', async () => {
mockExec
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
.mockResolvedValueOnce(`relay-0.1.5+fff\n${REMOTE_INSTALL_ORDER_OK}`)
.mockResolvedValueOnce('OPEN')
.mockResolvedValueOnce('COMPLETE')
.mockRejectedValueOnce(
Object.assign(new Error('SSH command timed out'), { sshChannelCloseConfirmed: true })
)
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host, {
nodePath: '/usr/bin/node'
})
const commands = mockExec.mock.calls.map(([, command]) => command)
expect(commands[4]).toContain('.relay-pid')
expect(commands.some((command) => command.includes('gc-claim'))).toBe(false)
expect(mockExec).toHaveBeenCalledTimes(5)
})
})
@@ -0,0 +1,106 @@
/**
* The version dir a model ran before the current one, pinned against GC like orcad's
* rollback target (design D5 "plus the relay's previous version as a pin").
*
* The relay keeps no activation record, so "previous" is the most recently completed install
* of the same model other than the current one, by `.install-complete` mtime.
*/
import type { SshConnection } from './ssh-connection'
import { shellEscape } from './ssh-connection-utils'
import { execCommand } from './ssh-relay-deploy-helpers'
import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
import {
remoteInstallGcPermits,
remoteInstallVersionDirRegex,
type RemoteInstallModel
} from './remote-install-model'
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell'
export const REMOTE_INSTALL_ORDER_OK = 'ORCA_INSTALL_ORDER_OK'
export function listCompletedInstallsNewestFirstCommand(
host: RemoteHostPlatform,
baseDir: string,
model: RemoteInstallModel
): string {
const prefix = `${model.dirPrefix}-`
if (isWindowsRemoteHost(host)) {
return powerShellCommand(
[
"$ErrorActionPreference = 'Stop'",
`$base = ${powerShellLiteral(baseDir)}`,
'if (Test-Path -LiteralPath $base -PathType Container) {',
`Get-ChildItem -LiteralPath $base -Directory -Filter '${prefix}*' | ForEach-Object { ` +
`$marker = Join-Path $_.FullName ${powerShellLiteral(model.installCompleteFilename)}; ` +
'if (Test-Path -LiteralPath $marker -PathType Leaf) { Get-Item -LiteralPath $marker } ' +
'} | Sort-Object LastWriteTimeUtc -Descending | ForEach-Object { $_.Directory.Name }',
'}',
`'${REMOTE_INSTALL_ORDER_OK}'`
].join('\n')
)
}
return [
`base=${shellEscape(baseDir)}`,
`set -- "$base"/${prefix}*/${shellEscape(model.installCompleteFilename)}`,
`[ -e "$1" ] || { echo ${REMOTE_INSTALL_ORDER_OK}; exit 0; }`,
'ls -1t -- "$@" || exit 1',
`echo ${REMOTE_INSTALL_ORDER_OK}`
].join('\n')
}
/** Dir names newest first, or null when the host could not answer. */
export function parseCompletedInstallsNewestFirst(
output: string,
model: RemoteInstallModel
): string[] | null {
const lines = output
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean)
if (!lines.includes(REMOTE_INSTALL_ORDER_OK)) {
return null
}
const versionDirRegex = remoteInstallVersionDirRegex(model)
const names: string[] = []
for (const line of lines) {
// POSIX prints `<base>/<dir>/.install-complete`; PowerShell prints `<dir>`.
const segments = line.split(/[\\/]/)
const name = segments.length > 1 ? segments.at(-2) : segments[0]
if (name && versionDirRegex.test(name) && remoteInstallGcPermits(model, name)) {
names.push(name)
}
}
return names
}
export type PreviousInstallResult = { state: 'ok'; dirName: string | null } | { state: 'unknown' }
export async function findPreviousRemoteInstall(
conn: SshConnection,
host: RemoteHostPlatform,
baseDir: string,
model: RemoteInstallModel,
currentDirName: string
): Promise<PreviousInstallResult> {
let output: string
try {
output = await execCommand(
conn,
listCompletedInstallsNewestFirstCommand(host, baseDir, model),
{
wrapCommand: host.commandDialect !== 'powershell'
}
)
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
return { state: 'unknown' }
}
const ordered = parseCompletedInstallsNewestFirst(output, model)
if (!ordered) {
return { state: 'unknown' }
}
return { state: 'ok', dirName: ordered.find((name) => name !== currentDirName) ?? null }
}
@@ -0,0 +1,279 @@
import { execFileSync, spawn, type ChildProcess } from 'node:child_process'
import {
chmodSync,
existsSync,
mkdirSync,
mkdtempSync,
rmSync,
symlinkSync,
utimesSync,
writeFileSync
} from 'node:fs'
import { once } from 'node:events'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as DeployHelpers from './ssh-relay-deploy-helpers'
vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({
...(await importOriginal<typeof DeployHelpers>()),
execCommand: vi.fn()
}))
import type { SshConnection } from './ssh-connection'
import { gcRemoteNodeRuntimeStore, planRuntimeStoreGc } from './remote-node-runtime-store-gc'
import {
parseRuntimeStoreInventory,
RUNTIME_REF_NODE_PREFIX,
runtimeStoreInventoryCommand,
type RuntimeStoreInventory
} from './remote-node-runtime-store-inventory'
import { execCommand } from './ssh-relay-deploy-helpers'
import { getRemoteHostPlatform } from './ssh-remote-platform'
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock.
const conn = {} as SshConnection
const host = getRemoteHostPlatform('linux-x64')
const mockExec = vi.mocked(execCommand)
const sha = (c: string): string => c.repeat(64)
function inventory(overrides: Partial<RuntimeStoreInventory> = {}): RuntimeStoreInventory {
return {
entries: [],
verifiedNewestFirst: [],
referenced: new Set(),
held: new Set(),
processCheckRan: true,
dirNames: [],
...overrides
}
}
describe('planRuntimeStoreGc', () => {
const all = ['a', 'b', 'c', 'd'].map((c) => `node-${sha(c)}`)
it('keeps the current pin and the newest other verified runtime', () => {
const plan = planRuntimeStoreGc(
inventory({ entries: all, verifiedNewestFirst: [all[0], all[1], all[2], all[3]] }),
[sha('b')]
)
expect(plan.remove).toEqual([all[2], all[3]])
expect(plan.kept).toEqual([all[0], all[1]])
})
it('keeps referenced, process-held and unverified runtimes', () => {
const plan = planRuntimeStoreGc(
inventory({
entries: all,
verifiedNewestFirst: [all[0], all[1], all[2]],
referenced: new Set([sha('b')]),
held: new Set([sha('c')])
}),
[sha('a')]
)
expect(plan.remove).toEqual([])
})
it('keeps everything when no process check could run', () => {
const plan = planRuntimeStoreGc(
inventory({ entries: all, verifiedNewestFirst: all, processCheckRan: false }),
[sha('a')]
)
expect(plan.remove).toEqual([])
})
it('purges only abandoned tombstones of unwanted runtimes', () => {
const now = 10 * 60 * 60_000
const old = `.gc-tombstone-node-${sha('c')}.7.${now - 31 * 60_000}`
const fresh = `.gc-tombstone-node-${sha('d')}.7.${now - 60_000}`
const referenced = `.gc-tombstone-node-${sha('e')}.7.${now - 31 * 60_000}`
const plan = planRuntimeStoreGc(
inventory({ entries: [old, fresh, referenced], referenced: new Set([sha('e')]) }),
[sha('a')],
now
)
expect(plan.purgeTombstones).toEqual([old])
})
})
describe('parseRuntimeStoreInventory', () => {
it('rejects a partial listing, a reference error, and an unattributable reference', () => {
expect(parseRuntimeStoreInventory('ENTRY node-x')).toBeNull()
expect(
parseRuntimeStoreInventory('__ORCA_RUNTIME_STORE__REFS_ERR\n__ORCA_RUNTIME_STORE__OK')
).toBeNull()
expect(parseRuntimeStoreInventory('REF garbage\n__ORCA_RUNTIME_STORE__OK')).toBeNull()
})
it('attributes process holds by runtime path, including renamed tombstones', () => {
const parsed = parseRuntimeStoreInventory(
[
'PROCESS_CHECK ps',
`HOLD /h/.orca-remote/runtimes/node-${sha('a')}/bin/node server.js`,
`HOLD /h/.orca-remote/runtimes/.gc-tombstone-node-${sha('b')}.1.2/bin/node`,
'HOLD grep -F -- /h/.orca-remote/runtimes/',
'__ORCA_RUNTIME_STORE__OK'
].join('\n')
)
expect(parsed?.held).toEqual(new Set([sha('a'), sha('b')]))
})
})
const posixOnly = process.platform === 'win32' ? describe.skip : describe
posixOnly('gcRemoteNodeRuntimeStore (real shell)', () => {
let home: string
let root: string
let running: ChildProcess | null = null
beforeEach(() => {
home = mkdtempSync(join(tmpdir(), 'runtime-store-'))
root = join(home, '.orca-remote')
mkdirSync(join(root, 'runtimes'), { recursive: true })
mockExec.mockReset()
mockExec.mockImplementation(async (_conn, command) =>
execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' })
)
})
afterEach(() => {
running?.kill('SIGKILL')
running = null
rmSync(home, { recursive: true, force: true })
})
// `b` is always verified last, so it is the previous pin the store keeps.
function runtime(c: string, verified = true): string {
const dir = join(root, 'runtimes', `node-${sha(c)}`)
mkdirSync(join(dir, 'bin'), { recursive: true })
writeFileSync(join(dir, 'bin', 'node'), '#!/bin/sh\nsleep 30\n')
chmodSync(join(dir, 'bin', 'node'), 0o755)
if (verified) {
writeFileSync(join(dir, '.verified'), '')
const at = c === 'b' ? 2_000_000 : 1_000_000
utimesSync(join(dir, '.verified'), at, at)
}
return dir
}
function versionDir(name: string): string {
const dir = join(root, name)
mkdirSync(dir, { recursive: true })
return dir
}
it('removes only unreferenced, unpinned, idle, verified runtimes', async () => {
runtime('a') // current pin
runtime('c') // referenced by an orcad slot marker
runtime('d') // referenced by a relay ref file
runtime('e') // unreferenced: collected
const running_ = runtime('f') // executing
runtime('9', false) // mid-promotion
runtime('b') // newest other verified: the previous pin
writeFileSync(join(versionDir('orcad-0.1.0+aaa'), '.runtime-node'), `${sha('c')}\n`)
writeFileSync(join(versionDir('relay-0.1.0+aaa'), `${RUNTIME_REF_NODE_PREFIX}${sha('d')}`), '')
running = spawn(join(running_, 'bin', 'node'), [], { stdio: 'ignore' })
await once(running, 'spawn')
const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] })
expect(result).toMatchObject({
state: 'collected',
removed: [`node-${sha('e')}`],
legacyDirs: expect.arrayContaining(['orcad-0.1.0+aaa', 'relay-0.1.0+aaa'])
})
for (const kept of ['a', 'b', 'c', 'd', 'f', '9']) {
expect(existsSync(join(root, 'runtimes', `node-${sha(kept)}`))).toBe(true)
}
expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`))).toBe(false)
// D10 two-step: legacy dirs are reported, never deleted by this pass.
expect(existsSync(join(root, 'orcad-0.1.0+aaa'))).toBe(true)
expect(existsSync(join(root, 'relay-0.1.0+aaa'))).toBe(true)
})
it('holds a runtime a process runs through another spelling of a symlinked home', async () => {
runtime('a')
runtime('b')
const held = runtime('e')
const alias = mkdtempSync(join(tmpdir(), 'runtime-store-alias-'))
rmSync(alias, { recursive: true })
symlinkSync(home, alias)
try {
running = spawn(join(held, 'bin', 'node'), [], { stdio: 'ignore' })
await once(running, 'spawn')
const result = await gcRemoteNodeRuntimeStore(conn, host, alias, { currentPins: [sha('a')] })
expect(result).toMatchObject({ state: 'collected', removed: [] })
expect(existsSync(join(held, 'bin', 'node'))).toBe(true)
} finally {
rmSync(alias, { force: true })
}
})
it('keeps every runtime when a reference marker cannot be attributed', async () => {
runtime('a')
runtime('b')
runtime('e')
writeFileSync(join(versionDir('server-0.2.0+ccc'), '.runtime-node'), 'not-a-sha\n')
const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] })
expect(result.state).toBe('skipped')
expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`))).toBe(true)
})
it('restores a runtime that gained a reference after the rename', async () => {
runtime('a')
runtime('b')
runtime('e')
let inventories = 0
mockExec.mockImplementation(async (_conn, command) => {
if (command.includes('__ORCA_RUNTIME_STORE__OK') && ++inventories === 2) {
writeFileSync(join(versionDir('orcad-0.3.0+ddd'), '.runtime-node'), `${sha('e')}\n`)
}
return execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' })
})
const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] })
expect(result).toMatchObject({ state: 'collected', removed: [] })
expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`, 'bin', 'node'))).toBe(true)
})
it('is inert when the store does not exist', async () => {
rmSync(join(root, 'runtimes'), { recursive: true })
const out = execFileSync('/bin/sh', ['-c', runtimeStoreInventoryCommand(host, home)], {
encoding: 'utf8'
})
expect(parseRuntimeStoreInventory(out)?.entries).toEqual([])
})
})
describe('gcRemoteNodeRuntimeStore termination', () => {
beforeEach(() => {
mockExec.mockReset()
})
it('rethrows an unconfirmed inventory termination', async () => {
const error = Object.assign(new Error('SSH command timed out'), {
sshChannelCloseConfirmed: false
})
mockExec.mockRejectedValueOnce(error)
await expect(
gcRemoteNodeRuntimeStore(conn, host, '/home/u', { currentPins: [sha('a')] })
).rejects.toBe(error)
})
it('skips Windows hosts without running anything', async () => {
const result = await gcRemoteNodeRuntimeStore(
conn,
getRemoteHostPlatform('win32-x64'),
'C:/Users/u',
{
currentPins: [sha('a')]
}
)
expect(result.state).toBe('skipped')
expect(mockExec).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,249 @@
/**
* Collects the shared `~/.orca-remote/runtimes/node-<sha256>/` store (design D5 GC).
*
* A runtime is deleted only when all of these hold: no retained directory references it
* (`.runtime-node` or `.runtime-ref-node-<sha>`), it is neither a pin this client runs nor the
* newest other verified runtime (keep two), and a process check ran and found nothing executing
* from it. Process evidence can only add holds; a scan that could not run keeps everything.
*
* Legacy `relay-*` / `orcad-*` directories are read for references and reported as
* diagnostics, never deleted here (design D10 two-step hand-over).
*/
import { randomInt } from 'node:crypto'
import { ORCAD_RUNTIMES_DIRNAME } from '../../shared/orcad-artifacts'
import type { SshConnection } from './ssh-connection'
import { RELAY_REMOTE_DIR } from './relay-protocol'
import { inventoryRemoteInstallDirs } from './remote-install-model'
import {
parseRuntimeStoreInventory,
RUNTIME_STORE_ENTRY_NAME,
RUNTIME_STORE_TOMBSTONE_NAME,
RUNTIME_STORE_TOMBSTONE_PREFIX,
runtimeStoreInventoryCommand,
type RuntimeStoreInventory
} from './remote-node-runtime-store-inventory'
import { execCommand } from './ssh-relay-deploy-helpers'
import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
import {
moveRemoteTreeCommand,
removeRemoteTreeCommand,
restoreRemoteTreeCommand
} from './ssh-remote-commands'
import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
const MAX_REMOVALS_PER_PASS = 8
const ABANDONED_TOMBSTONE_MS = 30 * 60_000
export type RuntimeStoreGcPlan = {
/** `node-<sha>` entries to rename away and delete. */
remove: string[]
/** Abandoned tombstones whose runtime is still unwanted. */
purgeTombstones: string[]
kept: string[]
}
/** Why a sha must stay, or null when nothing holds it. */
function holdReason(
sha: string,
inventory: RuntimeStoreInventory,
pins: ReadonlySet<string>
): string | null {
if (!inventory.processCheckRan) {
return 'process check unavailable'
}
if (pins.has(sha)) {
return 'pinned'
}
if (inventory.referenced.has(sha)) {
return 'referenced'
}
if (inventory.held.has(sha)) {
return 'in use by a process'
}
return null
}
export function planRuntimeStoreGc(
inventory: RuntimeStoreInventory,
currentPins: readonly string[],
now: number = Date.now()
): RuntimeStoreGcPlan {
const pins = new Set(currentPins)
// Keep two: the pin this client runs and the newest other verified runtime (the previous pin).
const previous = inventory.verifiedNewestFirst
.map((name) => RUNTIME_STORE_ENTRY_NAME.exec(name)?.[1])
.find((sha): sha is string => !!sha && !pins.has(sha))
if (previous) {
pins.add(previous)
}
const verified = new Set(inventory.verifiedNewestFirst)
const plan: RuntimeStoreGcPlan = { remove: [], purgeTombstones: [], kept: [] }
for (const name of inventory.entries) {
const entry = RUNTIME_STORE_ENTRY_NAME.exec(name)
if (entry) {
// Unverified means mid-promotion or torn; the installer, not GC, owns that state.
const idle = verified.has(name) && holdReason(entry[1], inventory, pins) === null
if (idle && plan.remove.length < MAX_REMOVALS_PER_PASS) {
plan.remove.push(name)
} else {
plan.kept.push(name)
}
continue
}
const tombstone = RUNTIME_STORE_TOMBSTONE_NAME.exec(name)
if (
tombstone &&
now - Number(tombstone[2]) >= ABANDONED_TOMBSTONE_MS &&
holdReason(tombstone[1], inventory, pins) === null
) {
plan.purgeTombstones.push(name)
}
}
return plan
}
export type RuntimeStoreGcResult =
| { state: 'skipped'; reason: string }
| { state: 'collected'; removed: string[]; kept: string[]; legacyDirs: string[] }
function exec(conn: SshConnection, command: string, signal?: AbortSignal): Promise<string> {
return execCommand(conn, command, { wrapCommand: true, signal })
}
async function readInventory(
conn: SshConnection,
host: RemoteHostPlatform,
remoteHome: string,
signal?: AbortSignal
): Promise<RuntimeStoreInventory | null> {
try {
return parseRuntimeStoreInventory(
await exec(conn, runtimeStoreInventoryCommand(host, remoteHome), signal)
)
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
return null
}
}
/**
* One pass over the runtime store. Confirmed failures keep the runtime and end quietly; an
* unconfirmed SSH termination is rethrown so the caller stops its cleanup chain.
*/
export async function gcRemoteNodeRuntimeStore(
conn: SshConnection,
host: RemoteHostPlatform,
remoteHome: string,
options: { currentPins: readonly string[]; signal?: AbortSignal }
): Promise<RuntimeStoreGcResult> {
if (isWindowsRemoteHost(host)) {
return { state: 'skipped', reason: 'Windows hosts have no managed runtime store yet' }
}
const inventory = await readInventory(conn, host, remoteHome, options.signal)
if (!inventory) {
return { state: 'skipped', reason: 'runtime store inventory was unverifiable' }
}
const legacy = inventoryRemoteInstallDirs(inventory.dirNames)
const legacyDirs = [...legacy.relay, ...legacy.orcad]
const plan = planRuntimeStoreGc(inventory, options.currentPins)
const store = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_RUNTIMES_DIRNAME)
const removed: string[] = []
const kept = [...plan.kept]
for (const name of plan.purgeTombstones) {
if (await removeTree(conn, host, joinRemotePath(host, store, name), options.signal)) {
removed.push(name)
}
}
for (const name of plan.remove) {
const sha = RUNTIME_STORE_ENTRY_NAME.exec(name)?.[1] ?? ''
const entryDir = joinRemotePath(host, store, name)
const tombstone = joinRemotePath(
host,
store,
`${RUNTIME_STORE_TOMBSTONE_PREFIX}${name}.${randomInt(1, 2 ** 47)}.${Date.now()}`
)
if (!(await moveTree(conn, host, entryDir, tombstone, options.signal))) {
kept.push(name)
continue
}
// Why recheck after the rename: an installer that saw this runtime present may be writing
// its reference now; restoring is the only outcome that leaves its slot launchable.
const recheck = await readInventory(conn, host, remoteHome, options.signal).catch(
async (err: unknown) => {
await restoreTree(conn, host, tombstone, entryDir, options.signal).catch(() => {})
throw err
}
)
if (!recheck || holdReason(sha, recheck, new Set(options.currentPins)) !== null) {
await restoreTree(conn, host, tombstone, entryDir, options.signal)
kept.push(name)
continue
}
if (await removeTree(conn, host, tombstone, options.signal)) {
removed.push(name)
} else {
kept.push(name)
}
}
if (removed.length > 0) {
const legacyNote =
legacyDirs.length > 0
? `; legacy install dirs left for the migration sweep: ${legacyDirs.join(', ')}`
: ''
console.log(`[runtime-store] GC: removed ${removed.join(', ')}${legacyNote}`)
}
return { state: 'collected', removed, kept, legacyDirs }
}
async function moveTree(
conn: SshConnection,
host: RemoteHostPlatform,
source: string,
destination: string,
signal?: AbortSignal
): Promise<boolean> {
try {
return (
(await exec(conn, moveRemoteTreeCommand(host, source, destination), signal)).trim() ===
'MOVED'
)
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
return false
}
}
async function restoreTree(
conn: SshConnection,
host: RemoteHostPlatform,
tombstone: string,
entryDir: string,
signal?: AbortSignal
): Promise<void> {
await exec(conn, restoreRemoteTreeCommand(host, tombstone, entryDir), signal).catch((err) => {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
})
}
async function removeTree(
conn: SshConnection,
host: RemoteHostPlatform,
path: string,
signal?: AbortSignal
): Promise<boolean> {
try {
await exec(conn, removeRemoteTreeCommand(host, path), signal)
return true
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
return false
}
}
@@ -0,0 +1,140 @@
/**
* One read-only pass over `~/.orca-remote/` answering what the runtime store GC needs: store
* entries, which runtimes are referenced, verified order, and which a running process holds.
*/
import {
ORCAD_NODE_RUNTIME_DIR_PREFIX,
ORCAD_NODE_RUNTIME_MARKER_FILENAME,
ORCAD_RUNTIMES_DIRNAME
} from '../../shared/orcad-artifacts'
import { shellEscape } from './ssh-connection-utils'
import { RELAY_REMOTE_DIR } from './relay-protocol'
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
/** A version dir names a runtime it needs with an empty file of this prefix + sha (design D5). */
export const RUNTIME_REF_NODE_PREFIX = '.runtime-ref-node-'
export const RUNTIME_STORE_TOMBSTONE_PREFIX = '.gc-tombstone-'
const INVENTORY_OK = '__ORCA_RUNTIME_STORE__OK'
const REFS_ERR = '__ORCA_RUNTIME_STORE__REFS_ERR'
const MAX_DIRS = 512
const SHA256 = /^[0-9a-f]{64}$/
export const RUNTIME_STORE_ENTRY_NAME = new RegExp(
`^${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})$`
)
export const RUNTIME_STORE_TOMBSTONE_NAME = new RegExp(
`^${RUNTIME_STORE_TOMBSTONE_PREFIX.replace(/\./g, '\\.')}${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})\\.[0-9]+\\.([0-9]+)$`
)
const HELD_PATH = new RegExp(
`/${ORCAD_RUNTIMES_DIRNAME}/(?:${RUNTIME_STORE_TOMBSTONE_PREFIX.replace(/\./g, '\\.')})?${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})[./]`
)
export type RuntimeStoreInventory = {
/** Store entry names: `node-<sha>` and this GC's tombstones. */
entries: string[]
/** `node-<sha>` names with `.verified`, newest first. */
verifiedNewestFirst: string[]
referenced: Set<string>
held: Set<string>
/** False when neither `ps` nor `/proc` answered; nothing may then be called idle. */
processCheckRan: boolean
/** Other `~/.orca-remote/` directory names, for legacy diagnostics. */
dirNames: string[]
}
export function runtimeStoreInventoryCommand(host: RemoteHostPlatform, remoteHome: string): string {
const root = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR)
const refPrefix = RUNTIME_REF_NODE_PREFIX
return [
`root=${shellEscape(root)}`,
`rt="$root"/${ORCAD_RUNTIMES_DIRNAME}`,
`[ -d "$rt" ] || { printf '%s\\n' ${INVENTORY_OK}; exit 0; }`,
`[ -r "$root" ] && [ -x "$root" ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
'n=0',
// Why every sibling and not a prefix: a directory this client does not recognise may still
// be a newer Orca's install that names a runtime.
'for d in "$root"/* "$root"/.[!.]*; do',
' [ -d "$d" ] || continue',
' name=${d##*/}',
` [ "$name" = ${ORCAD_RUNTIMES_DIRNAME} ] && continue`,
` [ -r "$d" ] && [ -x "$d" ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
' n=$((n+1))',
` [ "$n" -le ${MAX_DIRS} ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
` printf 'DIR %s\\n' "$name"`,
` if [ -e "$d"/${ORCAD_NODE_RUNTIME_MARKER_FILENAME} ]; then`,
` sha=$(cat "$d"/${ORCAD_NODE_RUNTIME_MARKER_FILENAME}) || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
` printf 'REF %s\\n' "$sha"`,
' fi',
` for f in "$d"/${refPrefix}*; do`,
' [ -e "$f" ] || continue',
` printf 'REF %s\\n' "\${f##*/${refPrefix}}"`,
' done',
'done',
`for e in "$rt"/${ORCAD_NODE_RUNTIME_DIR_PREFIX}* "$rt"/${RUNTIME_STORE_TOMBSTONE_PREFIX}${ORCAD_NODE_RUNTIME_DIR_PREFIX}*; do`,
` [ -d "$e" ] && printf 'ENTRY %s\\n' "\${e##*/}"`,
'done',
`set -- "$rt"/${ORCAD_NODE_RUNTIME_DIR_PREFIX}*/.verified`,
'if [ -e "$1" ]; then',
` order=$(ls -1t -- "$@") || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
` printf '%s\\n' "$order" | while IFS= read -r v; do v=\${v%/.verified}; printf 'VERIFIED %s\\n' "\${v##*/}"; done`,
'fi',
// Process checks only add holds, so an unmatched `grep` is not an error. Why not "$rt/":
// /proc exe and argv may name the store through another spelling of a symlinked home.
'if ps_out=$(ps -e -o args= 2>/dev/null); then',
" printf 'PROCESS_CHECK ps\\n'",
` printf '%s\\n' "$ps_out" | grep -F -- /${ORCAD_RUNTIMES_DIRNAME}/ | sed 's/^/HOLD /'`,
'fi',
'if [ -n "$(readlink /proc/self/exe 2>/dev/null)" ]; then',
" printf 'PROCESS_CHECK proc\\n'",
' for p in /proc/[0-9]*/exe; do',
' t=$(readlink "$p" 2>/dev/null) || continue',
` case "$t" in */${ORCAD_RUNTIMES_DIRNAME}/*) printf 'HOLD %s\\n' "$t";; esac`,
' done',
'fi',
`printf '%s\\n' ${INVENTORY_OK}`
].join('\n')
}
/** Null when the host could not produce a complete inventory; that keeps every runtime. */
export function parseRuntimeStoreInventory(output: string): RuntimeStoreInventory | null {
const lines = output.split(/\r?\n/).map((line) => line.trim())
if (!lines.includes(INVENTORY_OK) || lines.includes(REFS_ERR)) {
return null
}
const inventory: RuntimeStoreInventory = {
entries: [],
verifiedNewestFirst: [],
referenced: new Set(),
held: new Set(),
processCheckRan: false,
dirNames: []
}
for (const line of lines) {
const space = line.indexOf(' ')
const tag = space === -1 ? line : line.slice(0, space)
const value = space === -1 ? '' : line.slice(space + 1).trim()
if (tag === 'REF') {
// An unattributable reference could name any runtime, so it stops the pass.
if (!SHA256.test(value)) {
return null
}
inventory.referenced.add(value)
} else if (
tag === 'ENTRY' &&
(RUNTIME_STORE_ENTRY_NAME.test(value) || RUNTIME_STORE_TOMBSTONE_NAME.test(value))
) {
inventory.entries.push(value)
} else if (tag === 'VERIFIED' && RUNTIME_STORE_ENTRY_NAME.test(value)) {
inventory.verifiedNewestFirst.push(value)
} else if (tag === 'HOLD') {
const sha = HELD_PATH.exec(value)?.[1]
if (sha) {
inventory.held.add(sha)
}
} else if (tag === 'PROCESS_CHECK') {
inventory.processCheckRan = true
} else if (tag === 'DIR' && value) {
inventory.dirNames.push(value)
}
}
return inventory
}
@@ -61,6 +61,7 @@ vi.mock('./ssh-connection-utils', () => ({
import { deployAndLaunchRelay } from './ssh-relay-deploy'
import { execCommand } from './ssh-relay-deploy-helpers'
import type { SshConnection } from './ssh-connection'
import { REMOTE_INSTALL_ORDER_OK } from './remote-install-previous-version'
function makeMockConnection(): SshConnection {
return {
@@ -150,6 +151,12 @@ describe('cross-version isolation', () => {
if (command.includes('test -S') && command.includes('echo ALIVE || echo DEAD')) {
return Promise.resolve('DEAD')
}
if (command.includes(REMOTE_INSTALL_ORDER_OK)) {
// A newer v0 is the previous build, so v1 is kept only by its live socket.
return Promise.resolve(
`relay-0.1.0+222222222222\nrelay-0.1.0+000000000000\nrelay-0.1.0+111111111111\n${REMOTE_INSTALL_ORDER_OK}`
)
}
if (command.includes('__ORCA_RELAY_GC_FIND_STATUS__')) {
return Promise.resolve('relay-0.1.0+111111111111\nrelay-0.1.0+222222222222\n')
}
+1
View File
@@ -693,6 +693,7 @@ async function deployAndLaunchRelayAttempt(
gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, {
windowsNodePath: launched.nodePath,
windowsSockNames: [relaySocketNameForInstanceId(relayInstanceId)],
nodePath: launched.nodePath,
// Why pin rather than rely on the symlink alone: a deploy that fell back to a
// per-directory install has no reference to show, and its key must still survive.
nativeDepsCacheKeys: [
+2 -2
View File
@@ -69,7 +69,7 @@ const PROBE_END = 'ORCA-INCUMBENT-END'
const CONNECT_PROBE_TIMEOUT_MS = 1000
// Why ES5 syntax: nodePath may be a host-resolved system node, not the bundled one.
const CONNECT_PROBE_JS = [
export const RELAY_CONNECT_PROBE_JS = [
'var s=require("net").connect(process.argv[1]);',
'var done=false;',
'function say(v){if(done)return;done=true;try{s.destroy()}catch(e){};',
@@ -100,7 +100,7 @@ export function relayEndpointIncumbentProbeCommand(nodePath: string, sockPath: s
`printf '%s\\n' ${shellEscape(PROBE_BEGIN)}`,
'if [ -S "$sock" ]; then',
" printf 'PRESENT=yes\\n'",
` listen=$("$node" -e ${shellEscape(CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`,
` listen=$("$node" -e ${shellEscape(RELAY_CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`,
' [ -n "$listen" ] || listen=unknown',
'else',
" printf 'PRESENT=no\\n'",
@@ -15,6 +15,11 @@ vi.mock('./ssh-connection-utils', () => ({
shellEscape: (s: string) => `'${s}'`
}))
// The previous-build pin has its own tests; here it names a build that is never a candidate.
vi.mock('./remote-install-previous-version', () => ({
findPreviousRemoteInstall: vi.fn().mockResolvedValue({ state: 'ok', dirName: 'relay-0.1.0+fff' })
}))
import { existsSync, readFileSync } from 'node:fs'
import {
readLocalFullVersion,
+1 -1
View File
@@ -201,7 +201,7 @@ export function probeFileExistsCommand(host: RemoteHostPlatform, remotePath: str
)
}
type WindowsRelayLivenessOptions = {
export type WindowsRelayLivenessOptions = {
nodePath: string
pipePaths: string[]
}
+2
View File
@@ -14,6 +14,7 @@ import {
restrictWindowsRelayEndpointCredential
} from './relay-endpoint-credential-publication'
import { SKILL_RELAY_CAPABILITIES } from './skill-install-handler'
import { publishRelayPid } from './relay-pid-publication'
export async function runRelayDaemon(options: RelayLaunchOptions): Promise<void> {
if (options.detached && options.logFile) {
@@ -105,6 +106,7 @@ export async function runRelayDaemon(options: RelayLaunchOptions): Promise<void>
// exits inside start() and never reaches the credential file, so racing starters cannot
// rotate the secret a surviving daemon enforces.
await reconnectListener.start()
publishRelayPid()
reconnectListener.setEndpointCredential(publishRelayEndpointCredential(options.credentialFile))
agentHooks.publishEndpointFile()
} catch (error) {
+41
View File
@@ -0,0 +1,41 @@
import { existsSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
import { rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { RELAY_PID_FILENAME, RELAY_VERSION_FILENAME } from '../shared/relay-artifacts'
import { publishRelayPid } from './relay-pid-publication'
describe('publishRelayPid', () => {
const directories: string[] = []
afterEach(async () => {
await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true })))
})
function versionDir(installed: boolean): string {
const dir = mkdtempSync(join(tmpdir(), 'relay-pid-'))
directories.push(dir)
writeFileSync(join(dir, 'relay.js'), '')
if (installed) {
writeFileSync(join(dir, RELAY_VERSION_FILENAME), '0.1.0+abc\n')
}
return dir
}
it('records the daemon PID beside an installed relay.js', () => {
const dir = versionDir(true)
publishRelayPid(join(dir, 'relay.js'))
expect(readFileSync(join(dir, RELAY_PID_FILENAME), 'utf8')).toBe(`${process.pid}\n`)
expect(readdirSync(dir).filter((name) => name.endsWith('.tmp'))).toEqual([])
})
it('leaves a build dir without .version untouched', () => {
const dir = versionDir(false)
publishRelayPid(join(dir, 'relay.js'))
expect(existsSync(join(dir, RELAY_PID_FILENAME))).toBe(false)
})
it('does not throw when the entry cannot be resolved', () => {
expect(() => publishRelayPid(join(tmpdir(), 'missing-relay-dir', 'relay.js'))).not.toThrow()
})
})
+39
View File
@@ -0,0 +1,39 @@
import { existsSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { RELAY_PID_FILENAME, RELAY_VERSION_FILENAME } from '../shared/relay-artifacts'
import { relayLogLine } from './relay-diagnostic-log'
/**
* Record this daemon's PID in its version dir once it owns its socket, so version GC can
* tell a stale socket from a live daemon without connecting to one that is about to idle.
*/
export function publishRelayPid(entry: string | undefined = process.argv[1]): void {
if (!entry) {
return
}
let temporary: string | null = null
try {
const versionDir = dirname(realpathSync(entry))
// Why: only an installed version dir carries `.version`; a dev or test build dir is not GC'd.
if (!existsSync(join(versionDir, RELAY_VERSION_FILENAME))) {
return
}
temporary = join(versionDir, `${RELAY_PID_FILENAME}.${process.pid}.tmp`)
writeFileSync(temporary, `${process.pid}\n`, { mode: 0o600 })
// Why rename: GC must never read a half-written PID as a different, dead process.
renameSync(temporary, join(versionDir, RELAY_PID_FILENAME))
temporary = null
} catch (error) {
relayLogLine(
`[relay] Could not record relay PID: ${error instanceof Error ? error.message : String(error)}`
)
} finally {
if (temporary) {
try {
rmSync(temporary, { force: true })
} catch {
// A leftover temp file is inert; GC reads only the renamed name.
}
}
}
}
+3
View File
@@ -88,6 +88,9 @@ export const RELAY_VERSION_FILENAME = '.version'
/** Written last by the installer; its absence means a torn install. */
export const RELAY_INSTALL_COMPLETE_FILENAME = '.install-complete'
/** PID of the last relay daemon that bound a socket from this version dir; GC liveness evidence. */
export const RELAY_PID_FILENAME = '.relay-pid'
/** Artifacts every relay must have; the remote install probe requires each one. */
export function relayArtifactFilenames(isWindows: boolean): string[] {
return RELAY_ARTIFACTS.filter(