mirror of
https://github.com/stablyai/orca.git
synced 2026-10-07 00:02:29 +00:00
fix(ssh): collect relay versions only when provably exited; runtimes/ store GC (#24130)
* fix(ssh): relay version GC deletes only on an exited verdict and keeps the previous build The relay records .relay-pid in its version dir once it owns its socket. GC calls a relay version dir exited only when that PID is provably dead and every relay-*.sock refuses a connection; a dir without a PID file keeps the test -S rule. The most recently completed other relay build is pinned like orcad's rollback target. Design D5 GC liveness. * feat(ssh): collect the shared runtimes/ Node store and give it its own owner runtimes/ gets its own owner in the install model, so no version-dir GC (new or old clients, whose listings are prefix-scoped) can list or delete it. A store pass removes node-<sha> only when no retained dir references it, it is neither a current pin nor the newest other verified runtime, and a ps or /proc check ran and found no process using it. Legacy relay-*/orcad-* dirs are read for references and reported as diagnostics only (design D10 two-step). Wired behind orcad GC's nodeRuntimePins. * fix(ssh): runtime store process check holds runtimes reached through a symlinked home /proc exe resolves symlinks and argv keeps whatever spelling launched the runtime, so filtering on the exact $root path missed in-use runtimes on hosts like /home -> /var/home. Filter on the store segment instead; the parser already attributes holds root-agnostically. * test(ssh): wait for the holder process to spawn instead of a fixed delay --------- Co-authored-by: m4air <m4air@m4airs-Air.localdomain>
This commit is contained in:
@@ -201,4 +201,24 @@ describe('orcad GC', () => {
|
||||
|
||||
expect(removed).toEqual(['orcad-0.0.9+dead'])
|
||||
})
|
||||
|
||||
it('collects the runtime store only when the caller names its runtime pins', async () => {
|
||||
const removed: string[] = []
|
||||
scriptHost({ listing: [], removed })
|
||||
const options = {
|
||||
conn,
|
||||
host,
|
||||
remoteHome: '/home/u',
|
||||
currentDirAbsPath: '/home/u/.orca-remote/orcad-0.2.0+bb',
|
||||
record: emptyOrcadActivationRecord()
|
||||
}
|
||||
const inventories = (): number =>
|
||||
mockExec.mock.calls.filter(([, command]) => String(command).includes('RUNTIME_STORE')).length
|
||||
|
||||
await gcOldOrcadVersions(options)
|
||||
expect(inventories()).toBe(0)
|
||||
|
||||
await gcOldOrcadVersions({ ...options, nodeRuntimePins: ['a'.repeat(64)] })
|
||||
expect(inventories()).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
/**
|
||||
* orcad's garbage collection, and the half of §06 falsifier 1 that says who owns it.
|
||||
* orcad's garbage collection, and who owns it (design D10; to be tracked in
|
||||
* docs/reference/remote-server-install-model.md).
|
||||
*
|
||||
* **Each model GCs only its own namespace, permanently.** orcad removes `orcad-<v>/`
|
||||
* directories; the relay removes `relay-<v>/` directories; neither ever removes the other's,
|
||||
* and no plan item makes one the winner. That is not a migration compromise — the two models
|
||||
* serve different users on the same machine (SSH target vs paired peer), so there is no
|
||||
* moment at which one of them is entitled to clean up after the other. A pass that deleted
|
||||
* the sibling's tree would be reaching across the execution boundary the whole design exists
|
||||
* to keep intact.
|
||||
* **Each model GCs only its own namespace.** orcad removes `orcad-<v>/` directories; the relay
|
||||
* removes `relay-<v>/` directories; neither ever removes the other's. The converged server's
|
||||
* migration sweep takes over legacy directories only in the release after it has listed them
|
||||
* as diagnostics, and only on an `exited` verdict. A pass that deleted the sibling's tree
|
||||
* would be reaching across the execution boundary the whole design exists to keep intact.
|
||||
*
|
||||
* On top of the ownership rule, orcad pins three directories that are idle-looking but
|
||||
* load-bearing: the active version, the rollback target, and whichever version the LIVE
|
||||
@@ -25,6 +24,7 @@ import {
|
||||
parseOrcadLiveness
|
||||
} from './orcad-remote-launch'
|
||||
import type { RemoteHostPlatform } from './ssh-remote-platform'
|
||||
import { gcRemoteNodeRuntimeStore } from './remote-node-runtime-store-gc'
|
||||
|
||||
export type OrcadGcOptions = {
|
||||
conn: SshConnection
|
||||
@@ -41,6 +41,11 @@ export type OrcadGcOptions = {
|
||||
* would remove the tree under a running process.
|
||||
*/
|
||||
liveDaemonVersion?: string | null
|
||||
/**
|
||||
* executableSha256 of every runtime pin this client runs. Also the gate for the shared
|
||||
* runtime store pass: without it this client cannot say which runtime is current.
|
||||
*/
|
||||
nodeRuntimePins?: readonly string[]
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
@@ -75,4 +80,11 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise<void>
|
||||
}
|
||||
}
|
||||
)
|
||||
// Why after the version pass: removing version dirs is what drops their runtime references.
|
||||
if (options.nodeRuntimePins?.length) {
|
||||
await gcRemoteNodeRuntimeStore(options.conn, options.host, options.remoteHome, {
|
||||
currentPins: options.nodeRuntimePins,
|
||||
signal: options.signal
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
/** Runs the generated POSIX liveness probe through a real `/bin/sh` against real sockets. */
|
||||
import { execFileSync, spawnSync } from 'node:child_process'
|
||||
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { createServer, type Server } from 'node:net'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { RELAY_PID_FILENAME } from '../../shared/relay-artifacts'
|
||||
import {
|
||||
parseRelayVersionDirLiveness,
|
||||
relayVersionDirLivenessCommand
|
||||
} from './relay-version-dir-liveness'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const host = getRemoteHostPlatform('linux-x64')
|
||||
const posixOnly = process.platform === 'win32' ? describe.skip : describe
|
||||
|
||||
function probe(dir: string, nodePath: string | undefined = process.execPath): string {
|
||||
return execFileSync('/bin/sh', ['-c', relayVersionDirLivenessCommand(host, dir, { nodePath })], {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
}
|
||||
|
||||
/** A socket inode left by a SIGKILLed listener: connect is refused. Returns the dead PID. */
|
||||
function leaveStaleSocket(sockPath: string): number {
|
||||
const child = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
'-e',
|
||||
'require("net").createServer().listen(process.argv[1],()=>{' +
|
||||
'process.stdout.write(String(process.pid));process.kill(process.pid,"SIGKILL")})',
|
||||
sockPath
|
||||
],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
return Number.parseInt(child.stdout, 10)
|
||||
}
|
||||
|
||||
posixOnly('relayVersionDirLivenessCommand (real shell)', () => {
|
||||
const dirs: string[] = []
|
||||
const servers: Server[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
servers.splice(0).map((server) => new Promise<void>((done) => server.close(() => done())))
|
||||
)
|
||||
for (const dir of dirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function versionDir(): string {
|
||||
// Short base: sun_path caps socket paths near 104 bytes on macOS.
|
||||
const dir = mkdtempSync(join('/tmp', 'rvl-'))
|
||||
dirs.push(dir)
|
||||
return dir
|
||||
}
|
||||
|
||||
it('is exited for a stale socket whose recorded PID is dead', () => {
|
||||
const dir = versionDir()
|
||||
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
|
||||
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
|
||||
|
||||
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('exited')
|
||||
})
|
||||
|
||||
it('is live for a stale socket whose recorded PID is still running', () => {
|
||||
const dir = versionDir()
|
||||
leaveStaleSocket(join(dir, 'relay-a.sock'))
|
||||
writeFileSync(join(dir, RELAY_PID_FILENAME), `${process.pid}\n`)
|
||||
|
||||
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live')
|
||||
})
|
||||
|
||||
it('keeps the test -S rule for a refused socket without a PID file', () => {
|
||||
const dir = versionDir()
|
||||
leaveStaleSocket(join(dir, 'relay-a.sock'))
|
||||
|
||||
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live')
|
||||
})
|
||||
|
||||
it('is exited without a PID file only when no socket is left', () => {
|
||||
expect(parseRelayVersionDirLiveness(probe(versionDir()))).toBe('exited')
|
||||
})
|
||||
|
||||
it('is live when another relay of this build still accepts on its socket', async () => {
|
||||
const dir = versionDir()
|
||||
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
|
||||
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
|
||||
const server = createServer()
|
||||
servers.push(server)
|
||||
await new Promise<void>((done) => server.listen(join(dir, 'relay-b.sock'), done))
|
||||
|
||||
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('live')
|
||||
})
|
||||
|
||||
it('is unverifiable when the connect probe times out', () => {
|
||||
const dir = versionDir()
|
||||
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
|
||||
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
|
||||
// Stands in for node: the connect probe prints `unknown` when its timer fires first.
|
||||
const timedOutNode = join(dir, 'node')
|
||||
writeFileSync(timedOutNode, '#!/bin/sh\nprintf unknown\n')
|
||||
chmodSync(timedOutNode, 0o755)
|
||||
|
||||
expect(parseRelayVersionDirLiveness(probe(dir, timedOutNode))).toBe('unverifiable')
|
||||
})
|
||||
|
||||
it('is unverifiable when no Node is available to test a leftover socket', () => {
|
||||
const dir = versionDir()
|
||||
const deadPid = leaveStaleSocket(join(dir, 'relay-a.sock'))
|
||||
writeFileSync(join(dir, RELAY_PID_FILENAME), `${deadPid}\n`)
|
||||
|
||||
expect(parseRelayVersionDirLiveness(probe(dir, ''))).toBe('unverifiable')
|
||||
})
|
||||
|
||||
it('is unverifiable for an unreadable PID record', () => {
|
||||
const dir = versionDir()
|
||||
writeFileSync(join(dir, RELAY_PID_FILENAME), 'not-a-pid\n')
|
||||
|
||||
expect(parseRelayVersionDirLiveness(probe(dir))).toBe('unverifiable')
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseRelayVersionDirLiveness', () => {
|
||||
it('maps the Windows pipe vocabulary and treats anything else as unverifiable', () => {
|
||||
expect(parseRelayVersionDirLiveness('ALIVE')).toBe('live')
|
||||
expect(parseRelayVersionDirLiveness('WAITING')).toBe('exited')
|
||||
expect(parseRelayVersionDirLiveness('DEAD\n')).toBe('exited')
|
||||
expect(parseRelayVersionDirLiveness('')).toBe('unverifiable')
|
||||
expect(parseRelayVersionDirLiveness('UNKNOWN')).toBe('unverifiable')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Whether a relay version directory is still in use, answered with the execution-boundary
|
||||
* vocabulary (docs/reference/ssh-execution-boundary.md): `live` / `unverifiable` / `exited`.
|
||||
*
|
||||
* Design D5: a directory is `exited` only when its recorded `.relay-pid` is provably dead AND
|
||||
* every `relay-*.sock` in it refuses a connection. The PID is checked first so a live daemon
|
||||
* about to idle is never connected to (a connection would cancel its grace timer). A directory
|
||||
* with no PID file was last used by a relay that predates it and keeps the `test -S` rule.
|
||||
*/
|
||||
import { RELAY_PID_FILENAME } from '../../shared/relay-artifacts'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { posixProcessAliveShellFunction } from './orcad-remote-host-support'
|
||||
import { RELAY_CONNECT_PROBE_JS, type RelayEndpointVerdict } from './ssh-relay-endpoint-incumbent'
|
||||
import { relayLivenessProbeCommand, type WindowsRelayLivenessOptions } from './ssh-remote-commands'
|
||||
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
export function relayVersionDirLivenessCommand(
|
||||
host: RemoteHostPlatform,
|
||||
dir: string,
|
||||
options: { nodePath?: string; windows?: WindowsRelayLivenessOptions } = {}
|
||||
): string {
|
||||
if (isWindowsRemoteHost(host)) {
|
||||
return relayLivenessProbeCommand(host, dir, options.windows)
|
||||
}
|
||||
return [
|
||||
`dir=${shellEscape(dir)}`,
|
||||
`node=${shellEscape(options.nodePath ?? '')}`,
|
||||
`pid_file="$dir"/${RELAY_PID_FILENAME}`,
|
||||
'socks=',
|
||||
'for f in "$dir"/relay-*.sock "$dir"/relay.sock; do [ -S "$f" ] && socks=yes; done',
|
||||
'if [ ! -e "$pid_file" ]; then',
|
||||
' if [ -n "$socks" ]; then echo LIVE; else echo EXITED; fi',
|
||||
' exit 0',
|
||||
'fi',
|
||||
// Prints UNKNOWN and exits when kill -0 fails for any reason but "No such process".
|
||||
posixProcessAliveShellFunction({ refuseUnverifiable: true }),
|
||||
'pid=$(cat "$pid_file" 2>/dev/null) || { echo UNVERIFIABLE; exit 0; }',
|
||||
'case "$pid" in "" | *[!0-9]*) echo UNVERIFIABLE; exit 0;; esac',
|
||||
'if orcad_alive "$pid"; then echo LIVE; exit 0; fi',
|
||||
'for f in "$dir"/relay-*.sock "$dir"/relay.sock; do',
|
||||
' [ -S "$f" ] || continue',
|
||||
' [ -n "$node" ] || { echo UNVERIFIABLE; exit 0; }',
|
||||
// Another relay of this build may share the dir under its own socket; only a refusal clears it.
|
||||
` r=$("$node" -e ${shellEscape(RELAY_CONNECT_PROBE_JS)} "$f" 2>/dev/null) || r=unknown`,
|
||||
' case "$r" in',
|
||||
' refused | absent) ;;',
|
||||
' accepted) echo LIVE; exit 0;;',
|
||||
' *) echo UNVERIFIABLE; exit 0;;',
|
||||
' esac',
|
||||
'done',
|
||||
'echo EXITED'
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
export function parseRelayVersionDirLiveness(output: string): RelayEndpointVerdict {
|
||||
const token = output.trim().split('\n').pop()?.trim() ?? ''
|
||||
// ALIVE / DEAD / WAITING are the Windows pipe probe's vocabulary.
|
||||
if (token === 'LIVE' || token === 'ALIVE') {
|
||||
return 'live'
|
||||
}
|
||||
if (token === 'EXITED' || token === 'DEAD' || token === 'WAITING') {
|
||||
return 'exited'
|
||||
}
|
||||
return 'unverifiable'
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { gcOldRelayVersions } from './remote-install-gc'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { gcRelayNativeDepsCache } from './ssh-relay-native-deps-cache-gc'
|
||||
import { gcRemoteRipgrepCache } from './ssh-relay-ripgrep-cache-gc'
|
||||
import { REMOTE_INSTALL_ORDER_OK } from './remote-install-previous-version'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock.
|
||||
@@ -41,8 +42,17 @@ function collectRelayVersions(): Promise<void> {
|
||||
return gcOldRelayVersions(conn, home, currentDir, host, { nativeDepsCacheKeys: [nativeKey] })
|
||||
}
|
||||
|
||||
// ddd is the previous build, so it is pinned and never probed.
|
||||
const installOrder = [
|
||||
`${home}/.orca-remote/relay-0.1.0+bbb/.install-complete`,
|
||||
`${home}/.orca-remote/relay-0.1.0+ddd/.install-complete`,
|
||||
`${home}/.orca-remote/relay-0.1.0+aaa/.install-complete`,
|
||||
REMOTE_INSTALL_ORDER_OK
|
||||
].join('\n')
|
||||
|
||||
const versionSteps = [
|
||||
['listing', 'relay-0.1.0+aaa\nrelay-0.1.0+ccc'],
|
||||
['listing', 'relay-0.1.0+aaa\nrelay-0.1.0+ddd'],
|
||||
['previous install order', installOrder],
|
||||
['install lock probe', 'OPEN'],
|
||||
['completion probe', 'COMPLETE'],
|
||||
['liveness probe', 'DEAD'],
|
||||
@@ -73,8 +83,8 @@ describe('version GC termination', () => {
|
||||
'stops after an unconfirmed stale lock probe with claim held: %s',
|
||||
async (claimed) => {
|
||||
const replies = claimed
|
||||
? versionSteps.slice(0, 6).map(([, reply]) => String(reply))
|
||||
: [versionSteps[0][1]]
|
||||
? versionSteps.slice(0, 7).map(([, reply]) => String(reply))
|
||||
: versionSteps.slice(0, 2).map(([, reply]) => String(reply))
|
||||
const error = failAfter([...replies, 'LOCKED'])
|
||||
|
||||
await expect(collectRelayVersions()).rejects.toBe(error)
|
||||
|
||||
@@ -28,7 +28,6 @@ import {
|
||||
MAX_RELAY_GC_LISTING_ENTRIES,
|
||||
moveRemoteTreeCommand,
|
||||
probeFileExistsCommand,
|
||||
relayLivenessProbeCommand,
|
||||
removeRemoteTreeCommand
|
||||
} from './ssh-remote-commands'
|
||||
import {
|
||||
@@ -39,7 +38,13 @@ import {
|
||||
type RemoteHostPlatform
|
||||
} from './ssh-remote-platform'
|
||||
import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints'
|
||||
import type { RelayEndpointVerdict } from './ssh-relay-endpoint-incumbent'
|
||||
import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
|
||||
import { findPreviousRemoteInstall } from './remote-install-previous-version'
|
||||
import {
|
||||
parseRelayVersionDirLiveness,
|
||||
relayVersionDirLivenessCommand
|
||||
} from './relay-version-dir-liveness'
|
||||
|
||||
// Legacy relay dirs predate `.install-complete`; they need a liveness-only GC check so they
|
||||
// eventually drain. There is no orcad equivalent — orcad has never shipped without one.
|
||||
@@ -68,13 +73,19 @@ export type RemoteInstallGcOptions = {
|
||||
* the rollback target, and GC'ing it turns a recoverable bad update into a re-deploy.
|
||||
*/
|
||||
pinnedDirNames?: readonly string[]
|
||||
/**
|
||||
* More pins, resolved only once a candidate exists. Null means the host could not say which
|
||||
* directories to keep, so this pass deletes nothing.
|
||||
*/
|
||||
resolveExtraPinnedDirNames?: () => Promise<readonly string[] | null>
|
||||
}
|
||||
|
||||
/**
|
||||
* Garbage-collect one model's old version directories.
|
||||
*
|
||||
* **GC ownership (design §06 falsifier 1):** a pass only ever sees, and only ever deletes,
|
||||
* directories belonging to `model`. The remote listing is scoped by prefix, and
|
||||
* **GC ownership (design D10; to be tracked in docs/reference/remote-server-install-model.md):**
|
||||
* a pass only ever sees, and only ever deletes, directories belonging to `model`. The remote
|
||||
* listing is scoped by prefix, and
|
||||
* `remoteInstallGcPermits` re-checks every candidate locally, so neither a widened glob nor
|
||||
* a hand-rolled listing can make one model delete the other's live install.
|
||||
*/
|
||||
@@ -121,10 +132,17 @@ export async function gcOldRemoteInstallVersions(
|
||||
if (candidates.length === 0) {
|
||||
return
|
||||
}
|
||||
const extraPins = options.resolveExtraPinnedDirNames
|
||||
? await options.resolveExtraPinnedDirNames()
|
||||
: []
|
||||
if (!extraPins) {
|
||||
return
|
||||
}
|
||||
const survivors = candidates.filter((name) => !extraPins.includes(name))
|
||||
|
||||
const removed: string[] = []
|
||||
const kept: string[] = []
|
||||
for (const name of candidates) {
|
||||
for (const name of survivors) {
|
||||
const dir = joinRemotePath(host, baseDir, name)
|
||||
try {
|
||||
const safe = await isCandidateSafeToRemove(conn, model, dir, name, host, options)
|
||||
@@ -249,8 +267,8 @@ async function isCandidateSafeToRemove(
|
||||
}
|
||||
|
||||
/**
|
||||
* The relay's GC, bound to its own namespace and its own liveness probe (a live unix socket
|
||||
* or Windows pipe inside the version dir).
|
||||
* The relay's GC, bound to its own namespace. A version dir goes only on an `exited` verdict
|
||||
* (relay-version-dir-liveness.ts), and the previous completed build is pinned (design D5).
|
||||
*/
|
||||
export async function gcOldRelayVersions(
|
||||
conn: SshConnection,
|
||||
@@ -260,6 +278,8 @@ export async function gcOldRelayVersions(
|
||||
options?: {
|
||||
windowsNodePath?: string
|
||||
windowsSockNames?: string[]
|
||||
/** Host Node that runs the connect probe once a recorded relay PID is dead. */
|
||||
nodePath?: string
|
||||
/**
|
||||
* Cache entries this connection depends on, whether or not it links to them. Also the gate:
|
||||
* a caller that could not compute a key is not using the shared-cache model on this host, and
|
||||
@@ -270,7 +290,22 @@ export async function gcOldRelayVersions(
|
||||
): Promise<void> {
|
||||
await gcOldRemoteInstallVersions(conn, RELAY_INSTALL_MODEL, remoteHome, currentDirAbsPath, host, {
|
||||
...options,
|
||||
isDirLive: (dir) => hasLiveRelaySocket(conn, dir, host, options)
|
||||
resolveExtraPinnedDirNames: async () => {
|
||||
const previous = await findPreviousRemoteInstall(
|
||||
conn,
|
||||
host,
|
||||
joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR),
|
||||
RELAY_INSTALL_MODEL,
|
||||
remoteBasename(currentDirAbsPath, host)
|
||||
)
|
||||
// Why null rather than a guess: without the order, any candidate could be the previous build.
|
||||
if (previous.state !== 'ok') {
|
||||
return null
|
||||
}
|
||||
return previous.dirName ? [previous.dirName] : []
|
||||
},
|
||||
isDirLive: async (dir) =>
|
||||
(await probeRelayVersionDirLiveness(conn, dir, host, options)) !== 'exited'
|
||||
})
|
||||
// Why after and not before: version-dir removal is what turns a cache entry unreferenced, so
|
||||
// running it second lets one pass reclaim both instead of leaving the tree for the next connect.
|
||||
@@ -285,18 +320,19 @@ export async function gcOldRelayVersions(
|
||||
}
|
||||
}
|
||||
|
||||
async function hasLiveRelaySocket(
|
||||
/** GC deletes a relay version dir only on `exited`; see relay-version-dir-liveness.ts. */
|
||||
export async function probeRelayVersionDirLiveness(
|
||||
conn: SshConnection,
|
||||
dir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
|
||||
options?: {
|
||||
windowsNodePath?: string
|
||||
windowsSockNames?: string[]
|
||||
nodePath?: string
|
||||
}
|
||||
): Promise<boolean> {
|
||||
): Promise<RelayEndpointVerdict> {
|
||||
try {
|
||||
// Why: `test -S` only — a connect-and-close probe would race with a daemon about to idle.
|
||||
const windowsOptions =
|
||||
const windows =
|
||||
isWindowsRemoteHost(host) && options?.windowsNodePath
|
||||
? {
|
||||
nodePath: options.windowsNodePath,
|
||||
@@ -308,15 +344,14 @@ async function hasLiveRelaySocket(
|
||||
const out = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
relayLivenessProbeCommand(host, dir, windowsOptions)
|
||||
relayVersionDirLivenessCommand(host, dir, { nodePath: options?.nodePath, windows })
|
||||
)
|
||||
const state = out.trim()
|
||||
return state !== 'DEAD' && state !== 'WAITING'
|
||||
return parseRelayVersionDirLiveness(out)
|
||||
} catch (err) {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
// Why: an inconclusive liveness probe must never authorize deletion.
|
||||
return true
|
||||
// Why: an unanswered probe observes nothing; it never authorizes deletion.
|
||||
return 'unverifiable'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,10 @@ import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { orcadRipgrepArtifact } from '../../shared/orcad-artifacts'
|
||||
import { probeRemoteInstallCompleteCommand } from './ssh-remote-commands'
|
||||
import {
|
||||
listRemoteInstallBaseDirsCommand,
|
||||
probeRemoteInstallCompleteCommand
|
||||
} from './ssh-remote-commands'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
import {
|
||||
@@ -120,4 +123,35 @@ describe('GC ownership — each model collects only its own namespace', () => {
|
||||
expect(inventory.orcad).toEqual(ORCAD_DIRS)
|
||||
expect(inventory.unknown).toEqual(['something-else'])
|
||||
})
|
||||
|
||||
it('gives the shared runtime store its own owner that no version-dir GC may take', () => {
|
||||
expect(remoteInstallDirOwner('runtimes')).toBe('runtimes')
|
||||
expect(remoteInstallGcPermits(RELAY_INSTALL_MODEL, 'runtimes')).toBe(false)
|
||||
expect(remoteInstallGcPermits(ORCAD_INSTALL_MODEL, 'runtimes')).toBe(false)
|
||||
expect(inventoryRemoteInstallDirs(['runtimes', ...RELAY_DIRS]).runtimes).toEqual(['runtimes'])
|
||||
})
|
||||
|
||||
it.skipIf(process.platform === 'win32')(
|
||||
'keeps runtimes/ out of every model listing, including older clients’ prefix scans',
|
||||
() => {
|
||||
const base = mkdtempSync(join(tmpdir(), 'install-listing-'))
|
||||
try {
|
||||
for (const name of ['runtimes', 'relay-0.1.0+aa', 'orcad-0.1.0+aa']) {
|
||||
mkdirSync(join(base, name))
|
||||
}
|
||||
mkdirSync(join(base, 'runtimes', `node-${'a'.repeat(64)}`))
|
||||
const host = getRemoteHostPlatform('linux-x64')
|
||||
for (const model of [RELAY_INSTALL_MODEL, ORCAD_INSTALL_MODEL]) {
|
||||
const listed = execFileSync(
|
||||
'/bin/sh',
|
||||
['-c', listRemoteInstallBaseDirsCommand(host, base, model)],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
expect(listed.trim().split('\n')).toEqual([`${model.dirPrefix}-0.1.0+aa`])
|
||||
}
|
||||
} finally {
|
||||
rmSync(base, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
/**
|
||||
* The two things Orca installs into `~/.orca-remote/`, and the rules that keep them from
|
||||
* The things Orca installs into `~/.orca-remote/`, and the rules that keep them from
|
||||
* touching each other.
|
||||
*
|
||||
* `docs/design/shipping-orcad.html` §06 settles that on-disk coexistence is permanent: the
|
||||
* relay is the dumb execution host for SSH-target users, orcad is the peer for paired
|
||||
* environments, and no plan item retires either. So `relay-<version>/` and `orcad-<version>/`
|
||||
* sit side by side forever, and the namespace has to be a parameter rather than a literal.
|
||||
* Design D10 (to be tracked in docs/reference/remote-server-install-model.md): the relay and
|
||||
* orcad converge into one server package; on the host that is `server-<version>/` plus the
|
||||
* shared `runtimes/node-<sha256>/` store. Legacy `relay-*` / `orcad-*` directories belong to a
|
||||
* migration sweep that deletes only on an `exited` verdict, handed over in two releases: this
|
||||
* one lists them as diagnostics only. Until then each is its own namespace, so the prefix has
|
||||
* to be a parameter rather than a literal.
|
||||
*
|
||||
* GC ownership is the trap that parameterization creates. Each model garbage-collects ONLY
|
||||
* its own directories — see `remoteInstallDirOwner`. Relay's regex happened to be narrow
|
||||
@@ -21,6 +23,7 @@ import {
|
||||
import {
|
||||
orcadArtifactFilenames,
|
||||
ORCAD_INSTALL_COMPLETE_FILENAME,
|
||||
ORCAD_RUNTIMES_DIRNAME,
|
||||
ORCAD_VERSION_FILENAME
|
||||
} from '../../shared/orcad-artifacts'
|
||||
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
@@ -104,15 +107,18 @@ export function remoteInstallListingRegexSource(model: RemoteInstallModel): stri
|
||||
}
|
||||
|
||||
/**
|
||||
* Which model owns a directory found in `~/.orca-remote/`, or null for anything neither
|
||||
* model created.
|
||||
* Which owner a directory found in `~/.orca-remote/` belongs to, or null for anything no
|
||||
* owner created.
|
||||
*
|
||||
* This is the answer to §06 falsifier 1's first half: **the model that created a directory
|
||||
* owns it, and nothing else may delete it.** A relay GC pass that saw `orcad-0.1.0+abc`
|
||||
* would be looking at the live install of a peer whose lifecycle it has no view into — the
|
||||
* SSH-execution-boundary collapse in directory form.
|
||||
* Design D10: **the model that created a directory owns it, and nothing else may delete it**
|
||||
* (amended only in the release after the two-step hand-over). A relay GC pass that saw
|
||||
* `orcad-0.1.0+abc` would be looking at the live install of a peer whose lifecycle it has no
|
||||
* view into — the SSH-execution-boundary collapse in directory form.
|
||||
*/
|
||||
export function remoteInstallDirOwner(dirName: string): RemoteInstallModelId | null {
|
||||
export function remoteInstallDirOwner(dirName: string): RemoteInstallDirOwner | null {
|
||||
if (dirName === ORCAD_RUNTIMES_DIRNAME) {
|
||||
return 'runtimes'
|
||||
}
|
||||
for (const model of REMOTE_INSTALL_MODELS) {
|
||||
if (new RegExp(remoteInstallListingRegexSource(model)).test(dirName)) {
|
||||
return model.id
|
||||
@@ -126,11 +132,17 @@ export function remoteInstallGcPermits(model: RemoteInstallModel, dirName: strin
|
||||
return remoteInstallDirOwner(dirName) === model.id
|
||||
}
|
||||
|
||||
export type RemoteInstallInventory = Record<RemoteInstallModelId | 'unknown', string[]>
|
||||
/**
|
||||
* `runtimes` is the shared Node store (design D5). No version-dir model owns it, so neither
|
||||
* model's GC can list or delete it; only `remote-node-runtime-store-gc.ts` collects inside it.
|
||||
*/
|
||||
export type RemoteInstallDirOwner = RemoteInstallModelId | 'runtimes'
|
||||
|
||||
export type RemoteInstallInventory = Record<RemoteInstallDirOwner | 'unknown', string[]>
|
||||
|
||||
/** Group a raw `~/.orca-remote/` listing by owning model, for diagnostics and the client's choice. */
|
||||
export function inventoryRemoteInstallDirs(dirNames: readonly string[]): RemoteInstallInventory {
|
||||
const inventory: RemoteInstallInventory = { relay: [], orcad: [], unknown: [] }
|
||||
const inventory: RemoteInstallInventory = { relay: [], orcad: [], runtimes: [], unknown: [] }
|
||||
for (const name of dirNames) {
|
||||
const owner = remoteInstallDirOwner(name)
|
||||
if (owner) {
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as DeployHelpers from './ssh-relay-deploy-helpers'
|
||||
|
||||
vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof DeployHelpers>()),
|
||||
execCommand: vi.fn()
|
||||
}))
|
||||
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { gcOldRelayVersions } from './remote-install-gc'
|
||||
import { ORCAD_INSTALL_MODEL, RELAY_INSTALL_MODEL } from './remote-install-model'
|
||||
import {
|
||||
listCompletedInstallsNewestFirstCommand,
|
||||
parseCompletedInstallsNewestFirst,
|
||||
REMOTE_INSTALL_ORDER_OK
|
||||
} from './remote-install-previous-version'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock.
|
||||
const conn = {} as SshConnection
|
||||
const host = getRemoteHostPlatform('linux-x64')
|
||||
const mockExec = vi.mocked(execCommand)
|
||||
|
||||
describe('parseCompletedInstallsNewestFirst', () => {
|
||||
it('keeps only this model’s version dirs, in host order', () => {
|
||||
const output = [
|
||||
'/h/.orca-remote/relay-0.2.0+bbb/.install-complete',
|
||||
'/h/.orca-remote/orcad-0.2.0+bbb/.install-complete',
|
||||
'/h/.orca-remote/relay-0.1.0+aaa/.install-complete',
|
||||
REMOTE_INSTALL_ORDER_OK
|
||||
].join('\n')
|
||||
expect(parseCompletedInstallsNewestFirst(output, RELAY_INSTALL_MODEL)).toEqual([
|
||||
'relay-0.2.0+bbb',
|
||||
'relay-0.1.0+aaa'
|
||||
])
|
||||
})
|
||||
|
||||
it('is null when the host did not finish the listing', () => {
|
||||
expect(
|
||||
parseCompletedInstallsNewestFirst(
|
||||
'/h/.orca-remote/relay-0.1.0+aaa/.install-complete',
|
||||
RELAY_INSTALL_MODEL
|
||||
)
|
||||
).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
const posixOnly = process.platform === 'win32' ? describe.skip : describe
|
||||
|
||||
posixOnly('listCompletedInstallsNewestFirstCommand (real shell)', () => {
|
||||
let base: string
|
||||
beforeEach(() => {
|
||||
base = mkdtempSync(join(tmpdir(), 'install-order-'))
|
||||
})
|
||||
afterEach(() => {
|
||||
rmSync(base, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function install(name: string, mtimeSeconds: number, complete = true): void {
|
||||
mkdirSync(join(base, name))
|
||||
if (complete) {
|
||||
const marker = join(base, name, '.install-complete')
|
||||
writeFileSync(marker, '')
|
||||
utimesSync(marker, mtimeSeconds, mtimeSeconds)
|
||||
}
|
||||
}
|
||||
|
||||
function run(): string[] | null {
|
||||
const out = execFileSync(
|
||||
'/bin/sh',
|
||||
['-c', listCompletedInstallsNewestFirstCommand(host, base, RELAY_INSTALL_MODEL)],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
return parseCompletedInstallsNewestFirst(out, RELAY_INSTALL_MODEL)
|
||||
}
|
||||
|
||||
it('orders completed installs by marker mtime and skips torn ones', () => {
|
||||
install('relay-0.1.0+aaa', 1_000)
|
||||
install('relay-0.3.0+ccc', 3_000)
|
||||
install('relay-0.2.0+bbb', 2_000)
|
||||
install('relay-0.4.0+ddd', 4_000, false)
|
||||
install('orcad-0.9.0+eee', 9_000)
|
||||
|
||||
expect(run()).toEqual(['relay-0.3.0+ccc', 'relay-0.2.0+bbb', 'relay-0.1.0+aaa'])
|
||||
})
|
||||
|
||||
it('answers an empty order for a base without completed installs', () => {
|
||||
expect(run()).toEqual([])
|
||||
})
|
||||
|
||||
it('is scoped to the model prefix', () => {
|
||||
install('orcad-0.9.0+eee', 9_000)
|
||||
const out = execFileSync(
|
||||
'/bin/sh',
|
||||
['-c', listCompletedInstallsNewestFirstCommand(host, base, ORCAD_INSTALL_MODEL)],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
expect(parseCompletedInstallsNewestFirst(out, ORCAD_INSTALL_MODEL)).toEqual(['orcad-0.9.0+eee'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('relay GC keeps the previous build', () => {
|
||||
beforeEach(() => {
|
||||
mockExec.mockReset()
|
||||
mockExec.mockResolvedValue('')
|
||||
})
|
||||
|
||||
it('never probes or removes the most recent other completed install', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
.mockResolvedValueOnce(
|
||||
[
|
||||
'/home/u/.orca-remote/relay-0.2.0+bbb/.install-complete',
|
||||
'/home/u/.orca-remote/relay-0.1.0+aaa/.install-complete',
|
||||
REMOTE_INSTALL_ORDER_OK
|
||||
].join('\n')
|
||||
)
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host)
|
||||
|
||||
expect(mockExec).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('deletes nothing when the host cannot report install order', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\nrelay-0.0.9+zzz\n')
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error('ls failed'), { sshChannelCloseConfirmed: true })
|
||||
)
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host)
|
||||
|
||||
expect(mockExec).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('keeps an old build whose liveness probe times out', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
.mockResolvedValueOnce(`relay-0.1.5+fff\n${REMOTE_INSTALL_ORDER_OK}`)
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error('SSH command timed out'), { sshChannelCloseConfirmed: true })
|
||||
)
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.2.0+bbb', host, {
|
||||
nodePath: '/usr/bin/node'
|
||||
})
|
||||
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(commands[4]).toContain('.relay-pid')
|
||||
expect(commands.some((command) => command.includes('gc-claim'))).toBe(false)
|
||||
expect(mockExec).toHaveBeenCalledTimes(5)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* The version dir a model ran before the current one, pinned against GC like orcad's
|
||||
* rollback target (design D5 "plus the relay's previous version as a pin").
|
||||
*
|
||||
* The relay keeps no activation record, so "previous" is the most recently completed install
|
||||
* of the same model other than the current one, by `.install-complete` mtime.
|
||||
*/
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
|
||||
import {
|
||||
remoteInstallGcPermits,
|
||||
remoteInstallVersionDirRegex,
|
||||
type RemoteInstallModel
|
||||
} from './remote-install-model'
|
||||
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell'
|
||||
|
||||
export const REMOTE_INSTALL_ORDER_OK = 'ORCA_INSTALL_ORDER_OK'
|
||||
|
||||
export function listCompletedInstallsNewestFirstCommand(
|
||||
host: RemoteHostPlatform,
|
||||
baseDir: string,
|
||||
model: RemoteInstallModel
|
||||
): string {
|
||||
const prefix = `${model.dirPrefix}-`
|
||||
if (isWindowsRemoteHost(host)) {
|
||||
return powerShellCommand(
|
||||
[
|
||||
"$ErrorActionPreference = 'Stop'",
|
||||
`$base = ${powerShellLiteral(baseDir)}`,
|
||||
'if (Test-Path -LiteralPath $base -PathType Container) {',
|
||||
`Get-ChildItem -LiteralPath $base -Directory -Filter '${prefix}*' | ForEach-Object { ` +
|
||||
`$marker = Join-Path $_.FullName ${powerShellLiteral(model.installCompleteFilename)}; ` +
|
||||
'if (Test-Path -LiteralPath $marker -PathType Leaf) { Get-Item -LiteralPath $marker } ' +
|
||||
'} | Sort-Object LastWriteTimeUtc -Descending | ForEach-Object { $_.Directory.Name }',
|
||||
'}',
|
||||
`'${REMOTE_INSTALL_ORDER_OK}'`
|
||||
].join('\n')
|
||||
)
|
||||
}
|
||||
return [
|
||||
`base=${shellEscape(baseDir)}`,
|
||||
`set -- "$base"/${prefix}*/${shellEscape(model.installCompleteFilename)}`,
|
||||
`[ -e "$1" ] || { echo ${REMOTE_INSTALL_ORDER_OK}; exit 0; }`,
|
||||
'ls -1t -- "$@" || exit 1',
|
||||
`echo ${REMOTE_INSTALL_ORDER_OK}`
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
/** Dir names newest first, or null when the host could not answer. */
|
||||
export function parseCompletedInstallsNewestFirst(
|
||||
output: string,
|
||||
model: RemoteInstallModel
|
||||
): string[] | null {
|
||||
const lines = output
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
if (!lines.includes(REMOTE_INSTALL_ORDER_OK)) {
|
||||
return null
|
||||
}
|
||||
const versionDirRegex = remoteInstallVersionDirRegex(model)
|
||||
const names: string[] = []
|
||||
for (const line of lines) {
|
||||
// POSIX prints `<base>/<dir>/.install-complete`; PowerShell prints `<dir>`.
|
||||
const segments = line.split(/[\\/]/)
|
||||
const name = segments.length > 1 ? segments.at(-2) : segments[0]
|
||||
if (name && versionDirRegex.test(name) && remoteInstallGcPermits(model, name)) {
|
||||
names.push(name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
export type PreviousInstallResult = { state: 'ok'; dirName: string | null } | { state: 'unknown' }
|
||||
|
||||
export async function findPreviousRemoteInstall(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
baseDir: string,
|
||||
model: RemoteInstallModel,
|
||||
currentDirName: string
|
||||
): Promise<PreviousInstallResult> {
|
||||
let output: string
|
||||
try {
|
||||
output = await execCommand(
|
||||
conn,
|
||||
listCompletedInstallsNewestFirstCommand(host, baseDir, model),
|
||||
{
|
||||
wrapCommand: host.commandDialect !== 'powershell'
|
||||
}
|
||||
)
|
||||
} catch (err) {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
return { state: 'unknown' }
|
||||
}
|
||||
const ordered = parseCompletedInstallsNewestFirst(output, model)
|
||||
if (!ordered) {
|
||||
return { state: 'unknown' }
|
||||
}
|
||||
return { state: 'ok', dirName: ordered.find((name) => name !== currentDirName) ?? null }
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
import { execFileSync, spawn, type ChildProcess } from 'node:child_process'
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
utimesSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { once } from 'node:events'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as DeployHelpers from './ssh-relay-deploy-helpers'
|
||||
|
||||
vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof DeployHelpers>()),
|
||||
execCommand: vi.fn()
|
||||
}))
|
||||
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { gcRemoteNodeRuntimeStore, planRuntimeStoreGc } from './remote-node-runtime-store-gc'
|
||||
import {
|
||||
parseRuntimeStoreInventory,
|
||||
RUNTIME_REF_NODE_PREFIX,
|
||||
runtimeStoreInventoryCommand,
|
||||
type RuntimeStoreInventory
|
||||
} from './remote-node-runtime-store-inventory'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: all connection access is replaced by execCommand's mock.
|
||||
const conn = {} as SshConnection
|
||||
const host = getRemoteHostPlatform('linux-x64')
|
||||
const mockExec = vi.mocked(execCommand)
|
||||
const sha = (c: string): string => c.repeat(64)
|
||||
|
||||
function inventory(overrides: Partial<RuntimeStoreInventory> = {}): RuntimeStoreInventory {
|
||||
return {
|
||||
entries: [],
|
||||
verifiedNewestFirst: [],
|
||||
referenced: new Set(),
|
||||
held: new Set(),
|
||||
processCheckRan: true,
|
||||
dirNames: [],
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
describe('planRuntimeStoreGc', () => {
|
||||
const all = ['a', 'b', 'c', 'd'].map((c) => `node-${sha(c)}`)
|
||||
|
||||
it('keeps the current pin and the newest other verified runtime', () => {
|
||||
const plan = planRuntimeStoreGc(
|
||||
inventory({ entries: all, verifiedNewestFirst: [all[0], all[1], all[2], all[3]] }),
|
||||
[sha('b')]
|
||||
)
|
||||
expect(plan.remove).toEqual([all[2], all[3]])
|
||||
expect(plan.kept).toEqual([all[0], all[1]])
|
||||
})
|
||||
|
||||
it('keeps referenced, process-held and unverified runtimes', () => {
|
||||
const plan = planRuntimeStoreGc(
|
||||
inventory({
|
||||
entries: all,
|
||||
verifiedNewestFirst: [all[0], all[1], all[2]],
|
||||
referenced: new Set([sha('b')]),
|
||||
held: new Set([sha('c')])
|
||||
}),
|
||||
[sha('a')]
|
||||
)
|
||||
expect(plan.remove).toEqual([])
|
||||
})
|
||||
|
||||
it('keeps everything when no process check could run', () => {
|
||||
const plan = planRuntimeStoreGc(
|
||||
inventory({ entries: all, verifiedNewestFirst: all, processCheckRan: false }),
|
||||
[sha('a')]
|
||||
)
|
||||
expect(plan.remove).toEqual([])
|
||||
})
|
||||
|
||||
it('purges only abandoned tombstones of unwanted runtimes', () => {
|
||||
const now = 10 * 60 * 60_000
|
||||
const old = `.gc-tombstone-node-${sha('c')}.7.${now - 31 * 60_000}`
|
||||
const fresh = `.gc-tombstone-node-${sha('d')}.7.${now - 60_000}`
|
||||
const referenced = `.gc-tombstone-node-${sha('e')}.7.${now - 31 * 60_000}`
|
||||
const plan = planRuntimeStoreGc(
|
||||
inventory({ entries: [old, fresh, referenced], referenced: new Set([sha('e')]) }),
|
||||
[sha('a')],
|
||||
now
|
||||
)
|
||||
expect(plan.purgeTombstones).toEqual([old])
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseRuntimeStoreInventory', () => {
|
||||
it('rejects a partial listing, a reference error, and an unattributable reference', () => {
|
||||
expect(parseRuntimeStoreInventory('ENTRY node-x')).toBeNull()
|
||||
expect(
|
||||
parseRuntimeStoreInventory('__ORCA_RUNTIME_STORE__REFS_ERR\n__ORCA_RUNTIME_STORE__OK')
|
||||
).toBeNull()
|
||||
expect(parseRuntimeStoreInventory('REF garbage\n__ORCA_RUNTIME_STORE__OK')).toBeNull()
|
||||
})
|
||||
|
||||
it('attributes process holds by runtime path, including renamed tombstones', () => {
|
||||
const parsed = parseRuntimeStoreInventory(
|
||||
[
|
||||
'PROCESS_CHECK ps',
|
||||
`HOLD /h/.orca-remote/runtimes/node-${sha('a')}/bin/node server.js`,
|
||||
`HOLD /h/.orca-remote/runtimes/.gc-tombstone-node-${sha('b')}.1.2/bin/node`,
|
||||
'HOLD grep -F -- /h/.orca-remote/runtimes/',
|
||||
'__ORCA_RUNTIME_STORE__OK'
|
||||
].join('\n')
|
||||
)
|
||||
expect(parsed?.held).toEqual(new Set([sha('a'), sha('b')]))
|
||||
})
|
||||
})
|
||||
|
||||
const posixOnly = process.platform === 'win32' ? describe.skip : describe
|
||||
|
||||
posixOnly('gcRemoteNodeRuntimeStore (real shell)', () => {
|
||||
let home: string
|
||||
let root: string
|
||||
let running: ChildProcess | null = null
|
||||
|
||||
beforeEach(() => {
|
||||
home = mkdtempSync(join(tmpdir(), 'runtime-store-'))
|
||||
root = join(home, '.orca-remote')
|
||||
mkdirSync(join(root, 'runtimes'), { recursive: true })
|
||||
mockExec.mockReset()
|
||||
mockExec.mockImplementation(async (_conn, command) =>
|
||||
execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' })
|
||||
)
|
||||
})
|
||||
afterEach(() => {
|
||||
running?.kill('SIGKILL')
|
||||
running = null
|
||||
rmSync(home, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
// `b` is always verified last, so it is the previous pin the store keeps.
|
||||
function runtime(c: string, verified = true): string {
|
||||
const dir = join(root, 'runtimes', `node-${sha(c)}`)
|
||||
mkdirSync(join(dir, 'bin'), { recursive: true })
|
||||
writeFileSync(join(dir, 'bin', 'node'), '#!/bin/sh\nsleep 30\n')
|
||||
chmodSync(join(dir, 'bin', 'node'), 0o755)
|
||||
if (verified) {
|
||||
writeFileSync(join(dir, '.verified'), '')
|
||||
const at = c === 'b' ? 2_000_000 : 1_000_000
|
||||
utimesSync(join(dir, '.verified'), at, at)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
function versionDir(name: string): string {
|
||||
const dir = join(root, name)
|
||||
mkdirSync(dir, { recursive: true })
|
||||
return dir
|
||||
}
|
||||
|
||||
it('removes only unreferenced, unpinned, idle, verified runtimes', async () => {
|
||||
runtime('a') // current pin
|
||||
runtime('c') // referenced by an orcad slot marker
|
||||
runtime('d') // referenced by a relay ref file
|
||||
runtime('e') // unreferenced: collected
|
||||
const running_ = runtime('f') // executing
|
||||
runtime('9', false) // mid-promotion
|
||||
runtime('b') // newest other verified: the previous pin
|
||||
writeFileSync(join(versionDir('orcad-0.1.0+aaa'), '.runtime-node'), `${sha('c')}\n`)
|
||||
writeFileSync(join(versionDir('relay-0.1.0+aaa'), `${RUNTIME_REF_NODE_PREFIX}${sha('d')}`), '')
|
||||
running = spawn(join(running_, 'bin', 'node'), [], { stdio: 'ignore' })
|
||||
await once(running, 'spawn')
|
||||
|
||||
const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] })
|
||||
|
||||
expect(result).toMatchObject({
|
||||
state: 'collected',
|
||||
removed: [`node-${sha('e')}`],
|
||||
legacyDirs: expect.arrayContaining(['orcad-0.1.0+aaa', 'relay-0.1.0+aaa'])
|
||||
})
|
||||
for (const kept of ['a', 'b', 'c', 'd', 'f', '9']) {
|
||||
expect(existsSync(join(root, 'runtimes', `node-${sha(kept)}`))).toBe(true)
|
||||
}
|
||||
expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`))).toBe(false)
|
||||
// D10 two-step: legacy dirs are reported, never deleted by this pass.
|
||||
expect(existsSync(join(root, 'orcad-0.1.0+aaa'))).toBe(true)
|
||||
expect(existsSync(join(root, 'relay-0.1.0+aaa'))).toBe(true)
|
||||
})
|
||||
|
||||
it('holds a runtime a process runs through another spelling of a symlinked home', async () => {
|
||||
runtime('a')
|
||||
runtime('b')
|
||||
const held = runtime('e')
|
||||
const alias = mkdtempSync(join(tmpdir(), 'runtime-store-alias-'))
|
||||
rmSync(alias, { recursive: true })
|
||||
symlinkSync(home, alias)
|
||||
try {
|
||||
running = spawn(join(held, 'bin', 'node'), [], { stdio: 'ignore' })
|
||||
await once(running, 'spawn')
|
||||
|
||||
const result = await gcRemoteNodeRuntimeStore(conn, host, alias, { currentPins: [sha('a')] })
|
||||
|
||||
expect(result).toMatchObject({ state: 'collected', removed: [] })
|
||||
expect(existsSync(join(held, 'bin', 'node'))).toBe(true)
|
||||
} finally {
|
||||
rmSync(alias, { force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps every runtime when a reference marker cannot be attributed', async () => {
|
||||
runtime('a')
|
||||
runtime('b')
|
||||
runtime('e')
|
||||
writeFileSync(join(versionDir('server-0.2.0+ccc'), '.runtime-node'), 'not-a-sha\n')
|
||||
|
||||
const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] })
|
||||
|
||||
expect(result.state).toBe('skipped')
|
||||
expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`))).toBe(true)
|
||||
})
|
||||
|
||||
it('restores a runtime that gained a reference after the rename', async () => {
|
||||
runtime('a')
|
||||
runtime('b')
|
||||
runtime('e')
|
||||
let inventories = 0
|
||||
mockExec.mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('__ORCA_RUNTIME_STORE__OK') && ++inventories === 2) {
|
||||
writeFileSync(join(versionDir('orcad-0.3.0+ddd'), '.runtime-node'), `${sha('e')}\n`)
|
||||
}
|
||||
return execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' })
|
||||
})
|
||||
|
||||
const result = await gcRemoteNodeRuntimeStore(conn, host, home, { currentPins: [sha('a')] })
|
||||
|
||||
expect(result).toMatchObject({ state: 'collected', removed: [] })
|
||||
expect(existsSync(join(root, 'runtimes', `node-${sha('e')}`, 'bin', 'node'))).toBe(true)
|
||||
})
|
||||
|
||||
it('is inert when the store does not exist', async () => {
|
||||
rmSync(join(root, 'runtimes'), { recursive: true })
|
||||
const out = execFileSync('/bin/sh', ['-c', runtimeStoreInventoryCommand(host, home)], {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
expect(parseRuntimeStoreInventory(out)?.entries).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('gcRemoteNodeRuntimeStore termination', () => {
|
||||
beforeEach(() => {
|
||||
mockExec.mockReset()
|
||||
})
|
||||
|
||||
it('rethrows an unconfirmed inventory termination', async () => {
|
||||
const error = Object.assign(new Error('SSH command timed out'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
mockExec.mockRejectedValueOnce(error)
|
||||
await expect(
|
||||
gcRemoteNodeRuntimeStore(conn, host, '/home/u', { currentPins: [sha('a')] })
|
||||
).rejects.toBe(error)
|
||||
})
|
||||
|
||||
it('skips Windows hosts without running anything', async () => {
|
||||
const result = await gcRemoteNodeRuntimeStore(
|
||||
conn,
|
||||
getRemoteHostPlatform('win32-x64'),
|
||||
'C:/Users/u',
|
||||
{
|
||||
currentPins: [sha('a')]
|
||||
}
|
||||
)
|
||||
expect(result.state).toBe('skipped')
|
||||
expect(mockExec).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Collects the shared `~/.orca-remote/runtimes/node-<sha256>/` store (design D5 GC).
|
||||
*
|
||||
* A runtime is deleted only when all of these hold: no retained directory references it
|
||||
* (`.runtime-node` or `.runtime-ref-node-<sha>`), it is neither a pin this client runs nor the
|
||||
* newest other verified runtime (keep two), and a process check ran and found nothing executing
|
||||
* from it. Process evidence can only add holds; a scan that could not run keeps everything.
|
||||
*
|
||||
* Legacy `relay-*` / `orcad-*` directories are read for references and reported as
|
||||
* diagnostics, never deleted here (design D10 two-step hand-over).
|
||||
*/
|
||||
import { randomInt } from 'node:crypto'
|
||||
import { ORCAD_RUNTIMES_DIRNAME } from '../../shared/orcad-artifacts'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import { inventoryRemoteInstallDirs } from './remote-install-model'
|
||||
import {
|
||||
parseRuntimeStoreInventory,
|
||||
RUNTIME_STORE_ENTRY_NAME,
|
||||
RUNTIME_STORE_TOMBSTONE_NAME,
|
||||
RUNTIME_STORE_TOMBSTONE_PREFIX,
|
||||
runtimeStoreInventoryCommand,
|
||||
type RuntimeStoreInventory
|
||||
} from './remote-node-runtime-store-inventory'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
|
||||
import {
|
||||
moveRemoteTreeCommand,
|
||||
removeRemoteTreeCommand,
|
||||
restoreRemoteTreeCommand
|
||||
} from './ssh-remote-commands'
|
||||
import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const MAX_REMOVALS_PER_PASS = 8
|
||||
const ABANDONED_TOMBSTONE_MS = 30 * 60_000
|
||||
|
||||
export type RuntimeStoreGcPlan = {
|
||||
/** `node-<sha>` entries to rename away and delete. */
|
||||
remove: string[]
|
||||
/** Abandoned tombstones whose runtime is still unwanted. */
|
||||
purgeTombstones: string[]
|
||||
kept: string[]
|
||||
}
|
||||
|
||||
/** Why a sha must stay, or null when nothing holds it. */
|
||||
function holdReason(
|
||||
sha: string,
|
||||
inventory: RuntimeStoreInventory,
|
||||
pins: ReadonlySet<string>
|
||||
): string | null {
|
||||
if (!inventory.processCheckRan) {
|
||||
return 'process check unavailable'
|
||||
}
|
||||
if (pins.has(sha)) {
|
||||
return 'pinned'
|
||||
}
|
||||
if (inventory.referenced.has(sha)) {
|
||||
return 'referenced'
|
||||
}
|
||||
if (inventory.held.has(sha)) {
|
||||
return 'in use by a process'
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function planRuntimeStoreGc(
|
||||
inventory: RuntimeStoreInventory,
|
||||
currentPins: readonly string[],
|
||||
now: number = Date.now()
|
||||
): RuntimeStoreGcPlan {
|
||||
const pins = new Set(currentPins)
|
||||
// Keep two: the pin this client runs and the newest other verified runtime (the previous pin).
|
||||
const previous = inventory.verifiedNewestFirst
|
||||
.map((name) => RUNTIME_STORE_ENTRY_NAME.exec(name)?.[1])
|
||||
.find((sha): sha is string => !!sha && !pins.has(sha))
|
||||
if (previous) {
|
||||
pins.add(previous)
|
||||
}
|
||||
const verified = new Set(inventory.verifiedNewestFirst)
|
||||
const plan: RuntimeStoreGcPlan = { remove: [], purgeTombstones: [], kept: [] }
|
||||
for (const name of inventory.entries) {
|
||||
const entry = RUNTIME_STORE_ENTRY_NAME.exec(name)
|
||||
if (entry) {
|
||||
// Unverified means mid-promotion or torn; the installer, not GC, owns that state.
|
||||
const idle = verified.has(name) && holdReason(entry[1], inventory, pins) === null
|
||||
if (idle && plan.remove.length < MAX_REMOVALS_PER_PASS) {
|
||||
plan.remove.push(name)
|
||||
} else {
|
||||
plan.kept.push(name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const tombstone = RUNTIME_STORE_TOMBSTONE_NAME.exec(name)
|
||||
if (
|
||||
tombstone &&
|
||||
now - Number(tombstone[2]) >= ABANDONED_TOMBSTONE_MS &&
|
||||
holdReason(tombstone[1], inventory, pins) === null
|
||||
) {
|
||||
plan.purgeTombstones.push(name)
|
||||
}
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
export type RuntimeStoreGcResult =
|
||||
| { state: 'skipped'; reason: string }
|
||||
| { state: 'collected'; removed: string[]; kept: string[]; legacyDirs: string[] }
|
||||
|
||||
function exec(conn: SshConnection, command: string, signal?: AbortSignal): Promise<string> {
|
||||
return execCommand(conn, command, { wrapCommand: true, signal })
|
||||
}
|
||||
|
||||
async function readInventory(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
remoteHome: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<RuntimeStoreInventory | null> {
|
||||
try {
|
||||
return parseRuntimeStoreInventory(
|
||||
await exec(conn, runtimeStoreInventoryCommand(host, remoteHome), signal)
|
||||
)
|
||||
} catch (err) {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One pass over the runtime store. Confirmed failures keep the runtime and end quietly; an
|
||||
* unconfirmed SSH termination is rethrown so the caller stops its cleanup chain.
|
||||
*/
|
||||
export async function gcRemoteNodeRuntimeStore(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
remoteHome: string,
|
||||
options: { currentPins: readonly string[]; signal?: AbortSignal }
|
||||
): Promise<RuntimeStoreGcResult> {
|
||||
if (isWindowsRemoteHost(host)) {
|
||||
return { state: 'skipped', reason: 'Windows hosts have no managed runtime store yet' }
|
||||
}
|
||||
const inventory = await readInventory(conn, host, remoteHome, options.signal)
|
||||
if (!inventory) {
|
||||
return { state: 'skipped', reason: 'runtime store inventory was unverifiable' }
|
||||
}
|
||||
const legacy = inventoryRemoteInstallDirs(inventory.dirNames)
|
||||
const legacyDirs = [...legacy.relay, ...legacy.orcad]
|
||||
const plan = planRuntimeStoreGc(inventory, options.currentPins)
|
||||
const store = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_RUNTIMES_DIRNAME)
|
||||
const removed: string[] = []
|
||||
const kept = [...plan.kept]
|
||||
for (const name of plan.purgeTombstones) {
|
||||
if (await removeTree(conn, host, joinRemotePath(host, store, name), options.signal)) {
|
||||
removed.push(name)
|
||||
}
|
||||
}
|
||||
for (const name of plan.remove) {
|
||||
const sha = RUNTIME_STORE_ENTRY_NAME.exec(name)?.[1] ?? ''
|
||||
const entryDir = joinRemotePath(host, store, name)
|
||||
const tombstone = joinRemotePath(
|
||||
host,
|
||||
store,
|
||||
`${RUNTIME_STORE_TOMBSTONE_PREFIX}${name}.${randomInt(1, 2 ** 47)}.${Date.now()}`
|
||||
)
|
||||
if (!(await moveTree(conn, host, entryDir, tombstone, options.signal))) {
|
||||
kept.push(name)
|
||||
continue
|
||||
}
|
||||
// Why recheck after the rename: an installer that saw this runtime present may be writing
|
||||
// its reference now; restoring is the only outcome that leaves its slot launchable.
|
||||
const recheck = await readInventory(conn, host, remoteHome, options.signal).catch(
|
||||
async (err: unknown) => {
|
||||
await restoreTree(conn, host, tombstone, entryDir, options.signal).catch(() => {})
|
||||
throw err
|
||||
}
|
||||
)
|
||||
if (!recheck || holdReason(sha, recheck, new Set(options.currentPins)) !== null) {
|
||||
await restoreTree(conn, host, tombstone, entryDir, options.signal)
|
||||
kept.push(name)
|
||||
continue
|
||||
}
|
||||
if (await removeTree(conn, host, tombstone, options.signal)) {
|
||||
removed.push(name)
|
||||
} else {
|
||||
kept.push(name)
|
||||
}
|
||||
}
|
||||
if (removed.length > 0) {
|
||||
const legacyNote =
|
||||
legacyDirs.length > 0
|
||||
? `; legacy install dirs left for the migration sweep: ${legacyDirs.join(', ')}`
|
||||
: ''
|
||||
console.log(`[runtime-store] GC: removed ${removed.join(', ')}${legacyNote}`)
|
||||
}
|
||||
return { state: 'collected', removed, kept, legacyDirs }
|
||||
}
|
||||
|
||||
async function moveTree(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
source: string,
|
||||
destination: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
return (
|
||||
(await exec(conn, moveRemoteTreeCommand(host, source, destination), signal)).trim() ===
|
||||
'MOVED'
|
||||
)
|
||||
} catch (err) {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function restoreTree(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
tombstone: string,
|
||||
entryDir: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
await exec(conn, restoreRemoteTreeCommand(host, tombstone, entryDir), signal).catch((err) => {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function removeTree(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
path: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
await exec(conn, removeRemoteTreeCommand(host, path), signal)
|
||||
return true
|
||||
} catch (err) {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* One read-only pass over `~/.orca-remote/` answering what the runtime store GC needs: store
|
||||
* entries, which runtimes are referenced, verified order, and which a running process holds.
|
||||
*/
|
||||
import {
|
||||
ORCAD_NODE_RUNTIME_DIR_PREFIX,
|
||||
ORCAD_NODE_RUNTIME_MARKER_FILENAME,
|
||||
ORCAD_RUNTIMES_DIRNAME
|
||||
} from '../../shared/orcad-artifacts'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
/** A version dir names a runtime it needs with an empty file of this prefix + sha (design D5). */
|
||||
export const RUNTIME_REF_NODE_PREFIX = '.runtime-ref-node-'
|
||||
export const RUNTIME_STORE_TOMBSTONE_PREFIX = '.gc-tombstone-'
|
||||
const INVENTORY_OK = '__ORCA_RUNTIME_STORE__OK'
|
||||
const REFS_ERR = '__ORCA_RUNTIME_STORE__REFS_ERR'
|
||||
const MAX_DIRS = 512
|
||||
const SHA256 = /^[0-9a-f]{64}$/
|
||||
export const RUNTIME_STORE_ENTRY_NAME = new RegExp(
|
||||
`^${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})$`
|
||||
)
|
||||
export const RUNTIME_STORE_TOMBSTONE_NAME = new RegExp(
|
||||
`^${RUNTIME_STORE_TOMBSTONE_PREFIX.replace(/\./g, '\\.')}${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})\\.[0-9]+\\.([0-9]+)$`
|
||||
)
|
||||
const HELD_PATH = new RegExp(
|
||||
`/${ORCAD_RUNTIMES_DIRNAME}/(?:${RUNTIME_STORE_TOMBSTONE_PREFIX.replace(/\./g, '\\.')})?${ORCAD_NODE_RUNTIME_DIR_PREFIX}([0-9a-f]{64})[./]`
|
||||
)
|
||||
|
||||
export type RuntimeStoreInventory = {
|
||||
/** Store entry names: `node-<sha>` and this GC's tombstones. */
|
||||
entries: string[]
|
||||
/** `node-<sha>` names with `.verified`, newest first. */
|
||||
verifiedNewestFirst: string[]
|
||||
referenced: Set<string>
|
||||
held: Set<string>
|
||||
/** False when neither `ps` nor `/proc` answered; nothing may then be called idle. */
|
||||
processCheckRan: boolean
|
||||
/** Other `~/.orca-remote/` directory names, for legacy diagnostics. */
|
||||
dirNames: string[]
|
||||
}
|
||||
|
||||
export function runtimeStoreInventoryCommand(host: RemoteHostPlatform, remoteHome: string): string {
|
||||
const root = joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR)
|
||||
const refPrefix = RUNTIME_REF_NODE_PREFIX
|
||||
return [
|
||||
`root=${shellEscape(root)}`,
|
||||
`rt="$root"/${ORCAD_RUNTIMES_DIRNAME}`,
|
||||
`[ -d "$rt" ] || { printf '%s\\n' ${INVENTORY_OK}; exit 0; }`,
|
||||
`[ -r "$root" ] && [ -x "$root" ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
|
||||
'n=0',
|
||||
// Why every sibling and not a prefix: a directory this client does not recognise may still
|
||||
// be a newer Orca's install that names a runtime.
|
||||
'for d in "$root"/* "$root"/.[!.]*; do',
|
||||
' [ -d "$d" ] || continue',
|
||||
' name=${d##*/}',
|
||||
` [ "$name" = ${ORCAD_RUNTIMES_DIRNAME} ] && continue`,
|
||||
` [ -r "$d" ] && [ -x "$d" ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
|
||||
' n=$((n+1))',
|
||||
` [ "$n" -le ${MAX_DIRS} ] || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
|
||||
` printf 'DIR %s\\n' "$name"`,
|
||||
` if [ -e "$d"/${ORCAD_NODE_RUNTIME_MARKER_FILENAME} ]; then`,
|
||||
` sha=$(cat "$d"/${ORCAD_NODE_RUNTIME_MARKER_FILENAME}) || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
|
||||
` printf 'REF %s\\n' "$sha"`,
|
||||
' fi',
|
||||
` for f in "$d"/${refPrefix}*; do`,
|
||||
' [ -e "$f" ] || continue',
|
||||
` printf 'REF %s\\n' "\${f##*/${refPrefix}}"`,
|
||||
' done',
|
||||
'done',
|
||||
`for e in "$rt"/${ORCAD_NODE_RUNTIME_DIR_PREFIX}* "$rt"/${RUNTIME_STORE_TOMBSTONE_PREFIX}${ORCAD_NODE_RUNTIME_DIR_PREFIX}*; do`,
|
||||
` [ -d "$e" ] && printf 'ENTRY %s\\n' "\${e##*/}"`,
|
||||
'done',
|
||||
`set -- "$rt"/${ORCAD_NODE_RUNTIME_DIR_PREFIX}*/.verified`,
|
||||
'if [ -e "$1" ]; then',
|
||||
` order=$(ls -1t -- "$@") || { printf '%s\\n' ${REFS_ERR}; exit 0; }`,
|
||||
` printf '%s\\n' "$order" | while IFS= read -r v; do v=\${v%/.verified}; printf 'VERIFIED %s\\n' "\${v##*/}"; done`,
|
||||
'fi',
|
||||
// Process checks only add holds, so an unmatched `grep` is not an error. Why not "$rt/":
|
||||
// /proc exe and argv may name the store through another spelling of a symlinked home.
|
||||
'if ps_out=$(ps -e -o args= 2>/dev/null); then',
|
||||
" printf 'PROCESS_CHECK ps\\n'",
|
||||
` printf '%s\\n' "$ps_out" | grep -F -- /${ORCAD_RUNTIMES_DIRNAME}/ | sed 's/^/HOLD /'`,
|
||||
'fi',
|
||||
'if [ -n "$(readlink /proc/self/exe 2>/dev/null)" ]; then',
|
||||
" printf 'PROCESS_CHECK proc\\n'",
|
||||
' for p in /proc/[0-9]*/exe; do',
|
||||
' t=$(readlink "$p" 2>/dev/null) || continue',
|
||||
` case "$t" in */${ORCAD_RUNTIMES_DIRNAME}/*) printf 'HOLD %s\\n' "$t";; esac`,
|
||||
' done',
|
||||
'fi',
|
||||
`printf '%s\\n' ${INVENTORY_OK}`
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
/** Null when the host could not produce a complete inventory; that keeps every runtime. */
|
||||
export function parseRuntimeStoreInventory(output: string): RuntimeStoreInventory | null {
|
||||
const lines = output.split(/\r?\n/).map((line) => line.trim())
|
||||
if (!lines.includes(INVENTORY_OK) || lines.includes(REFS_ERR)) {
|
||||
return null
|
||||
}
|
||||
const inventory: RuntimeStoreInventory = {
|
||||
entries: [],
|
||||
verifiedNewestFirst: [],
|
||||
referenced: new Set(),
|
||||
held: new Set(),
|
||||
processCheckRan: false,
|
||||
dirNames: []
|
||||
}
|
||||
for (const line of lines) {
|
||||
const space = line.indexOf(' ')
|
||||
const tag = space === -1 ? line : line.slice(0, space)
|
||||
const value = space === -1 ? '' : line.slice(space + 1).trim()
|
||||
if (tag === 'REF') {
|
||||
// An unattributable reference could name any runtime, so it stops the pass.
|
||||
if (!SHA256.test(value)) {
|
||||
return null
|
||||
}
|
||||
inventory.referenced.add(value)
|
||||
} else if (
|
||||
tag === 'ENTRY' &&
|
||||
(RUNTIME_STORE_ENTRY_NAME.test(value) || RUNTIME_STORE_TOMBSTONE_NAME.test(value))
|
||||
) {
|
||||
inventory.entries.push(value)
|
||||
} else if (tag === 'VERIFIED' && RUNTIME_STORE_ENTRY_NAME.test(value)) {
|
||||
inventory.verifiedNewestFirst.push(value)
|
||||
} else if (tag === 'HOLD') {
|
||||
const sha = HELD_PATH.exec(value)?.[1]
|
||||
if (sha) {
|
||||
inventory.held.add(sha)
|
||||
}
|
||||
} else if (tag === 'PROCESS_CHECK') {
|
||||
inventory.processCheckRan = true
|
||||
} else if (tag === 'DIR' && value) {
|
||||
inventory.dirNames.push(value)
|
||||
}
|
||||
}
|
||||
return inventory
|
||||
}
|
||||
@@ -61,6 +61,7 @@ vi.mock('./ssh-connection-utils', () => ({
|
||||
import { deployAndLaunchRelay } from './ssh-relay-deploy'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { REMOTE_INSTALL_ORDER_OK } from './remote-install-previous-version'
|
||||
|
||||
function makeMockConnection(): SshConnection {
|
||||
return {
|
||||
@@ -150,6 +151,12 @@ describe('cross-version isolation', () => {
|
||||
if (command.includes('test -S') && command.includes('echo ALIVE || echo DEAD')) {
|
||||
return Promise.resolve('DEAD')
|
||||
}
|
||||
if (command.includes(REMOTE_INSTALL_ORDER_OK)) {
|
||||
// A newer v0 is the previous build, so v1 is kept only by its live socket.
|
||||
return Promise.resolve(
|
||||
`relay-0.1.0+222222222222\nrelay-0.1.0+000000000000\nrelay-0.1.0+111111111111\n${REMOTE_INSTALL_ORDER_OK}`
|
||||
)
|
||||
}
|
||||
if (command.includes('__ORCA_RELAY_GC_FIND_STATUS__')) {
|
||||
return Promise.resolve('relay-0.1.0+111111111111\nrelay-0.1.0+222222222222\n')
|
||||
}
|
||||
|
||||
@@ -693,6 +693,7 @@ async function deployAndLaunchRelayAttempt(
|
||||
gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, {
|
||||
windowsNodePath: launched.nodePath,
|
||||
windowsSockNames: [relaySocketNameForInstanceId(relayInstanceId)],
|
||||
nodePath: launched.nodePath,
|
||||
// Why pin rather than rely on the symlink alone: a deploy that fell back to a
|
||||
// per-directory install has no reference to show, and its key must still survive.
|
||||
nativeDepsCacheKeys: [
|
||||
|
||||
@@ -69,7 +69,7 @@ const PROBE_END = 'ORCA-INCUMBENT-END'
|
||||
const CONNECT_PROBE_TIMEOUT_MS = 1000
|
||||
|
||||
// Why ES5 syntax: nodePath may be a host-resolved system node, not the bundled one.
|
||||
const CONNECT_PROBE_JS = [
|
||||
export const RELAY_CONNECT_PROBE_JS = [
|
||||
'var s=require("net").connect(process.argv[1]);',
|
||||
'var done=false;',
|
||||
'function say(v){if(done)return;done=true;try{s.destroy()}catch(e){};',
|
||||
@@ -100,7 +100,7 @@ export function relayEndpointIncumbentProbeCommand(nodePath: string, sockPath: s
|
||||
`printf '%s\\n' ${shellEscape(PROBE_BEGIN)}`,
|
||||
'if [ -S "$sock" ]; then',
|
||||
" printf 'PRESENT=yes\\n'",
|
||||
` listen=$("$node" -e ${shellEscape(CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`,
|
||||
` listen=$("$node" -e ${shellEscape(RELAY_CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`,
|
||||
' [ -n "$listen" ] || listen=unknown',
|
||||
'else',
|
||||
" printf 'PRESENT=no\\n'",
|
||||
|
||||
@@ -15,6 +15,11 @@ vi.mock('./ssh-connection-utils', () => ({
|
||||
shellEscape: (s: string) => `'${s}'`
|
||||
}))
|
||||
|
||||
// The previous-build pin has its own tests; here it names a build that is never a candidate.
|
||||
vi.mock('./remote-install-previous-version', () => ({
|
||||
findPreviousRemoteInstall: vi.fn().mockResolvedValue({ state: 'ok', dirName: 'relay-0.1.0+fff' })
|
||||
}))
|
||||
|
||||
import { existsSync, readFileSync } from 'node:fs'
|
||||
import {
|
||||
readLocalFullVersion,
|
||||
|
||||
@@ -201,7 +201,7 @@ export function probeFileExistsCommand(host: RemoteHostPlatform, remotePath: str
|
||||
)
|
||||
}
|
||||
|
||||
type WindowsRelayLivenessOptions = {
|
||||
export type WindowsRelayLivenessOptions = {
|
||||
nodePath: string
|
||||
pipePaths: string[]
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
restrictWindowsRelayEndpointCredential
|
||||
} from './relay-endpoint-credential-publication'
|
||||
import { SKILL_RELAY_CAPABILITIES } from './skill-install-handler'
|
||||
import { publishRelayPid } from './relay-pid-publication'
|
||||
|
||||
export async function runRelayDaemon(options: RelayLaunchOptions): Promise<void> {
|
||||
if (options.detached && options.logFile) {
|
||||
@@ -105,6 +106,7 @@ export async function runRelayDaemon(options: RelayLaunchOptions): Promise<void>
|
||||
// exits inside start() and never reaches the credential file, so racing starters cannot
|
||||
// rotate the secret a surviving daemon enforces.
|
||||
await reconnectListener.start()
|
||||
publishRelayPid()
|
||||
reconnectListener.setEndpointCredential(publishRelayEndpointCredential(options.credentialFile))
|
||||
agentHooks.publishEndpointFile()
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { existsSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { RELAY_PID_FILENAME, RELAY_VERSION_FILENAME } from '../shared/relay-artifacts'
|
||||
import { publishRelayPid } from './relay-pid-publication'
|
||||
|
||||
describe('publishRelayPid', () => {
|
||||
const directories: string[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true })))
|
||||
})
|
||||
|
||||
function versionDir(installed: boolean): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'relay-pid-'))
|
||||
directories.push(dir)
|
||||
writeFileSync(join(dir, 'relay.js'), '')
|
||||
if (installed) {
|
||||
writeFileSync(join(dir, RELAY_VERSION_FILENAME), '0.1.0+abc\n')
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
it('records the daemon PID beside an installed relay.js', () => {
|
||||
const dir = versionDir(true)
|
||||
publishRelayPid(join(dir, 'relay.js'))
|
||||
expect(readFileSync(join(dir, RELAY_PID_FILENAME), 'utf8')).toBe(`${process.pid}\n`)
|
||||
expect(readdirSync(dir).filter((name) => name.endsWith('.tmp'))).toEqual([])
|
||||
})
|
||||
|
||||
it('leaves a build dir without .version untouched', () => {
|
||||
const dir = versionDir(false)
|
||||
publishRelayPid(join(dir, 'relay.js'))
|
||||
expect(existsSync(join(dir, RELAY_PID_FILENAME))).toBe(false)
|
||||
})
|
||||
|
||||
it('does not throw when the entry cannot be resolved', () => {
|
||||
expect(() => publishRelayPid(join(tmpdir(), 'missing-relay-dir', 'relay.js'))).not.toThrow()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,39 @@
|
||||
import { existsSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { RELAY_PID_FILENAME, RELAY_VERSION_FILENAME } from '../shared/relay-artifacts'
|
||||
import { relayLogLine } from './relay-diagnostic-log'
|
||||
|
||||
/**
|
||||
* Record this daemon's PID in its version dir once it owns its socket, so version GC can
|
||||
* tell a stale socket from a live daemon without connecting to one that is about to idle.
|
||||
*/
|
||||
export function publishRelayPid(entry: string | undefined = process.argv[1]): void {
|
||||
if (!entry) {
|
||||
return
|
||||
}
|
||||
let temporary: string | null = null
|
||||
try {
|
||||
const versionDir = dirname(realpathSync(entry))
|
||||
// Why: only an installed version dir carries `.version`; a dev or test build dir is not GC'd.
|
||||
if (!existsSync(join(versionDir, RELAY_VERSION_FILENAME))) {
|
||||
return
|
||||
}
|
||||
temporary = join(versionDir, `${RELAY_PID_FILENAME}.${process.pid}.tmp`)
|
||||
writeFileSync(temporary, `${process.pid}\n`, { mode: 0o600 })
|
||||
// Why rename: GC must never read a half-written PID as a different, dead process.
|
||||
renameSync(temporary, join(versionDir, RELAY_PID_FILENAME))
|
||||
temporary = null
|
||||
} catch (error) {
|
||||
relayLogLine(
|
||||
`[relay] Could not record relay PID: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
} finally {
|
||||
if (temporary) {
|
||||
try {
|
||||
rmSync(temporary, { force: true })
|
||||
} catch {
|
||||
// A leftover temp file is inert; GC reads only the renamed name.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,9 @@ export const RELAY_VERSION_FILENAME = '.version'
|
||||
/** Written last by the installer; its absence means a torn install. */
|
||||
export const RELAY_INSTALL_COMPLETE_FILENAME = '.install-complete'
|
||||
|
||||
/** PID of the last relay daemon that bound a socket from this version dir; GC liveness evidence. */
|
||||
export const RELAY_PID_FILENAME = '.relay-pid'
|
||||
|
||||
/** Artifacts every relay must have; the remote install probe requires each one. */
|
||||
export function relayArtifactFilenames(isWindows: boolean): string[] {
|
||||
return RELAY_ARTIFACTS.filter(
|
||||
|
||||
Reference in New Issue
Block a user