Merge remote-tracking branch 'origin/fix/ssh-relay-endpoint-takeover' into adhoc/ssh-sweep-combined

This commit is contained in:
Neil
2026-09-01 13:37:07 -07:00
10 changed files with 1366 additions and 13 deletions
+2 -2
View File
@@ -186,9 +186,9 @@ describe('deployAndLaunchRelay', () => {
expect(progress).toContain('Starting relay...')
})
it('does not launch fresh after unconfirmed stale-socket cleanup', async () => {
it('does not launch fresh after an unconfirmed endpoint-incumbent probe', async () => {
const conn = makeMockConnection()
const unconfirmedCleanup = Object.assign(new Error('socket cleanup still running'), {
const unconfirmedCleanup = Object.assign(new Error('endpoint probe still running'), {
sshChannelCloseConfirmed: false
})
vi.mocked(waitForSentinel).mockRejectedValueOnce(new Error('stale relay reconnect failed'))
+20 -9
View File
@@ -77,6 +77,8 @@ import {
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
import { powerShellCommand, powerShellLiteral, powerShellNativeArg } from './ssh-remote-powershell'
import { relaySocketNameForInstanceId } from './ssh-relay-instance-id'
import { resolveRelayEndpointBeforeRelaunch } from './ssh-relay-endpoint-takeover'
import { sweepSupersededRelayEndpoints } from './ssh-relay-superseded-endpoints'
import { isSshSessionLimitError } from './ssh-session-limit-error'
import {
isWindowsRelayPipePath,
@@ -580,6 +582,17 @@ async function deployAndLaunchRelayAttempt(
hostPlatform,
recoverOneStaleRelayUploadStageCommand(hostPlatform, uploadStagePoolDir)
)
.catch(() => {})
// Why before GC: a superseded relay pins its version dir via the live-socket probe, so the
// sweep has to settle first or GC keeps every orphan's tree forever.
.then(() =>
sweepSupersededRelayEndpoints(conn, hostPlatform, {
remoteHome,
currentRelayDir: remoteRelayDir,
sockName: relaySocketNameForInstanceId(relayInstanceId),
nodePath: launched.nodePath
})
)
.catch(() => {})
.then(() =>
gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, {
@@ -1457,17 +1470,15 @@ async function launchRelay(
} catch (err) {
signal?.throwIfAborted()
console.warn(
'[ssh-relay] Socket reconnect failed, launching fresh relay:',
'[ssh-relay] Socket reconnect failed, establishing what owns the endpoint:',
err instanceof Error ? err.message : String(err)
)
// Why: stale socket from a crashed relay — remove it so the fresh launch can bind at the same path.
await execCommand(conn, `rm -f ${shellEscape(sockFile)}`, { signal }).catch(
(cleanupErr) => {
if (isUnconfirmedSshCommandTermination(cleanupErr)) {
throw cleanupErr
}
}
)
// Why not `rm -f`: unlinking does not close the listener the incumbent already holds,
// so a refused --connect (version mismatch, rotated credential) used to leave a live
// relay running forever with its PTYs while a replacement bound the same path (#8585).
await resolveRelayEndpointBeforeRelaunch(conn, hostPlatform, nodePath, sockFile, err, {
signal
})
signal?.throwIfAborted()
}
}
@@ -0,0 +1,170 @@
/**
* The probe and reap scripts run on someone else's machine and decide whether a process is
* signalled, so the shell itself is the part worth testing for real. These cases run the
* generated scripts through /bin/sh against real unix sockets and real processes.
*/
import { execFile, spawn, type ChildProcess } from 'node:child_process'
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'
import {
isReapableRelayHusk,
parseRelayEndpointIncumbentProbe,
relayEndpointIncumbentProbeCommand,
type RelayEndpointIncumbent
} from './ssh-relay-endpoint-incumbent'
import { reapEmptyRelayHuskCommand } from './ssh-relay-endpoint-takeover'
const posixOnly = process.platform === 'win32' ? describe.skip : describe
const FAKE_RELAY_SOURCE = `
const net = require('net')
const sock = process.argv[process.argv.indexOf('--sock-path') + 1]
if (process.argv.includes('--with-child')) {
require('child_process').spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {
stdio: 'ignore'
})
}
net.createServer(() => {}).listen(sock, () => process.stdout.write('READY\\n'))
process.on('SIGTERM', () => process.exit(0))
`
function sh(script: string): Promise<string> {
return new Promise((resolve, reject) => {
execFile('/bin/sh', ['-c', script], { timeout: 20_000 }, (error, stdout) => {
if (error) {
reject(error)
return
}
resolve(stdout)
})
})
}
let workDir: string
let hasLsof = false
const running: ChildProcess[] = []
function startFakeRelay(sockPath: string, withChild = false): Promise<ChildProcess> {
const args = [join(workDir, 'relay.js'), '--sock-path', sockPath]
if (withChild) {
args.push('--with-child')
}
const child = spawn(process.execPath, args, { stdio: ['ignore', 'pipe', 'ignore'] })
running.push(child)
return new Promise((resolve, reject) => {
child.stdout.on('data', (chunk: Buffer) => {
if (chunk.toString().includes('READY')) {
resolve(child)
}
})
child.on('exit', () => reject(new Error('fake relay exited before listening')))
})
}
async function probe(sockPath: string): Promise<RelayEndpointIncumbent> {
const output = await sh(relayEndpointIncumbentProbeCommand(process.execPath, sockPath))
return parseRelayEndpointIncumbentProbe(sockPath, output)
}
beforeAll(async () => {
workDir = mkdtempSync(join(tmpdir(), 'orca-relay-incumbent-'))
writeFileSync(join(workDir, 'relay.js'), FAKE_RELAY_SOURCE)
hasLsof = await sh('command -v lsof >/dev/null 2>&1 && echo yes || echo no').then(
(out) => out.trim() === 'yes'
)
})
afterEach(() => {
while (running.length > 0) {
running.pop()?.kill('SIGKILL')
}
})
afterAll(() => {
rmSync(workDir, { recursive: true, force: true })
})
it('runs the holder-enumeration assertions on this machine', () => {
// Why asserted rather than assumed: the cases below degrade to verdict-only checks without
// lsof, and a silently degraded suite would stop covering the reap gate entirely.
expect(hasLsof).toBe(true)
})
posixOnly('relay endpoint probe against a real socket', () => {
it('reports live, and identifies the holding process, for a listening relay', async () => {
const sockPath = join(workDir, 'live.sock')
const relay = await startFakeRelay(sockPath)
const incumbent = await probe(sockPath)
expect(incumbent.verdict).toBe('live')
expect(incumbent.evidence).toBe('accepted-connection')
expect(incumbent.socketPresent).toBe(true)
if (!hasLsof) {
return
}
expect(incumbent.holders.map((holder) => holder.pid)).toEqual([relay.pid])
expect(incumbent.holders[0]).toMatchObject({ matchesRelayArgv: true, childCount: 0 })
expect(isReapableRelayHusk(incumbent)).toBe(true)
})
it('refuses to call a relay with a live child an empty husk', async () => {
const sockPath = join(workDir, 'busy.sock')
await startFakeRelay(sockPath, true)
const incumbent = await probe(sockPath)
expect(incumbent.verdict).toBe('live')
if (!hasLsof) {
return
}
expect(incumbent.holders[0].childCount).toBeGreaterThan(0)
expect(isReapableRelayHusk(incumbent)).toBe(false)
})
it('reports exited for a socket inode a SIGKILLed relay left behind', async () => {
const sockPath = join(workDir, 'stale.sock')
const relay = await startFakeRelay(sockPath)
relay.kill('SIGKILL')
await new Promise((resolve) => relay.on('exit', resolve))
const incumbent = await probe(sockPath)
expect(incumbent.socketPresent).toBe(true)
expect(incumbent.verdict).toBe(hasLsof ? 'exited' : 'unverifiable')
})
it('reports no listener for a path that was never bound', async () => {
const incumbent = await probe(join(workDir, 'never-existed.sock'))
expect(incumbent.socketPresent).toBe(false)
expect(incumbent.verdict).toBe(hasLsof ? 'exited' : 'unverifiable')
})
})
posixOnly('empty relay husk reap against a real process', () => {
it('terminates a proven-empty relay and confirms the pid is gone', async () => {
const sockPath = join(workDir, 'husk.sock')
const relay = await startFakeRelay(sockPath)
const output = await sh(reapEmptyRelayHuskCommand(relay.pid!, sockPath))
expect(output.trim()).toBe('GONE')
})
it('refuses to signal a relay that acquired a child after it was probed', async () => {
const sockPath = join(workDir, 'raced.sock')
const relay = await startFakeRelay(sockPath, true)
const output = await sh(reapEmptyRelayHuskCommand(relay.pid!, sockPath))
expect(output.trim()).toBe('BUSY')
expect(relay.killed).toBe(false)
})
it('refuses to signal a pid whose argv is not this relay at this socket', async () => {
const sockPath = join(workDir, 'mismatch.sock')
await startFakeRelay(sockPath)
const bystander = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {
stdio: 'ignore'
})
running.push(bystander)
const output = await sh(reapEmptyRelayHuskCommand(bystander.pid!, sockPath))
expect(output.trim()).toBe('MISMATCH')
expect(bystander.killed).toBe(false)
})
})
@@ -0,0 +1,240 @@
import { describe, expect, it, vi } from 'vitest'
const execCommand = vi.fn()
vi.mock('./ssh-relay-deploy-helpers', () => ({
execCommand: (...args: unknown[]) => execCommand(...args),
isUnconfirmedSshCommandTermination: (error: unknown) =>
(error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false
}))
import {
describeRelayEndpointIncumbent,
isReapableRelayHusk,
mayLaunchOverRelayEndpoint,
parseRelayEndpointIncumbentProbe,
probeRelayEndpointIncumbent,
relayEndpointIncumbentProbeCommand,
withHandshakeRefusalEvidence,
type RelayEndpointIncumbent
} from './ssh-relay-endpoint-incumbent'
import type { SshConnection } from './ssh-connection'
import { getRemoteHostPlatform } from './ssh-remote-platform'
const SOCK = '/home/u/.orca-remote/relay-0.1.0+aaaa/relay-deadbeef.sock'
const POSIX_HOST = getRemoteHostPlatform('linux-x64')
const WINDOWS_HOST = getRemoteHostPlatform('win32-x64')
function probeOutput(lines: string[]): string {
return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n')
}
describe('parseRelayEndpointIncumbentProbe', () => {
it('reports live when the socket accepted a connection', () => {
const incumbent = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=4242 yes 13'])
)
expect(incumbent.verdict).toBe('live')
expect(incumbent.evidence).toBe('accepted-connection')
expect(incumbent.holders).toEqual([{ pid: 4242, matchesRelayArgv: true, childCount: 13 }])
})
it('reports live when a process still holds an inode that refuses connections', () => {
const incumbent = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof', 'HOLDER=91 yes 2'])
)
expect(incumbent.verdict).toBe('live')
expect(incumbent.evidence).toBe('holder-process')
})
it('reports exited only when the connect was refused AND nothing holds the socket', () => {
const incumbent = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])
)
expect(incumbent.verdict).toBe('exited')
expect(incumbent.evidence).toBe('no-holder')
expect(incumbent.socketPresent).toBe(true)
})
it('reports unverifiable when the host cannot enumerate socket holders', () => {
const incumbent = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=unavailable'])
)
expect(incumbent.verdict).toBe('unverifiable')
expect(incumbent.holdersEnumerable).toBe(false)
})
it('reports unverifiable when the connect probe timed out', () => {
const incumbent = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=lsof'])
)
expect(incumbent.verdict).toBe('unverifiable')
})
it('reports unverifiable for truncated or garbled probe output', () => {
expect(parseRelayEndpointIncumbentProbe(SOCK, 'PRESENT=yes\nLISTEN=refused').verdict).toBe(
'unverifiable'
)
expect(parseRelayEndpointIncumbentProbe(SOCK, '').verdict).toBe('unverifiable')
})
it('drops holder lines that do not carry a usable pid', () => {
const incumbent = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof', 'HOLDER=- no unknown'])
)
expect(incumbent.holders).toEqual([])
expect(incumbent.verdict).toBe('exited')
})
it('keeps an unreadable child count as null rather than zero', () => {
const [holder] = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=7 yes unknown'])
).holders
expect(holder.childCount).toBeNull()
})
})
describe('probeRelayEndpointIncumbent', () => {
it('never asserts death when the probe itself could not run', async () => {
execCommand.mockRejectedValueOnce(new Error('channel closed'))
const incumbent = await probeRelayEndpointIncumbent(
{} as SshConnection,
POSIX_HOST,
'/usr/bin/node',
SOCK
)
expect(incumbent.verdict).toBe('unverifiable')
expect(incumbent.holders).toEqual([])
})
it('does not shell out on Windows hosts, where the endpoint is a named pipe', async () => {
execCommand.mockClear()
const incumbent = await probeRelayEndpointIncumbent(
{} as SshConnection,
WINDOWS_HOST,
'node.exe',
SOCK
)
expect(execCommand).not.toHaveBeenCalled()
expect(incumbent.verdict).toBe('unverifiable')
})
})
describe('relayEndpointIncumbentProbeCommand', () => {
it('ANDs the lsof selectors so it cannot match unrelated unix-socket holders', () => {
expect(relayEndpointIncumbentProbeCommand('/usr/bin/node', SOCK)).toContain(
'lsof -t -a -U "$sock"'
)
})
it('never mutates the host: no unlink, no signal', () => {
const command = relayEndpointIncumbentProbeCommand('/usr/bin/node', SOCK)
expect(command).not.toMatch(/\brm\b/)
expect(command).not.toMatch(/\bkill\b/)
})
})
describe('withHandshakeRefusalEvidence', () => {
it('upgrades an unenumerable endpoint to live when the daemon answered the handshake', () => {
const probed = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
)
const incumbent = withHandshakeRefusalEvidence(probed)
expect(incumbent.verdict).toBe('live')
expect(incumbent.evidence).toBe('handshake-refusal')
expect(mayLaunchOverRelayEndpoint(incumbent)).toBe(false)
})
it('leaves stronger evidence in place', () => {
const probed = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof'])
)
expect(withHandshakeRefusalEvidence(probed).evidence).toBe('accepted-connection')
})
})
describe('mayLaunchOverRelayEndpoint', () => {
const verdicts: RelayEndpointIncumbent['verdict'][] = ['live', 'unverifiable', 'exited']
it.each(verdicts)('permits a relaunch for %s only when it is not live', (verdict) => {
const incumbent = { ...parseRelayEndpointIncumbentProbe(SOCK, ''), verdict }
expect(mayLaunchOverRelayEndpoint(incumbent)).toBe(verdict !== 'live')
})
})
describe('isReapableRelayHusk', () => {
const husk = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=500 yes 0'])
)
it('accepts a single proven relay holder with zero children', () => {
expect(isReapableRelayHusk(husk)).toBe(true)
})
it('refuses a relay that still holds children', () => {
expect(
isReapableRelayHusk({
...husk,
holders: [{ pid: 500, matchesRelayArgv: true, childCount: 1 }]
})
).toBe(false)
})
it('refuses a holder whose child count could not be read', () => {
expect(
isReapableRelayHusk({
...husk,
holders: [{ pid: 500, matchesRelayArgv: true, childCount: null }]
})
).toBe(false)
})
it('refuses a holder whose argv is not this relay at this socket', () => {
expect(
isReapableRelayHusk({
...husk,
holders: [{ pid: 500, matchesRelayArgv: false, childCount: 0 }]
})
).toBe(false)
})
it('refuses when more than one process holds the socket', () => {
expect(
isReapableRelayHusk({
...husk,
holders: [
{ pid: 500, matchesRelayArgv: true, childCount: 0 },
{ pid: 501, matchesRelayArgv: true, childCount: 0 }
]
})
).toBe(false)
})
it('refuses an unverifiable endpoint however empty it looks', () => {
expect(isReapableRelayHusk({ ...husk, verdict: 'unverifiable' })).toBe(false)
expect(isReapableRelayHusk({ ...husk, holdersEnumerable: false })).toBe(false)
})
})
describe('describeRelayEndpointIncumbent', () => {
it('distinguishes "no holders" from "could not enumerate holders"', () => {
const none = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])
)
const unknown = parseRelayEndpointIncumbentProbe(
SOCK,
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=unavailable'])
)
expect(describeRelayEndpointIncumbent(none)).toContain('holders=none')
expect(describeRelayEndpointIncumbent(unknown)).toContain('holders=unenumerable')
})
})
@@ -0,0 +1,285 @@
/**
* Who currently owns a relay socket path, answered with host evidence.
*
* The client used to answer this by assumption: a failed `--connect` was read as "the relay
* crashed", the socket was `rm -f`'d, and a fresh relay bound the same path. Unlinking a unix
* socket does not close the listener the incumbent already holds, so an alive-but-refusing
* relay (the `RelayVersionMismatchError` case, and the credential-rotation case) was left
* running forever with its PTYs (#8585).
*
* The verdict vocabulary is fixed by docs/reference/ssh-execution-boundary.md — `live` /
* `unverifiable` / `exited`, with no synonyms and no collapsing. Two consequences are load
* bearing here:
*
* - `exited` is a claim about **this endpoint**, not about every relay on the host. It means
* nothing holds this socket path, established positively (a connect that was refused *and*
* an enumeration that found no holder). A relay whose socket was already unlinked is
* invisible to this probe by construction — that is what the superseded sweep is for.
* - a probe that could not run, a host without `lsof`, or a connect that failed for any other
* reason is `unverifiable`. It never authorizes unlinking, rebinding over, or signalling.
*/
import type { SshConnection } from './ssh-connection'
import { shellEscape } from './ssh-connection-utils'
import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers'
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
export type RelayEndpointVerdict = 'live' | 'unverifiable' | 'exited'
export type RelayEndpointEvidence =
| 'accepted-connection'
| 'handshake-refusal'
| 'holder-process'
| 'no-holder'
| 'inconclusive'
export type RelayEndpointHolder = {
pid: number
/** The holder's argv names relay.js AND this exact socket path. */
matchesRelayArgv: boolean
/** Direct children, or null when `pgrep` could not answer. Never guessed. */
childCount: number | null
}
export type RelayEndpointIncumbent = {
sockPath: string
verdict: RelayEndpointVerdict
evidence: RelayEndpointEvidence
socketPresent: boolean
/** Pids proven to hold this exact socket. Empty when the host could not enumerate them. */
holders: RelayEndpointHolder[]
/** False when no enumeration tool was available — an empty `holders` then proves nothing. */
holdersEnumerable: boolean
}
const PROBE_BEGIN = 'ORCA-INCUMBENT-BEGIN'
const PROBE_END = 'ORCA-INCUMBENT-END'
const CONNECT_PROBE_TIMEOUT_MS = 1000
// Why ES5 syntax: nodePath may be a host-resolved system node, not the bundled one.
const CONNECT_PROBE_JS = [
'var s=require("net").connect(process.argv[1]);',
'var done=false;',
'function say(v){if(done)return;done=true;try{s.destroy()}catch(e){};',
'process.stdout.write(v);process.exit(0)}',
's.on("connect",function(){say("accepted")});',
's.on("error",function(e){',
'say(e.code==="ECONNREFUSED"?"refused":e.code==="ENOENT"?"absent":"unknown")});',
`setTimeout(function(){say("unknown")},${CONNECT_PROBE_TIMEOUT_MS})`
].join('')
/**
* A POSIX probe that reports only what the host actually observed. Every field has an
* explicit "could not tell" value; nothing is inferred from a missing tool.
*/
export function relayEndpointIncumbentProbeCommand(nodePath: string, sockPath: string): string {
const sock = shellEscape(sockPath)
const node = shellEscape(nodePath)
return [
`sock=${sock}`,
`node=${node}`,
`printf '%s\\n' ${shellEscape(PROBE_BEGIN)}`,
'if [ -S "$sock" ]; then',
" printf 'PRESENT=yes\\n'",
` listen=$("$node" -e ${shellEscape(CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`,
' [ -n "$listen" ] || listen=unknown',
'else',
" printf 'PRESENT=no\\n'",
' listen=absent',
'fi',
'printf \'LISTEN=%s\\n\' "$listen"',
'if command -v lsof >/dev/null 2>&1; then',
" printf 'HOLDERS_SOURCE=lsof\\n'",
// Why -a: lsof ORs its selectors, so without it every unix-socket holder on the box
// would be reported as holding this path (#8762).
' for pid in $(lsof -t -a -U "$sock" 2>/dev/null); do',
' args=$(ps -o args= -p "$pid" 2>/dev/null | tr "\\n" " ")',
' match=no',
' case "$args" in *relay.js*"$sock"*) match=yes ;; esac',
' kids=unknown',
' if command -v pgrep >/dev/null 2>&1; then',
' kids=$(pgrep -P "$pid" 2>/dev/null | grep -c .)',
' fi',
' printf \'HOLDER=%s %s %s\\n\' "$pid" "$match" "$kids"',
' done',
'else',
" printf 'HOLDERS_SOURCE=unavailable\\n'",
'fi',
`printf '%s\\n' ${shellEscape(PROBE_END)}`
].join('\n')
}
export function parseRelayEndpointIncumbentProbe(
sockPath: string,
output: string
): RelayEndpointIncumbent {
const lines = output.split('\n').map((line) => line.trim())
if (!lines.includes(PROBE_BEGIN) || !lines.includes(PROBE_END)) {
return unverifiableEndpoint(sockPath)
}
const socketPresent = lines.includes('PRESENT=yes')
const listen = lines.find((line) => line.startsWith('LISTEN='))?.slice('LISTEN='.length) ?? ''
const holdersEnumerable = lines.includes('HOLDERS_SOURCE=lsof')
const holders = lines
.filter((line) => line.startsWith('HOLDER='))
.map((line) => parseHolder(line.slice('HOLDER='.length)))
.filter((holder): holder is RelayEndpointHolder => holder !== null)
if (listen === 'accepted') {
return {
sockPath,
verdict: 'live',
evidence: 'accepted-connection',
socketPresent,
holders,
holdersEnumerable
}
}
if (holders.length > 0) {
// The inode is held by a running process that is not accepting — wedged, not gone.
return {
sockPath,
verdict: 'live',
evidence: 'holder-process',
socketPresent,
holders,
holdersEnumerable
}
}
if (holdersEnumerable && (listen === 'refused' || listen === 'absent')) {
return {
sockPath,
verdict: 'exited',
evidence: 'no-holder',
socketPresent,
holders,
holdersEnumerable
}
}
return { ...unverifiableEndpoint(sockPath), socketPresent, holders, holdersEnumerable }
}
function parseHolder(value: string): RelayEndpointHolder | null {
const [rawPid, rawMatch, rawKids] = value.split(/\s+/)
const pid = Number.parseInt(rawPid ?? '', 10)
if (!Number.isInteger(pid) || pid <= 0) {
return null
}
const childCount = Number.parseInt(rawKids ?? '', 10)
return {
pid,
matchesRelayArgv: rawMatch === 'yes',
childCount: Number.isInteger(childCount) && childCount >= 0 ? childCount : null
}
}
function unverifiableEndpoint(sockPath: string): RelayEndpointIncumbent {
return {
sockPath,
verdict: 'unverifiable',
evidence: 'inconclusive',
socketPresent: false,
holders: [],
holdersEnumerable: false
}
}
export async function probeRelayEndpointIncumbent(
conn: SshConnection,
hostPlatform: RemoteHostPlatform,
nodePath: string,
sockPath: string,
options?: { signal?: AbortSignal }
): Promise<RelayEndpointIncumbent> {
// Windows relays are named pipes: there is no inode to unlink and no `lsof`, so the
// orphan-by-unlink mechanism this probe defends against cannot occur there.
if (isWindowsRemoteHost(hostPlatform)) {
return unverifiableEndpoint(sockPath)
}
try {
const output = await execCommand(conn, relayEndpointIncumbentProbeCommand(nodePath, sockPath), {
wrapCommand: true,
signal: options?.signal
})
return parseRelayEndpointIncumbentProbe(sockPath, output)
} catch (err) {
// An exec whose channel never confirmed close may still be running remotely; the caller
// must not race a detached launch against it.
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
// Any other unanswered probe observes nothing. It is never evidence of death.
return unverifiableEndpoint(sockPath)
}
}
/**
* A relay that told us its version over the wire is `live` by positive host evidence, even on
* a host where nothing can enumerate socket holders.
*/
export function withHandshakeRefusalEvidence(
incumbent: RelayEndpointIncumbent
): RelayEndpointIncumbent {
if (incumbent.verdict === 'live') {
return incumbent
}
return { ...incumbent, verdict: 'live', evidence: 'handshake-refusal' }
}
/**
* May a fresh relay be launched onto this path?
*
* Only `live` forbids it. `unverifiable` is permitted because the *daemon* — not the client —
* performs the takeover: `RelaySocketOwnership.listen` re-probes on EADDRINUSE, refuses to
* steal a path that accepts connections, and only unlinks an inode whose identity is
* unchanged. That check is atomic with the bind, which a client-side `rm -f` can never be.
*/
export function mayLaunchOverRelayEndpoint(incumbent: RelayEndpointIncumbent): boolean {
return incumbent.verdict !== 'live'
}
/**
* A live relay that provably holds nothing: identity confirmed against its argv, exactly one
* holder, and zero children. Reaping it destroys no user work. Anything less is retained —
* killing the wrong pid on someone's remote host is the worst outcome available here.
*/
export function isReapableRelayHusk(incumbent: RelayEndpointIncumbent): boolean {
if (incumbent.verdict !== 'live' || !incumbent.holdersEnumerable) {
return false
}
if (incumbent.holders.length !== 1) {
return false
}
const [holder] = incumbent.holders
return holder.matchesRelayArgv && holder.childCount === 0
}
export function describeRelayEndpointIncumbent(incumbent: RelayEndpointIncumbent): string {
const holders = incumbent.holders
.map((holder) => `${holder.pid}(children=${holder.childCount ?? 'unknown'})`)
.join(',')
return (
`${incumbent.sockPath} verdict=${incumbent.verdict} evidence=${incumbent.evidence} ` +
`holders=${incumbent.holdersEnumerable ? holders || 'none' : 'unenumerable'}`
)
}
/**
* Thrown instead of orphaning: a live relay owns the endpoint and refused us, so the path is
* not ours to rebind. Terminal for this attempt — the user resolves it with Reset Relay,
* which signals the incumbent deliberately and with consent.
*/
export class RelayEndpointHeldError extends Error {
readonly name = 'RelayEndpointHeldError'
constructor(readonly incumbent: RelayEndpointIncumbent) {
super(
`A live relay still owns ${incumbent.sockPath} and refused this connection ` +
`(${describeRelayEndpointIncumbent(incumbent)}). Orca will not replace it, because ` +
'unlinking its socket would strand its terminals. Use Reset Relay for this host to ' +
'stop it, then reconnect.'
)
}
}
export function isRelayEndpointHeldError(err: unknown): err is RelayEndpointHeldError {
return err instanceof RelayEndpointHeldError
}
@@ -0,0 +1,159 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const execCommand = vi.fn()
vi.mock('./ssh-relay-deploy-helpers', () => ({
execCommand: (...args: unknown[]) => execCommand(...args),
isUnconfirmedSshCommandTermination: (error: unknown) =>
(error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false
}))
import { isRelayEndpointHeldError } from './ssh-relay-endpoint-incumbent'
import {
interpretRelayHuskReapOutput,
reapEmptyRelayHuskCommand,
resolveRelayEndpointBeforeRelaunch
} from './ssh-relay-endpoint-takeover'
import { RelayVersionMismatchError } from './ssh-relay-version-mismatch-error'
import type { SshConnection } from './ssh-connection'
import { getRemoteHostPlatform } from './ssh-remote-platform'
const SOCK = '/home/u/.orca-remote/relay-0.1.0+aaaa/relay-deadbeef.sock'
const HOST = getRemoteHostPlatform('linux-x64')
const CONN = {} as SshConnection
function probe(lines: string[]): string {
return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n')
}
function issuedCommands(): string[] {
return execCommand.mock.calls.map((call) => String(call[1]))
}
function resolve(reconnectError: unknown = new Error('connect failed')): Promise<unknown> {
return resolveRelayEndpointBeforeRelaunch(CONN, HOST, '/usr/bin/node', SOCK, reconnectError)
}
beforeEach(() => {
execCommand.mockReset()
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'log').mockImplementation(() => {})
})
describe('incumbent alive and refusing', () => {
it('refuses to rebind a live relay holding PTYs, and signals nothing', async () => {
execCommand.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13'])
)
await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError)
// The whole point of #8585: the incumbent's socket must survive so it is not orphaned.
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
expect(issuedCommands().some((command) => /\bkill\b/.test(command))).toBe(false)
})
it('names the incumbent pid and the Reset Relay escape hatch in the error', async () => {
execCommand.mockResolvedValue(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13'])
)
await expect(resolve()).rejects.toThrow(/3669803\(children=13\)/)
await expect(resolve()).rejects.toThrow(/Reset Relay/)
})
it('treats a version mismatch as live even where holders cannot be enumerated', async () => {
execCommand.mockResolvedValue(
probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
)
const mismatch = new RelayVersionMismatchError('0.1.0+new', '0.1.0+old', '')
await expect(resolve(mismatch)).rejects.toSatisfy(isRelayEndpointHeldError)
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
})
it('reaps a live relay only when it provably holds nothing, and confirms it is gone', async () => {
execCommand
.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
)
.mockResolvedValueOnce('GONE\n')
await expect(resolve()).resolves.toMatchObject({ verdict: 'live' })
expect(issuedCommands()[1]).toContain('kill -TERM "$pid"')
})
it('does not launch over an empty relay whose death could not be confirmed', async () => {
execCommand
.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
)
.mockResolvedValueOnce('LIVE\n')
await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError)
})
it('does not launch over a relay the host refused to signal on its own re-check', async () => {
execCommand
.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
)
.mockResolvedValueOnce('BUSY\n')
await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError)
})
})
describe('incumbent genuinely gone', () => {
it('permits the relaunch without unlinking anything itself', async () => {
execCommand.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])
)
await expect(resolve()).resolves.toMatchObject({ verdict: 'exited', evidence: 'no-holder' })
// The daemon unlinks under an identity check that is atomic with its bind; the client
// cannot be, which is what created the orphan in the first place.
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
})
})
describe('incumbent unverifiable', () => {
it('permits the relaunch but never claims the incumbent exited', async () => {
execCommand.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
)
await expect(resolve()).resolves.toMatchObject({ verdict: 'unverifiable' })
expect(issuedCommands()).toHaveLength(1)
})
it('stays unverifiable when the probe command itself fails', async () => {
execCommand.mockRejectedValueOnce(new Error('exec timeout'))
await expect(resolve()).resolves.toMatchObject({ verdict: 'unverifiable' })
})
})
describe('reapEmptyRelayHuskCommand', () => {
it('re-verifies argv and emptiness on the host immediately before signalling', () => {
const command = reapEmptyRelayHuskCommand(4242, SOCK)
expect(command.indexOf('MISMATCH')).toBeLessThan(command.indexOf('kill -TERM'))
expect(command.indexOf('BUSY')).toBeLessThan(command.indexOf('kill -TERM'))
})
it('sends SIGTERM only, so the relay runs its own socket cleanup', () => {
const command = reapEmptyRelayHuskCommand(4242, SOCK)
expect(command).toContain('kill -TERM')
expect(command).not.toContain('kill -KILL')
expect(command).not.toContain('-9')
})
it('aborts without signalling when the host cannot count children', () => {
expect(reapEmptyRelayHuskCommand(4242, SOCK)).toContain(
"command -v pgrep >/dev/null 2>&1 || { printf 'BUSY\\n'; exit 0; }"
)
})
})
describe('interpretRelayHuskReapOutput', () => {
it('claims reaped only for a post-signal liveness check that failed', () => {
expect(interpretRelayHuskReapOutput('GONE\n')).toBe('reaped')
expect(interpretRelayHuskReapOutput('LIVE\n')).toBe('reap-unconfirmed')
expect(interpretRelayHuskReapOutput('')).toBe('reap-unconfirmed')
expect(interpretRelayHuskReapOutput('unexpected noise')).toBe('reap-unconfirmed')
})
it('reports a host-side refusal as retained rather than as a failed kill', () => {
expect(interpretRelayHuskReapOutput('MISMATCH\n')).toBe('retained-live-work')
expect(interpretRelayHuskReapOutput('BUSY\n')).toBe('retained-live-work')
})
})
+133
View File
@@ -0,0 +1,133 @@
/**
* Deciding whether a relay socket path is ours to take, and acting on the answer.
*
* The only destructive action available here is a SIGTERM to a relay that has been proven —
* by argv, by socket-holder enumeration, and by a zero child count re-checked on the host
* immediately before the signal — to hold nothing at all. Everything else is left running.
* Per docs/reference/ssh-execution-boundary.md, a relay we merely failed to reach is
* `unverifiable`, and `unverifiable` never authorizes a kill or a rebind.
*/
import type { SshConnection } from './ssh-connection'
import { shellEscape } from './ssh-connection-utils'
import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers'
import {
describeRelayEndpointIncumbent,
isReapableRelayHusk,
mayLaunchOverRelayEndpoint,
probeRelayEndpointIncumbent,
RelayEndpointHeldError,
withHandshakeRefusalEvidence,
type RelayEndpointIncumbent
} from './ssh-relay-endpoint-incumbent'
import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error'
import type { RemoteHostPlatform } from './ssh-remote-platform'
/** `reaped` is only reachable from a post-signal `kill -0` that failed. Nothing else claims it. */
export type RelayHuskReapResult = 'reaped' | 'reap-unconfirmed' | 'retained-live-work'
const REAP_CONFIRM_ATTEMPTS = 15
/**
* Signal one relay, re-verifying identity and emptiness inside the same command.
*
* The re-verification is not belt-and-braces: a client can attach and spawn a PTY between the
* probe and the signal, and pids are reused. `MISMATCH`/`BUSY` abort without signalling.
*/
export function reapEmptyRelayHuskCommand(pid: number, sockPath: string): string {
return [
`pid=${shellEscape(String(pid))}`,
`sock=${shellEscape(sockPath)}`,
'args=$(ps -o args= -p "$pid" 2>/dev/null | tr "\\n" " ")',
'case "$args" in *relay.js*"$sock"*) ;; *) printf \'MISMATCH\\n\'; exit 0 ;; esac',
"command -v pgrep >/dev/null 2>&1 || { printf 'BUSY\\n'; exit 0; }",
'kids=$(pgrep -P "$pid" 2>/dev/null | grep -c .)',
'[ "$kids" = "0" ] || { printf \'BUSY\\n\'; exit 0; }',
// SIGTERM only: the relay's own handler disposes and unlinks. SIGKILL would leave the
// socket inode behind and skip that shutdown path for no gain on an empty daemon.
'kill -TERM "$pid" 2>/dev/null || true',
'i=0',
`while [ $i -lt ${REAP_CONFIRM_ATTEMPTS} ]; do`,
' kill -0 "$pid" 2>/dev/null || { printf \'GONE\\n\'; exit 0; }',
' sleep 0.2',
' i=$((i+1))',
'done',
"printf 'LIVE\\n'"
].join('\n')
}
export function interpretRelayHuskReapOutput(output: string): RelayHuskReapResult {
const state = output.trim().split('\n').pop()?.trim()
if (state === 'GONE') {
return 'reaped'
}
// The host refused on its own re-check: what is there is not the empty relay we probed, so
// nothing was signalled and nothing is claimed about it.
if (state === 'MISMATCH' || state === 'BUSY') {
return 'retained-live-work'
}
return 'reap-unconfirmed'
}
export async function reapEmptyRelayHusk(
conn: SshConnection,
incumbent: RelayEndpointIncumbent,
options?: { signal?: AbortSignal }
): Promise<RelayHuskReapResult> {
const holder = incumbent.holders[0]
if (!holder) {
return 'retained-live-work'
}
try {
const output = await execCommand(
conn,
reapEmptyRelayHuskCommand(holder.pid, incumbent.sockPath),
{ wrapCommand: true, signal: options?.signal }
)
return interpretRelayHuskReapOutput(output)
} catch (err) {
if (isUnconfirmedSshCommandTermination(err)) {
throw err
}
return 'reap-unconfirmed'
}
}
/**
* Called when `--connect` to an existing socket failed and the caller is about to launch a
* replacement at the same path. Resolves to nothing when the launch may proceed; throws
* `RelayEndpointHeldError` when a live relay owns the path and holds work.
*
* `unverifiable` deliberately permits the launch: the daemon, not the client, performs the
* takeover. `RelaySocketOwnership.listen` re-probes on EADDRINUSE, refuses a path that accepts
* connections, and only unlinks an inode whose identity is unchanged — a check that is atomic
* with the bind, which a client-side `rm -f` can never be.
*/
export async function resolveRelayEndpointBeforeRelaunch(
conn: SshConnection,
hostPlatform: RemoteHostPlatform,
nodePath: string,
sockPath: string,
reconnectError: unknown,
options?: { signal?: AbortSignal }
): Promise<RelayEndpointIncumbent> {
const probed = await probeRelayEndpointIncumbent(conn, hostPlatform, nodePath, sockPath, options)
// A daemon that answered the handshake with its own version is live by positive host
// evidence, even where nothing can enumerate socket holders.
const incumbent = isRelayVersionMismatchError(reconnectError)
? withHandshakeRefusalEvidence(probed)
: probed
console.warn(`[ssh-relay] Relay endpoint incumbent: ${describeRelayEndpointIncumbent(incumbent)}`)
if (mayLaunchOverRelayEndpoint(incumbent)) {
return incumbent
}
if (!isReapableRelayHusk(incumbent)) {
throw new RelayEndpointHeldError(incumbent)
}
const result = await reapEmptyRelayHusk(conn, incumbent, options)
if (result !== 'reaped') {
throw new RelayEndpointHeldError(incumbent)
}
console.log(`[ssh-relay] Reaped empty relay husk holding ${sockPath}`)
return incumbent
}
+15 -2
View File
@@ -6,6 +6,7 @@ import type { BrowserWindow } from 'electron'
import { deployAndLaunchRelay } from './ssh-relay-deploy'
import { execCommand } from './ssh-relay-deploy-helpers'
import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error'
import { isRelayEndpointHeldError } from './ssh-relay-endpoint-incumbent'
import { replayPendingSshPtyKills } from './ssh-pending-pty-kill-replay'
import { SshChannelMultiplexer } from './ssh-channel-multiplexer'
import { SshPtyProvider } from '../providers/ssh-pty-provider'
@@ -629,7 +630,13 @@ export class SshRelaySession {
}
// Why: terminal on first connect — a deployed binary against a still-running legacy daemon, or a
// claim another connection holds. Notify the callback but still rethrow.
if (isRelayVersionMismatchError(err) || isSshOwnerAdmissionBlockedError(err)) {
// RelayEndpointHeldError is terminal for the same reason: a live incumbent owns the
// socket path, and backoff cannot make it hand it over. The user resolves it.
if (
isRelayVersionMismatchError(err) ||
isRelayEndpointHeldError(err) ||
isSshOwnerAdmissionBlockedError(err)
) {
console.warn(
`[ssh-relay-session] Terminal relay error on initial connect for ${this.targetId}: ${err.message}`
)
@@ -783,7 +790,13 @@ export class SshRelaySession {
}
// Why terminal: neither a version mismatch nor a blocked owner claim is reconcilable by backoff
// retry, so fire the typed callback and drop out of 'reconnecting'.
if (isRelayVersionMismatchError(err) || isSshOwnerAdmissionBlockedError(err)) {
// RelayEndpointHeldError is terminal for the same reason: a live incumbent owns the
// socket path, and backoff cannot make it hand it over. The user resolves it.
if (
isRelayVersionMismatchError(err) ||
isRelayEndpointHeldError(err) ||
isSshOwnerAdmissionBlockedError(err)
) {
console.warn(
`[ssh-relay-session] Terminal relay error for ${this.targetId}: ${err.message}`
)
@@ -0,0 +1,164 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const execCommand = vi.fn()
vi.mock('./ssh-relay-deploy-helpers', () => ({
execCommand: (...args: unknown[]) => execCommand(...args),
isUnconfirmedSshCommandTermination: (error: unknown) =>
(error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false
}))
import { parseRelayEndpointIncumbentProbe } from './ssh-relay-endpoint-incumbent'
import {
classifySupersededRelay,
supersededRelayEndpointListCommand,
sweepSupersededRelayEndpoints
} from './ssh-relay-superseded-endpoints'
import type { SshConnection } from './ssh-connection'
import { getRemoteHostPlatform } from './ssh-remote-platform'
const HOME = '/home/u'
const SOCK_NAME = 'relay-deadbeef.sock'
const CURRENT_DIR = `${HOME}/.orca-remote/relay-0.1.0+bd3ec370d21d`
const OLD_SOCK = `${HOME}/.orca-remote/relay-0.1.0+7175e0a40ea7/${SOCK_NAME}`
const HOST = getRemoteHostPlatform('linux-x64')
const WINDOWS_HOST = getRemoteHostPlatform('win32-x64')
const CONN = {} as SshConnection
const SWEEP = {
remoteHome: HOME,
currentRelayDir: CURRENT_DIR,
sockName: SOCK_NAME,
nodePath: '/usr/bin/node'
}
function probe(lines: string[]): string {
return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n')
}
function incumbent(lines: string[]): ReturnType<typeof parseRelayEndpointIncumbentProbe> {
return parseRelayEndpointIncumbentProbe(OLD_SOCK, probe(lines))
}
function issuedCommands(): string[] {
return execCommand.mock.calls.map((call) => String(call[1]))
}
beforeEach(() => {
execCommand.mockReset()
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'log').mockImplementation(() => {})
})
describe('supersededRelayEndpointListCommand', () => {
it('globs sibling version dirs for this target socket and skips the current one', () => {
const command = supersededRelayEndpointListCommand(SWEEP)
expect(command).toContain('"$base"/relay-*/"$sock_name"')
expect(command).toContain('[ "$dir" = "$current" ] && continue')
expect(command).toContain(SOCK_NAME)
expect(command).toContain(CURRENT_DIR)
})
})
describe('classifySupersededRelay', () => {
it('retains a live relay that still owns PTYs', () => {
expect(
classifySupersededRelay(
incumbent([
'PRESENT=yes',
'LISTEN=accepted',
'HOLDERS_SOURCE=lsof',
'HOLDER=3669803 yes 13'
])
)
).toBe('retained-live-work')
})
it('nominates only a proven empty relay for reaping', () => {
expect(
classifySupersededRelay(
incumbent(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
)
).toBe('reap-candidate')
})
it('removes only a socket proven to have no holder', () => {
expect(
classifySupersededRelay(incumbent(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof']))
).toBe('stale-endpoint-removed')
})
it('does nothing at all for an unverifiable endpoint', () => {
expect(
classifySupersededRelay(
incumbent(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
)
).toBe('unverifiable')
})
})
describe('sweepSupersededRelayEndpoints', () => {
it('leaves an upgrade-orphaned relay that still owns terminals running, untouched', async () => {
execCommand
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13'])
)
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
expect(findings).toHaveLength(1)
expect(findings[0]).toMatchObject({ sockPath: OLD_SOCK, outcome: 'retained-live-work' })
expect(issuedCommands().some((command) => /\bkill\b/.test(command))).toBe(false)
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
})
it('reaps the empty husk an upgrade leaves behind, once the host confirms it is gone', async () => {
execCommand
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
)
.mockResolvedValueOnce('GONE\n')
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
expect(findings[0].outcome).toBe('reaped')
expect(issuedCommands()[2]).toContain('kill -TERM "$pid"')
})
it('reports reap-unconfirmed rather than reaped when the pid is still there', async () => {
execCommand
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
.mockResolvedValueOnce(
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
)
.mockResolvedValueOnce('LIVE\n')
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
expect(findings[0].outcome).toBe('reap-unconfirmed')
})
it('unlinks an orphaned socket only once nothing holds it, unpinning the dir for GC', async () => {
execCommand
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
.mockResolvedValueOnce(probe(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof']))
.mockResolvedValueOnce('')
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
expect(findings[0].outcome).toBe('stale-endpoint-removed')
expect(issuedCommands()[2]).toBe(`rm -f '${OLD_SOCK}'`)
})
it('touches nothing on a host it cannot interrogate', async () => {
execCommand
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
.mockResolvedValueOnce(probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable']))
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
expect(findings[0].outcome).toBe('unverifiable')
expect(issuedCommands()).toHaveLength(2)
})
it('is a no-op when the listing fails, and never guesses at what was there', async () => {
execCommand.mockRejectedValueOnce(new Error('exec failed'))
await expect(sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)).resolves.toEqual([])
})
it('does not run against Windows hosts, whose endpoints are named pipes', async () => {
await expect(sweepSupersededRelayEndpoints(CONN, WINDOWS_HOST, SWEEP)).resolves.toEqual([])
expect(execCommand).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,178 @@
/**
* Relays this target left behind at a *different* version directory.
*
* Every relay build installs to `~/.orca-remote/relay-<fullVersion>/` and binds its socket
* inside it, so the socket path moves on every app update even though the filename component
* is stable. After an update the new client binds a path the previous relay's PTYs were never
* associated with, and the previous relay is never contacted again (#13614, #13852). Nothing
* signals it and nothing reclaims it: with `--grace-time 0` it keeps its shells and agents
* alive forever.
*
* This sweep makes that population *visible and deliberate* rather than silent. It does not
* make it recoverable — the daemon handshake compares the build's content hash exactly
* (`relay-handshake.ts`), so a new client cannot speak to an old daemon at all. See the report
* on this change for what a real cross-version handoff would require.
*
* The one thing it will terminate is a relay that provably holds nothing. Everything else is
* retained, including everything it merely failed to reach.
*/
import type { SshConnection } from './ssh-connection'
import { shellEscape } from './ssh-connection-utils'
import { RELAY_REMOTE_DIR } from './relay-protocol'
import { execCommand } from './ssh-relay-deploy-helpers'
import {
describeRelayEndpointIncumbent,
isReapableRelayHusk,
probeRelayEndpointIncumbent,
type RelayEndpointIncumbent
} from './ssh-relay-endpoint-incumbent'
import { reapEmptyRelayHusk } from './ssh-relay-endpoint-takeover'
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
/**
* `reaped` is the only outcome that claims a process ended, and it is only reachable from a
* post-signal `kill -0` that failed. A signal we sent but could not confirm is
* `reap-unconfirmed`, which is `unverifiable` — not `exited` by another name.
*/
export type SupersededRelayOutcome =
| 'reaped'
| 'reap-unconfirmed'
| 'retained-live-work'
| 'stale-endpoint-removed'
| 'unverifiable'
export type SupersededRelayFinding = {
sockPath: string
outcome: SupersededRelayOutcome
incumbent: RelayEndpointIncumbent
}
export type SupersededRelaySweepOptions = {
remoteHome: string
/** Absolute path of the version directory this client just launched into; never swept. */
currentRelayDir: string
/** Stable per-target socket filename, from `relaySocketNameForInstanceId`. */
sockName: string
nodePath: string
signal?: AbortSignal
}
const MAX_SWEPT_ENDPOINTS = 32
export function supersededRelayEndpointListCommand(options: {
remoteHome: string
currentRelayDir: string
sockName: string
}): string {
return [
`base=${shellEscape(`${options.remoteHome}/${RELAY_REMOTE_DIR}`)}`,
`sock_name=${shellEscape(options.sockName)}`,
`current=${shellEscape(options.currentRelayDir)}`,
'for sock in "$base"/relay-*/"$sock_name"; do',
' [ -S "$sock" ] || continue',
' dir=${sock%/*}',
' [ "$dir" = "$current" ] && continue',
' printf \'%s\\n\' "$sock"',
'done'
].join('\n')
}
/** Remove a socket inode proven to have no holder, so version-dir GC can reclaim the tree. */
export function removeStaleRelayEndpointCommand(sockPath: string): string {
return `rm -f ${shellEscape(sockPath)}`
}
export function classifySupersededRelay(
incumbent: RelayEndpointIncumbent
): Exclude<SupersededRelayOutcome, 'reaped' | 'reap-unconfirmed'> | 'reap-candidate' {
if (incumbent.verdict === 'exited') {
return incumbent.socketPresent ? 'stale-endpoint-removed' : 'unverifiable'
}
if (incumbent.verdict !== 'live') {
return 'unverifiable'
}
return isReapableRelayHusk(incumbent) ? 'reap-candidate' : 'retained-live-work'
}
export async function sweepSupersededRelayEndpoints(
conn: SshConnection,
hostPlatform: RemoteHostPlatform,
options: SupersededRelaySweepOptions
): Promise<SupersededRelayFinding[]> {
if (isWindowsRemoteHost(hostPlatform)) {
return []
}
let listing: string
try {
listing = await execCommand(conn, supersededRelayEndpointListCommand(options), {
wrapCommand: true,
signal: options.signal
})
} catch {
return []
}
const sockPaths = listing
.split('\n')
.map((line) => line.trim())
.filter((line) => line.startsWith('/'))
.slice(0, MAX_SWEPT_ENDPOINTS)
const findings: SupersededRelayFinding[] = []
for (const sockPath of sockPaths) {
options.signal?.throwIfAborted()
const incumbent = await probeRelayEndpointIncumbent(
conn,
hostPlatform,
options.nodePath,
sockPath,
{ signal: options.signal }
)
findings.push({
sockPath,
outcome: await applySupersededRelayDecision(conn, incumbent, options),
incumbent
})
}
logSupersededRelayFindings(findings)
return findings
}
async function applySupersededRelayDecision(
conn: SshConnection,
incumbent: RelayEndpointIncumbent,
options: SupersededRelaySweepOptions
): Promise<SupersededRelayOutcome> {
const decision = classifySupersededRelay(incumbent)
if (decision === 'stale-endpoint-removed') {
try {
await execCommand(conn, removeStaleRelayEndpointCommand(incumbent.sockPath), {
wrapCommand: true,
signal: options.signal
})
return 'stale-endpoint-removed'
} catch {
return 'unverifiable'
}
}
if (decision !== 'reap-candidate') {
return decision
}
return reapEmptyRelayHusk(conn, incumbent, { signal: options.signal })
}
function logSupersededRelayFindings(findings: SupersededRelayFinding[]): void {
for (const finding of findings) {
const detail = describeRelayEndpointIncumbent(finding.incumbent)
if (finding.outcome === 'retained-live-work') {
console.warn(
`[ssh-relay] Superseded relay retained (holds live work; not signalled): ${detail}`
)
continue
}
if (finding.outcome === 'unverifiable' || finding.outcome === 'reap-unconfirmed') {
console.warn(`[ssh-relay] Superseded relay ${finding.outcome}: ${detail}`)
continue
}
console.log(`[ssh-relay] Superseded relay ${finding.outcome}: ${detail}`)
}
}