mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 08:03:12 +00:00
Merge remote-tracking branch 'origin/fix/ssh-relay-endpoint-takeover' into adhoc/ssh-sweep-combined
This commit is contained in:
@@ -186,9 +186,9 @@ describe('deployAndLaunchRelay', () => {
|
||||
expect(progress).toContain('Starting relay...')
|
||||
})
|
||||
|
||||
it('does not launch fresh after unconfirmed stale-socket cleanup', async () => {
|
||||
it('does not launch fresh after an unconfirmed endpoint-incumbent probe', async () => {
|
||||
const conn = makeMockConnection()
|
||||
const unconfirmedCleanup = Object.assign(new Error('socket cleanup still running'), {
|
||||
const unconfirmedCleanup = Object.assign(new Error('endpoint probe still running'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
vi.mocked(waitForSentinel).mockRejectedValueOnce(new Error('stale relay reconnect failed'))
|
||||
|
||||
@@ -77,6 +77,8 @@ import {
|
||||
import { detectRemoteHostPlatform } from './ssh-remote-platform-detection'
|
||||
import { powerShellCommand, powerShellLiteral, powerShellNativeArg } from './ssh-remote-powershell'
|
||||
import { relaySocketNameForInstanceId } from './ssh-relay-instance-id'
|
||||
import { resolveRelayEndpointBeforeRelaunch } from './ssh-relay-endpoint-takeover'
|
||||
import { sweepSupersededRelayEndpoints } from './ssh-relay-superseded-endpoints'
|
||||
import { isSshSessionLimitError } from './ssh-session-limit-error'
|
||||
import {
|
||||
isWindowsRelayPipePath,
|
||||
@@ -580,6 +582,17 @@ async function deployAndLaunchRelayAttempt(
|
||||
hostPlatform,
|
||||
recoverOneStaleRelayUploadStageCommand(hostPlatform, uploadStagePoolDir)
|
||||
)
|
||||
.catch(() => {})
|
||||
// Why before GC: a superseded relay pins its version dir via the live-socket probe, so the
|
||||
// sweep has to settle first or GC keeps every orphan's tree forever.
|
||||
.then(() =>
|
||||
sweepSupersededRelayEndpoints(conn, hostPlatform, {
|
||||
remoteHome,
|
||||
currentRelayDir: remoteRelayDir,
|
||||
sockName: relaySocketNameForInstanceId(relayInstanceId),
|
||||
nodePath: launched.nodePath
|
||||
})
|
||||
)
|
||||
.catch(() => {})
|
||||
.then(() =>
|
||||
gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, {
|
||||
@@ -1457,17 +1470,15 @@ async function launchRelay(
|
||||
} catch (err) {
|
||||
signal?.throwIfAborted()
|
||||
console.warn(
|
||||
'[ssh-relay] Socket reconnect failed, launching fresh relay:',
|
||||
'[ssh-relay] Socket reconnect failed, establishing what owns the endpoint:',
|
||||
err instanceof Error ? err.message : String(err)
|
||||
)
|
||||
// Why: stale socket from a crashed relay — remove it so the fresh launch can bind at the same path.
|
||||
await execCommand(conn, `rm -f ${shellEscape(sockFile)}`, { signal }).catch(
|
||||
(cleanupErr) => {
|
||||
if (isUnconfirmedSshCommandTermination(cleanupErr)) {
|
||||
throw cleanupErr
|
||||
}
|
||||
}
|
||||
)
|
||||
// Why not `rm -f`: unlinking does not close the listener the incumbent already holds,
|
||||
// so a refused --connect (version mismatch, rotated credential) used to leave a live
|
||||
// relay running forever with its PTYs while a replacement bound the same path (#8585).
|
||||
await resolveRelayEndpointBeforeRelaunch(conn, hostPlatform, nodePath, sockFile, err, {
|
||||
signal
|
||||
})
|
||||
signal?.throwIfAborted()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* The probe and reap scripts run on someone else's machine and decide whether a process is
|
||||
* signalled, so the shell itself is the part worth testing for real. These cases run the
|
||||
* generated scripts through /bin/sh against real unix sockets and real processes.
|
||||
*/
|
||||
import { execFile, spawn, type ChildProcess } from 'node:child_process'
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
isReapableRelayHusk,
|
||||
parseRelayEndpointIncumbentProbe,
|
||||
relayEndpointIncumbentProbeCommand,
|
||||
type RelayEndpointIncumbent
|
||||
} from './ssh-relay-endpoint-incumbent'
|
||||
import { reapEmptyRelayHuskCommand } from './ssh-relay-endpoint-takeover'
|
||||
|
||||
const posixOnly = process.platform === 'win32' ? describe.skip : describe
|
||||
|
||||
const FAKE_RELAY_SOURCE = `
|
||||
const net = require('net')
|
||||
const sock = process.argv[process.argv.indexOf('--sock-path') + 1]
|
||||
if (process.argv.includes('--with-child')) {
|
||||
require('child_process').spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
}
|
||||
net.createServer(() => {}).listen(sock, () => process.stdout.write('READY\\n'))
|
||||
process.on('SIGTERM', () => process.exit(0))
|
||||
`
|
||||
|
||||
function sh(script: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile('/bin/sh', ['-c', script], { timeout: 20_000 }, (error, stdout) => {
|
||||
if (error) {
|
||||
reject(error)
|
||||
return
|
||||
}
|
||||
resolve(stdout)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
let workDir: string
|
||||
let hasLsof = false
|
||||
const running: ChildProcess[] = []
|
||||
|
||||
function startFakeRelay(sockPath: string, withChild = false): Promise<ChildProcess> {
|
||||
const args = [join(workDir, 'relay.js'), '--sock-path', sockPath]
|
||||
if (withChild) {
|
||||
args.push('--with-child')
|
||||
}
|
||||
const child = spawn(process.execPath, args, { stdio: ['ignore', 'pipe', 'ignore'] })
|
||||
running.push(child)
|
||||
return new Promise((resolve, reject) => {
|
||||
child.stdout.on('data', (chunk: Buffer) => {
|
||||
if (chunk.toString().includes('READY')) {
|
||||
resolve(child)
|
||||
}
|
||||
})
|
||||
child.on('exit', () => reject(new Error('fake relay exited before listening')))
|
||||
})
|
||||
}
|
||||
|
||||
async function probe(sockPath: string): Promise<RelayEndpointIncumbent> {
|
||||
const output = await sh(relayEndpointIncumbentProbeCommand(process.execPath, sockPath))
|
||||
return parseRelayEndpointIncumbentProbe(sockPath, output)
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
workDir = mkdtempSync(join(tmpdir(), 'orca-relay-incumbent-'))
|
||||
writeFileSync(join(workDir, 'relay.js'), FAKE_RELAY_SOURCE)
|
||||
hasLsof = await sh('command -v lsof >/dev/null 2>&1 && echo yes || echo no').then(
|
||||
(out) => out.trim() === 'yes'
|
||||
)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
while (running.length > 0) {
|
||||
running.pop()?.kill('SIGKILL')
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(workDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('runs the holder-enumeration assertions on this machine', () => {
|
||||
// Why asserted rather than assumed: the cases below degrade to verdict-only checks without
|
||||
// lsof, and a silently degraded suite would stop covering the reap gate entirely.
|
||||
expect(hasLsof).toBe(true)
|
||||
})
|
||||
|
||||
posixOnly('relay endpoint probe against a real socket', () => {
|
||||
it('reports live, and identifies the holding process, for a listening relay', async () => {
|
||||
const sockPath = join(workDir, 'live.sock')
|
||||
const relay = await startFakeRelay(sockPath)
|
||||
const incumbent = await probe(sockPath)
|
||||
|
||||
expect(incumbent.verdict).toBe('live')
|
||||
expect(incumbent.evidence).toBe('accepted-connection')
|
||||
expect(incumbent.socketPresent).toBe(true)
|
||||
if (!hasLsof) {
|
||||
return
|
||||
}
|
||||
expect(incumbent.holders.map((holder) => holder.pid)).toEqual([relay.pid])
|
||||
expect(incumbent.holders[0]).toMatchObject({ matchesRelayArgv: true, childCount: 0 })
|
||||
expect(isReapableRelayHusk(incumbent)).toBe(true)
|
||||
})
|
||||
|
||||
it('refuses to call a relay with a live child an empty husk', async () => {
|
||||
const sockPath = join(workDir, 'busy.sock')
|
||||
await startFakeRelay(sockPath, true)
|
||||
const incumbent = await probe(sockPath)
|
||||
|
||||
expect(incumbent.verdict).toBe('live')
|
||||
if (!hasLsof) {
|
||||
return
|
||||
}
|
||||
expect(incumbent.holders[0].childCount).toBeGreaterThan(0)
|
||||
expect(isReapableRelayHusk(incumbent)).toBe(false)
|
||||
})
|
||||
|
||||
it('reports exited for a socket inode a SIGKILLed relay left behind', async () => {
|
||||
const sockPath = join(workDir, 'stale.sock')
|
||||
const relay = await startFakeRelay(sockPath)
|
||||
relay.kill('SIGKILL')
|
||||
await new Promise((resolve) => relay.on('exit', resolve))
|
||||
|
||||
const incumbent = await probe(sockPath)
|
||||
expect(incumbent.socketPresent).toBe(true)
|
||||
expect(incumbent.verdict).toBe(hasLsof ? 'exited' : 'unverifiable')
|
||||
})
|
||||
|
||||
it('reports no listener for a path that was never bound', async () => {
|
||||
const incumbent = await probe(join(workDir, 'never-existed.sock'))
|
||||
expect(incumbent.socketPresent).toBe(false)
|
||||
expect(incumbent.verdict).toBe(hasLsof ? 'exited' : 'unverifiable')
|
||||
})
|
||||
})
|
||||
|
||||
posixOnly('empty relay husk reap against a real process', () => {
|
||||
it('terminates a proven-empty relay and confirms the pid is gone', async () => {
|
||||
const sockPath = join(workDir, 'husk.sock')
|
||||
const relay = await startFakeRelay(sockPath)
|
||||
const output = await sh(reapEmptyRelayHuskCommand(relay.pid!, sockPath))
|
||||
expect(output.trim()).toBe('GONE')
|
||||
})
|
||||
|
||||
it('refuses to signal a relay that acquired a child after it was probed', async () => {
|
||||
const sockPath = join(workDir, 'raced.sock')
|
||||
const relay = await startFakeRelay(sockPath, true)
|
||||
const output = await sh(reapEmptyRelayHuskCommand(relay.pid!, sockPath))
|
||||
expect(output.trim()).toBe('BUSY')
|
||||
expect(relay.killed).toBe(false)
|
||||
})
|
||||
|
||||
it('refuses to signal a pid whose argv is not this relay at this socket', async () => {
|
||||
const sockPath = join(workDir, 'mismatch.sock')
|
||||
await startFakeRelay(sockPath)
|
||||
const bystander = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
running.push(bystander)
|
||||
const output = await sh(reapEmptyRelayHuskCommand(bystander.pid!, sockPath))
|
||||
expect(output.trim()).toBe('MISMATCH')
|
||||
expect(bystander.killed).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,240 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const execCommand = vi.fn()
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({
|
||||
execCommand: (...args: unknown[]) => execCommand(...args),
|
||||
isUnconfirmedSshCommandTermination: (error: unknown) =>
|
||||
(error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false
|
||||
}))
|
||||
|
||||
import {
|
||||
describeRelayEndpointIncumbent,
|
||||
isReapableRelayHusk,
|
||||
mayLaunchOverRelayEndpoint,
|
||||
parseRelayEndpointIncumbentProbe,
|
||||
probeRelayEndpointIncumbent,
|
||||
relayEndpointIncumbentProbeCommand,
|
||||
withHandshakeRefusalEvidence,
|
||||
type RelayEndpointIncumbent
|
||||
} from './ssh-relay-endpoint-incumbent'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const SOCK = '/home/u/.orca-remote/relay-0.1.0+aaaa/relay-deadbeef.sock'
|
||||
const POSIX_HOST = getRemoteHostPlatform('linux-x64')
|
||||
const WINDOWS_HOST = getRemoteHostPlatform('win32-x64')
|
||||
|
||||
function probeOutput(lines: string[]): string {
|
||||
return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n')
|
||||
}
|
||||
|
||||
describe('parseRelayEndpointIncumbentProbe', () => {
|
||||
it('reports live when the socket accepted a connection', () => {
|
||||
const incumbent = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=4242 yes 13'])
|
||||
)
|
||||
expect(incumbent.verdict).toBe('live')
|
||||
expect(incumbent.evidence).toBe('accepted-connection')
|
||||
expect(incumbent.holders).toEqual([{ pid: 4242, matchesRelayArgv: true, childCount: 13 }])
|
||||
})
|
||||
|
||||
it('reports live when a process still holds an inode that refuses connections', () => {
|
||||
const incumbent = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof', 'HOLDER=91 yes 2'])
|
||||
)
|
||||
expect(incumbent.verdict).toBe('live')
|
||||
expect(incumbent.evidence).toBe('holder-process')
|
||||
})
|
||||
|
||||
it('reports exited only when the connect was refused AND nothing holds the socket', () => {
|
||||
const incumbent = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])
|
||||
)
|
||||
expect(incumbent.verdict).toBe('exited')
|
||||
expect(incumbent.evidence).toBe('no-holder')
|
||||
expect(incumbent.socketPresent).toBe(true)
|
||||
})
|
||||
|
||||
it('reports unverifiable when the host cannot enumerate socket holders', () => {
|
||||
const incumbent = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=unavailable'])
|
||||
)
|
||||
expect(incumbent.verdict).toBe('unverifiable')
|
||||
expect(incumbent.holdersEnumerable).toBe(false)
|
||||
})
|
||||
|
||||
it('reports unverifiable when the connect probe timed out', () => {
|
||||
const incumbent = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=lsof'])
|
||||
)
|
||||
expect(incumbent.verdict).toBe('unverifiable')
|
||||
})
|
||||
|
||||
it('reports unverifiable for truncated or garbled probe output', () => {
|
||||
expect(parseRelayEndpointIncumbentProbe(SOCK, 'PRESENT=yes\nLISTEN=refused').verdict).toBe(
|
||||
'unverifiable'
|
||||
)
|
||||
expect(parseRelayEndpointIncumbentProbe(SOCK, '').verdict).toBe('unverifiable')
|
||||
})
|
||||
|
||||
it('drops holder lines that do not carry a usable pid', () => {
|
||||
const incumbent = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof', 'HOLDER=- no unknown'])
|
||||
)
|
||||
expect(incumbent.holders).toEqual([])
|
||||
expect(incumbent.verdict).toBe('exited')
|
||||
})
|
||||
|
||||
it('keeps an unreadable child count as null rather than zero', () => {
|
||||
const [holder] = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=7 yes unknown'])
|
||||
).holders
|
||||
expect(holder.childCount).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('probeRelayEndpointIncumbent', () => {
|
||||
it('never asserts death when the probe itself could not run', async () => {
|
||||
execCommand.mockRejectedValueOnce(new Error('channel closed'))
|
||||
const incumbent = await probeRelayEndpointIncumbent(
|
||||
{} as SshConnection,
|
||||
POSIX_HOST,
|
||||
'/usr/bin/node',
|
||||
SOCK
|
||||
)
|
||||
expect(incumbent.verdict).toBe('unverifiable')
|
||||
expect(incumbent.holders).toEqual([])
|
||||
})
|
||||
|
||||
it('does not shell out on Windows hosts, where the endpoint is a named pipe', async () => {
|
||||
execCommand.mockClear()
|
||||
const incumbent = await probeRelayEndpointIncumbent(
|
||||
{} as SshConnection,
|
||||
WINDOWS_HOST,
|
||||
'node.exe',
|
||||
SOCK
|
||||
)
|
||||
expect(execCommand).not.toHaveBeenCalled()
|
||||
expect(incumbent.verdict).toBe('unverifiable')
|
||||
})
|
||||
})
|
||||
|
||||
describe('relayEndpointIncumbentProbeCommand', () => {
|
||||
it('ANDs the lsof selectors so it cannot match unrelated unix-socket holders', () => {
|
||||
expect(relayEndpointIncumbentProbeCommand('/usr/bin/node', SOCK)).toContain(
|
||||
'lsof -t -a -U "$sock"'
|
||||
)
|
||||
})
|
||||
|
||||
it('never mutates the host: no unlink, no signal', () => {
|
||||
const command = relayEndpointIncumbentProbeCommand('/usr/bin/node', SOCK)
|
||||
expect(command).not.toMatch(/\brm\b/)
|
||||
expect(command).not.toMatch(/\bkill\b/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('withHandshakeRefusalEvidence', () => {
|
||||
it('upgrades an unenumerable endpoint to live when the daemon answered the handshake', () => {
|
||||
const probed = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
|
||||
)
|
||||
const incumbent = withHandshakeRefusalEvidence(probed)
|
||||
expect(incumbent.verdict).toBe('live')
|
||||
expect(incumbent.evidence).toBe('handshake-refusal')
|
||||
expect(mayLaunchOverRelayEndpoint(incumbent)).toBe(false)
|
||||
})
|
||||
|
||||
it('leaves stronger evidence in place', () => {
|
||||
const probed = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof'])
|
||||
)
|
||||
expect(withHandshakeRefusalEvidence(probed).evidence).toBe('accepted-connection')
|
||||
})
|
||||
})
|
||||
|
||||
describe('mayLaunchOverRelayEndpoint', () => {
|
||||
const verdicts: RelayEndpointIncumbent['verdict'][] = ['live', 'unverifiable', 'exited']
|
||||
it.each(verdicts)('permits a relaunch for %s only when it is not live', (verdict) => {
|
||||
const incumbent = { ...parseRelayEndpointIncumbentProbe(SOCK, ''), verdict }
|
||||
expect(mayLaunchOverRelayEndpoint(incumbent)).toBe(verdict !== 'live')
|
||||
})
|
||||
})
|
||||
|
||||
describe('isReapableRelayHusk', () => {
|
||||
const husk = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=500 yes 0'])
|
||||
)
|
||||
|
||||
it('accepts a single proven relay holder with zero children', () => {
|
||||
expect(isReapableRelayHusk(husk)).toBe(true)
|
||||
})
|
||||
|
||||
it('refuses a relay that still holds children', () => {
|
||||
expect(
|
||||
isReapableRelayHusk({
|
||||
...husk,
|
||||
holders: [{ pid: 500, matchesRelayArgv: true, childCount: 1 }]
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('refuses a holder whose child count could not be read', () => {
|
||||
expect(
|
||||
isReapableRelayHusk({
|
||||
...husk,
|
||||
holders: [{ pid: 500, matchesRelayArgv: true, childCount: null }]
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('refuses a holder whose argv is not this relay at this socket', () => {
|
||||
expect(
|
||||
isReapableRelayHusk({
|
||||
...husk,
|
||||
holders: [{ pid: 500, matchesRelayArgv: false, childCount: 0 }]
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('refuses when more than one process holds the socket', () => {
|
||||
expect(
|
||||
isReapableRelayHusk({
|
||||
...husk,
|
||||
holders: [
|
||||
{ pid: 500, matchesRelayArgv: true, childCount: 0 },
|
||||
{ pid: 501, matchesRelayArgv: true, childCount: 0 }
|
||||
]
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('refuses an unverifiable endpoint however empty it looks', () => {
|
||||
expect(isReapableRelayHusk({ ...husk, verdict: 'unverifiable' })).toBe(false)
|
||||
expect(isReapableRelayHusk({ ...husk, holdersEnumerable: false })).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('describeRelayEndpointIncumbent', () => {
|
||||
it('distinguishes "no holders" from "could not enumerate holders"', () => {
|
||||
const none = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])
|
||||
)
|
||||
const unknown = parseRelayEndpointIncumbentProbe(
|
||||
SOCK,
|
||||
probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=unavailable'])
|
||||
)
|
||||
expect(describeRelayEndpointIncumbent(none)).toContain('holders=none')
|
||||
expect(describeRelayEndpointIncumbent(unknown)).toContain('holders=unenumerable')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* Who currently owns a relay socket path, answered with host evidence.
|
||||
*
|
||||
* The client used to answer this by assumption: a failed `--connect` was read as "the relay
|
||||
* crashed", the socket was `rm -f`'d, and a fresh relay bound the same path. Unlinking a unix
|
||||
* socket does not close the listener the incumbent already holds, so an alive-but-refusing
|
||||
* relay (the `RelayVersionMismatchError` case, and the credential-rotation case) was left
|
||||
* running forever with its PTYs (#8585).
|
||||
*
|
||||
* The verdict vocabulary is fixed by docs/reference/ssh-execution-boundary.md — `live` /
|
||||
* `unverifiable` / `exited`, with no synonyms and no collapsing. Two consequences are load
|
||||
* bearing here:
|
||||
*
|
||||
* - `exited` is a claim about **this endpoint**, not about every relay on the host. It means
|
||||
* nothing holds this socket path, established positively (a connect that was refused *and*
|
||||
* an enumeration that found no holder). A relay whose socket was already unlinked is
|
||||
* invisible to this probe by construction — that is what the superseded sweep is for.
|
||||
* - a probe that could not run, a host without `lsof`, or a connect that failed for any other
|
||||
* reason is `unverifiable`. It never authorizes unlinking, rebinding over, or signalling.
|
||||
*/
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers'
|
||||
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
export type RelayEndpointVerdict = 'live' | 'unverifiable' | 'exited'
|
||||
|
||||
export type RelayEndpointEvidence =
|
||||
| 'accepted-connection'
|
||||
| 'handshake-refusal'
|
||||
| 'holder-process'
|
||||
| 'no-holder'
|
||||
| 'inconclusive'
|
||||
|
||||
export type RelayEndpointHolder = {
|
||||
pid: number
|
||||
/** The holder's argv names relay.js AND this exact socket path. */
|
||||
matchesRelayArgv: boolean
|
||||
/** Direct children, or null when `pgrep` could not answer. Never guessed. */
|
||||
childCount: number | null
|
||||
}
|
||||
|
||||
export type RelayEndpointIncumbent = {
|
||||
sockPath: string
|
||||
verdict: RelayEndpointVerdict
|
||||
evidence: RelayEndpointEvidence
|
||||
socketPresent: boolean
|
||||
/** Pids proven to hold this exact socket. Empty when the host could not enumerate them. */
|
||||
holders: RelayEndpointHolder[]
|
||||
/** False when no enumeration tool was available — an empty `holders` then proves nothing. */
|
||||
holdersEnumerable: boolean
|
||||
}
|
||||
|
||||
const PROBE_BEGIN = 'ORCA-INCUMBENT-BEGIN'
|
||||
const PROBE_END = 'ORCA-INCUMBENT-END'
|
||||
const CONNECT_PROBE_TIMEOUT_MS = 1000
|
||||
|
||||
// Why ES5 syntax: nodePath may be a host-resolved system node, not the bundled one.
|
||||
const CONNECT_PROBE_JS = [
|
||||
'var s=require("net").connect(process.argv[1]);',
|
||||
'var done=false;',
|
||||
'function say(v){if(done)return;done=true;try{s.destroy()}catch(e){};',
|
||||
'process.stdout.write(v);process.exit(0)}',
|
||||
's.on("connect",function(){say("accepted")});',
|
||||
's.on("error",function(e){',
|
||||
'say(e.code==="ECONNREFUSED"?"refused":e.code==="ENOENT"?"absent":"unknown")});',
|
||||
`setTimeout(function(){say("unknown")},${CONNECT_PROBE_TIMEOUT_MS})`
|
||||
].join('')
|
||||
|
||||
/**
|
||||
* A POSIX probe that reports only what the host actually observed. Every field has an
|
||||
* explicit "could not tell" value; nothing is inferred from a missing tool.
|
||||
*/
|
||||
export function relayEndpointIncumbentProbeCommand(nodePath: string, sockPath: string): string {
|
||||
const sock = shellEscape(sockPath)
|
||||
const node = shellEscape(nodePath)
|
||||
return [
|
||||
`sock=${sock}`,
|
||||
`node=${node}`,
|
||||
`printf '%s\\n' ${shellEscape(PROBE_BEGIN)}`,
|
||||
'if [ -S "$sock" ]; then',
|
||||
" printf 'PRESENT=yes\\n'",
|
||||
` listen=$("$node" -e ${shellEscape(CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`,
|
||||
' [ -n "$listen" ] || listen=unknown',
|
||||
'else',
|
||||
" printf 'PRESENT=no\\n'",
|
||||
' listen=absent',
|
||||
'fi',
|
||||
'printf \'LISTEN=%s\\n\' "$listen"',
|
||||
'if command -v lsof >/dev/null 2>&1; then',
|
||||
" printf 'HOLDERS_SOURCE=lsof\\n'",
|
||||
// Why -a: lsof ORs its selectors, so without it every unix-socket holder on the box
|
||||
// would be reported as holding this path (#8762).
|
||||
' for pid in $(lsof -t -a -U "$sock" 2>/dev/null); do',
|
||||
' args=$(ps -o args= -p "$pid" 2>/dev/null | tr "\\n" " ")',
|
||||
' match=no',
|
||||
' case "$args" in *relay.js*"$sock"*) match=yes ;; esac',
|
||||
' kids=unknown',
|
||||
' if command -v pgrep >/dev/null 2>&1; then',
|
||||
' kids=$(pgrep -P "$pid" 2>/dev/null | grep -c .)',
|
||||
' fi',
|
||||
' printf \'HOLDER=%s %s %s\\n\' "$pid" "$match" "$kids"',
|
||||
' done',
|
||||
'else',
|
||||
" printf 'HOLDERS_SOURCE=unavailable\\n'",
|
||||
'fi',
|
||||
`printf '%s\\n' ${shellEscape(PROBE_END)}`
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
export function parseRelayEndpointIncumbentProbe(
|
||||
sockPath: string,
|
||||
output: string
|
||||
): RelayEndpointIncumbent {
|
||||
const lines = output.split('\n').map((line) => line.trim())
|
||||
if (!lines.includes(PROBE_BEGIN) || !lines.includes(PROBE_END)) {
|
||||
return unverifiableEndpoint(sockPath)
|
||||
}
|
||||
const socketPresent = lines.includes('PRESENT=yes')
|
||||
const listen = lines.find((line) => line.startsWith('LISTEN='))?.slice('LISTEN='.length) ?? ''
|
||||
const holdersEnumerable = lines.includes('HOLDERS_SOURCE=lsof')
|
||||
const holders = lines
|
||||
.filter((line) => line.startsWith('HOLDER='))
|
||||
.map((line) => parseHolder(line.slice('HOLDER='.length)))
|
||||
.filter((holder): holder is RelayEndpointHolder => holder !== null)
|
||||
|
||||
if (listen === 'accepted') {
|
||||
return {
|
||||
sockPath,
|
||||
verdict: 'live',
|
||||
evidence: 'accepted-connection',
|
||||
socketPresent,
|
||||
holders,
|
||||
holdersEnumerable
|
||||
}
|
||||
}
|
||||
if (holders.length > 0) {
|
||||
// The inode is held by a running process that is not accepting — wedged, not gone.
|
||||
return {
|
||||
sockPath,
|
||||
verdict: 'live',
|
||||
evidence: 'holder-process',
|
||||
socketPresent,
|
||||
holders,
|
||||
holdersEnumerable
|
||||
}
|
||||
}
|
||||
if (holdersEnumerable && (listen === 'refused' || listen === 'absent')) {
|
||||
return {
|
||||
sockPath,
|
||||
verdict: 'exited',
|
||||
evidence: 'no-holder',
|
||||
socketPresent,
|
||||
holders,
|
||||
holdersEnumerable
|
||||
}
|
||||
}
|
||||
return { ...unverifiableEndpoint(sockPath), socketPresent, holders, holdersEnumerable }
|
||||
}
|
||||
|
||||
function parseHolder(value: string): RelayEndpointHolder | null {
|
||||
const [rawPid, rawMatch, rawKids] = value.split(/\s+/)
|
||||
const pid = Number.parseInt(rawPid ?? '', 10)
|
||||
if (!Number.isInteger(pid) || pid <= 0) {
|
||||
return null
|
||||
}
|
||||
const childCount = Number.parseInt(rawKids ?? '', 10)
|
||||
return {
|
||||
pid,
|
||||
matchesRelayArgv: rawMatch === 'yes',
|
||||
childCount: Number.isInteger(childCount) && childCount >= 0 ? childCount : null
|
||||
}
|
||||
}
|
||||
|
||||
function unverifiableEndpoint(sockPath: string): RelayEndpointIncumbent {
|
||||
return {
|
||||
sockPath,
|
||||
verdict: 'unverifiable',
|
||||
evidence: 'inconclusive',
|
||||
socketPresent: false,
|
||||
holders: [],
|
||||
holdersEnumerable: false
|
||||
}
|
||||
}
|
||||
|
||||
export async function probeRelayEndpointIncumbent(
|
||||
conn: SshConnection,
|
||||
hostPlatform: RemoteHostPlatform,
|
||||
nodePath: string,
|
||||
sockPath: string,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<RelayEndpointIncumbent> {
|
||||
// Windows relays are named pipes: there is no inode to unlink and no `lsof`, so the
|
||||
// orphan-by-unlink mechanism this probe defends against cannot occur there.
|
||||
if (isWindowsRemoteHost(hostPlatform)) {
|
||||
return unverifiableEndpoint(sockPath)
|
||||
}
|
||||
try {
|
||||
const output = await execCommand(conn, relayEndpointIncumbentProbeCommand(nodePath, sockPath), {
|
||||
wrapCommand: true,
|
||||
signal: options?.signal
|
||||
})
|
||||
return parseRelayEndpointIncumbentProbe(sockPath, output)
|
||||
} catch (err) {
|
||||
// An exec whose channel never confirmed close may still be running remotely; the caller
|
||||
// must not race a detached launch against it.
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
// Any other unanswered probe observes nothing. It is never evidence of death.
|
||||
return unverifiableEndpoint(sockPath)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A relay that told us its version over the wire is `live` by positive host evidence, even on
|
||||
* a host where nothing can enumerate socket holders.
|
||||
*/
|
||||
export function withHandshakeRefusalEvidence(
|
||||
incumbent: RelayEndpointIncumbent
|
||||
): RelayEndpointIncumbent {
|
||||
if (incumbent.verdict === 'live') {
|
||||
return incumbent
|
||||
}
|
||||
return { ...incumbent, verdict: 'live', evidence: 'handshake-refusal' }
|
||||
}
|
||||
|
||||
/**
|
||||
* May a fresh relay be launched onto this path?
|
||||
*
|
||||
* Only `live` forbids it. `unverifiable` is permitted because the *daemon* — not the client —
|
||||
* performs the takeover: `RelaySocketOwnership.listen` re-probes on EADDRINUSE, refuses to
|
||||
* steal a path that accepts connections, and only unlinks an inode whose identity is
|
||||
* unchanged. That check is atomic with the bind, which a client-side `rm -f` can never be.
|
||||
*/
|
||||
export function mayLaunchOverRelayEndpoint(incumbent: RelayEndpointIncumbent): boolean {
|
||||
return incumbent.verdict !== 'live'
|
||||
}
|
||||
|
||||
/**
|
||||
* A live relay that provably holds nothing: identity confirmed against its argv, exactly one
|
||||
* holder, and zero children. Reaping it destroys no user work. Anything less is retained —
|
||||
* killing the wrong pid on someone's remote host is the worst outcome available here.
|
||||
*/
|
||||
export function isReapableRelayHusk(incumbent: RelayEndpointIncumbent): boolean {
|
||||
if (incumbent.verdict !== 'live' || !incumbent.holdersEnumerable) {
|
||||
return false
|
||||
}
|
||||
if (incumbent.holders.length !== 1) {
|
||||
return false
|
||||
}
|
||||
const [holder] = incumbent.holders
|
||||
return holder.matchesRelayArgv && holder.childCount === 0
|
||||
}
|
||||
|
||||
export function describeRelayEndpointIncumbent(incumbent: RelayEndpointIncumbent): string {
|
||||
const holders = incumbent.holders
|
||||
.map((holder) => `${holder.pid}(children=${holder.childCount ?? 'unknown'})`)
|
||||
.join(',')
|
||||
return (
|
||||
`${incumbent.sockPath} verdict=${incumbent.verdict} evidence=${incumbent.evidence} ` +
|
||||
`holders=${incumbent.holdersEnumerable ? holders || 'none' : 'unenumerable'}`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Thrown instead of orphaning: a live relay owns the endpoint and refused us, so the path is
|
||||
* not ours to rebind. Terminal for this attempt — the user resolves it with Reset Relay,
|
||||
* which signals the incumbent deliberately and with consent.
|
||||
*/
|
||||
export class RelayEndpointHeldError extends Error {
|
||||
readonly name = 'RelayEndpointHeldError'
|
||||
constructor(readonly incumbent: RelayEndpointIncumbent) {
|
||||
super(
|
||||
`A live relay still owns ${incumbent.sockPath} and refused this connection ` +
|
||||
`(${describeRelayEndpointIncumbent(incumbent)}). Orca will not replace it, because ` +
|
||||
'unlinking its socket would strand its terminals. Use Reset Relay for this host to ' +
|
||||
'stop it, then reconnect.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function isRelayEndpointHeldError(err: unknown): err is RelayEndpointHeldError {
|
||||
return err instanceof RelayEndpointHeldError
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const execCommand = vi.fn()
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({
|
||||
execCommand: (...args: unknown[]) => execCommand(...args),
|
||||
isUnconfirmedSshCommandTermination: (error: unknown) =>
|
||||
(error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false
|
||||
}))
|
||||
|
||||
import { isRelayEndpointHeldError } from './ssh-relay-endpoint-incumbent'
|
||||
import {
|
||||
interpretRelayHuskReapOutput,
|
||||
reapEmptyRelayHuskCommand,
|
||||
resolveRelayEndpointBeforeRelaunch
|
||||
} from './ssh-relay-endpoint-takeover'
|
||||
import { RelayVersionMismatchError } from './ssh-relay-version-mismatch-error'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const SOCK = '/home/u/.orca-remote/relay-0.1.0+aaaa/relay-deadbeef.sock'
|
||||
const HOST = getRemoteHostPlatform('linux-x64')
|
||||
const CONN = {} as SshConnection
|
||||
|
||||
function probe(lines: string[]): string {
|
||||
return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n')
|
||||
}
|
||||
|
||||
function issuedCommands(): string[] {
|
||||
return execCommand.mock.calls.map((call) => String(call[1]))
|
||||
}
|
||||
|
||||
function resolve(reconnectError: unknown = new Error('connect failed')): Promise<unknown> {
|
||||
return resolveRelayEndpointBeforeRelaunch(CONN, HOST, '/usr/bin/node', SOCK, reconnectError)
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
execCommand.mockReset()
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
describe('incumbent alive and refusing', () => {
|
||||
it('refuses to rebind a live relay holding PTYs, and signals nothing', async () => {
|
||||
execCommand.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13'])
|
||||
)
|
||||
await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError)
|
||||
// The whole point of #8585: the incumbent's socket must survive so it is not orphaned.
|
||||
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
|
||||
expect(issuedCommands().some((command) => /\bkill\b/.test(command))).toBe(false)
|
||||
})
|
||||
|
||||
it('names the incumbent pid and the Reset Relay escape hatch in the error', async () => {
|
||||
execCommand.mockResolvedValue(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13'])
|
||||
)
|
||||
await expect(resolve()).rejects.toThrow(/3669803\(children=13\)/)
|
||||
await expect(resolve()).rejects.toThrow(/Reset Relay/)
|
||||
})
|
||||
|
||||
it('treats a version mismatch as live even where holders cannot be enumerated', async () => {
|
||||
execCommand.mockResolvedValue(
|
||||
probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
|
||||
)
|
||||
const mismatch = new RelayVersionMismatchError('0.1.0+new', '0.1.0+old', '')
|
||||
await expect(resolve(mismatch)).rejects.toSatisfy(isRelayEndpointHeldError)
|
||||
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
|
||||
})
|
||||
|
||||
it('reaps a live relay only when it provably holds nothing, and confirms it is gone', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
|
||||
)
|
||||
.mockResolvedValueOnce('GONE\n')
|
||||
await expect(resolve()).resolves.toMatchObject({ verdict: 'live' })
|
||||
expect(issuedCommands()[1]).toContain('kill -TERM "$pid"')
|
||||
})
|
||||
|
||||
it('does not launch over an empty relay whose death could not be confirmed', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
|
||||
)
|
||||
.mockResolvedValueOnce('LIVE\n')
|
||||
await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError)
|
||||
})
|
||||
|
||||
it('does not launch over a relay the host refused to signal on its own re-check', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
|
||||
)
|
||||
.mockResolvedValueOnce('BUSY\n')
|
||||
await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('incumbent genuinely gone', () => {
|
||||
it('permits the relaunch without unlinking anything itself', async () => {
|
||||
execCommand.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])
|
||||
)
|
||||
await expect(resolve()).resolves.toMatchObject({ verdict: 'exited', evidence: 'no-holder' })
|
||||
// The daemon unlinks under an identity check that is atomic with its bind; the client
|
||||
// cannot be, which is what created the orphan in the first place.
|
||||
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('incumbent unverifiable', () => {
|
||||
it('permits the relaunch but never claims the incumbent exited', async () => {
|
||||
execCommand.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
|
||||
)
|
||||
await expect(resolve()).resolves.toMatchObject({ verdict: 'unverifiable' })
|
||||
expect(issuedCommands()).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('stays unverifiable when the probe command itself fails', async () => {
|
||||
execCommand.mockRejectedValueOnce(new Error('exec timeout'))
|
||||
await expect(resolve()).resolves.toMatchObject({ verdict: 'unverifiable' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('reapEmptyRelayHuskCommand', () => {
|
||||
it('re-verifies argv and emptiness on the host immediately before signalling', () => {
|
||||
const command = reapEmptyRelayHuskCommand(4242, SOCK)
|
||||
expect(command.indexOf('MISMATCH')).toBeLessThan(command.indexOf('kill -TERM'))
|
||||
expect(command.indexOf('BUSY')).toBeLessThan(command.indexOf('kill -TERM'))
|
||||
})
|
||||
|
||||
it('sends SIGTERM only, so the relay runs its own socket cleanup', () => {
|
||||
const command = reapEmptyRelayHuskCommand(4242, SOCK)
|
||||
expect(command).toContain('kill -TERM')
|
||||
expect(command).not.toContain('kill -KILL')
|
||||
expect(command).not.toContain('-9')
|
||||
})
|
||||
|
||||
it('aborts without signalling when the host cannot count children', () => {
|
||||
expect(reapEmptyRelayHuskCommand(4242, SOCK)).toContain(
|
||||
"command -v pgrep >/dev/null 2>&1 || { printf 'BUSY\\n'; exit 0; }"
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('interpretRelayHuskReapOutput', () => {
|
||||
it('claims reaped only for a post-signal liveness check that failed', () => {
|
||||
expect(interpretRelayHuskReapOutput('GONE\n')).toBe('reaped')
|
||||
expect(interpretRelayHuskReapOutput('LIVE\n')).toBe('reap-unconfirmed')
|
||||
expect(interpretRelayHuskReapOutput('')).toBe('reap-unconfirmed')
|
||||
expect(interpretRelayHuskReapOutput('unexpected noise')).toBe('reap-unconfirmed')
|
||||
})
|
||||
|
||||
it('reports a host-side refusal as retained rather than as a failed kill', () => {
|
||||
expect(interpretRelayHuskReapOutput('MISMATCH\n')).toBe('retained-live-work')
|
||||
expect(interpretRelayHuskReapOutput('BUSY\n')).toBe('retained-live-work')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Deciding whether a relay socket path is ours to take, and acting on the answer.
|
||||
*
|
||||
* The only destructive action available here is a SIGTERM to a relay that has been proven —
|
||||
* by argv, by socket-holder enumeration, and by a zero child count re-checked on the host
|
||||
* immediately before the signal — to hold nothing at all. Everything else is left running.
|
||||
* Per docs/reference/ssh-execution-boundary.md, a relay we merely failed to reach is
|
||||
* `unverifiable`, and `unverifiable` never authorizes a kill or a rebind.
|
||||
*/
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers'
|
||||
import {
|
||||
describeRelayEndpointIncumbent,
|
||||
isReapableRelayHusk,
|
||||
mayLaunchOverRelayEndpoint,
|
||||
probeRelayEndpointIncumbent,
|
||||
RelayEndpointHeldError,
|
||||
withHandshakeRefusalEvidence,
|
||||
type RelayEndpointIncumbent
|
||||
} from './ssh-relay-endpoint-incumbent'
|
||||
import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error'
|
||||
import type { RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
/** `reaped` is only reachable from a post-signal `kill -0` that failed. Nothing else claims it. */
|
||||
export type RelayHuskReapResult = 'reaped' | 'reap-unconfirmed' | 'retained-live-work'
|
||||
|
||||
const REAP_CONFIRM_ATTEMPTS = 15
|
||||
|
||||
/**
|
||||
* Signal one relay, re-verifying identity and emptiness inside the same command.
|
||||
*
|
||||
* The re-verification is not belt-and-braces: a client can attach and spawn a PTY between the
|
||||
* probe and the signal, and pids are reused. `MISMATCH`/`BUSY` abort without signalling.
|
||||
*/
|
||||
export function reapEmptyRelayHuskCommand(pid: number, sockPath: string): string {
|
||||
return [
|
||||
`pid=${shellEscape(String(pid))}`,
|
||||
`sock=${shellEscape(sockPath)}`,
|
||||
'args=$(ps -o args= -p "$pid" 2>/dev/null | tr "\\n" " ")',
|
||||
'case "$args" in *relay.js*"$sock"*) ;; *) printf \'MISMATCH\\n\'; exit 0 ;; esac',
|
||||
"command -v pgrep >/dev/null 2>&1 || { printf 'BUSY\\n'; exit 0; }",
|
||||
'kids=$(pgrep -P "$pid" 2>/dev/null | grep -c .)',
|
||||
'[ "$kids" = "0" ] || { printf \'BUSY\\n\'; exit 0; }',
|
||||
// SIGTERM only: the relay's own handler disposes and unlinks. SIGKILL would leave the
|
||||
// socket inode behind and skip that shutdown path for no gain on an empty daemon.
|
||||
'kill -TERM "$pid" 2>/dev/null || true',
|
||||
'i=0',
|
||||
`while [ $i -lt ${REAP_CONFIRM_ATTEMPTS} ]; do`,
|
||||
' kill -0 "$pid" 2>/dev/null || { printf \'GONE\\n\'; exit 0; }',
|
||||
' sleep 0.2',
|
||||
' i=$((i+1))',
|
||||
'done',
|
||||
"printf 'LIVE\\n'"
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
export function interpretRelayHuskReapOutput(output: string): RelayHuskReapResult {
|
||||
const state = output.trim().split('\n').pop()?.trim()
|
||||
if (state === 'GONE') {
|
||||
return 'reaped'
|
||||
}
|
||||
// The host refused on its own re-check: what is there is not the empty relay we probed, so
|
||||
// nothing was signalled and nothing is claimed about it.
|
||||
if (state === 'MISMATCH' || state === 'BUSY') {
|
||||
return 'retained-live-work'
|
||||
}
|
||||
return 'reap-unconfirmed'
|
||||
}
|
||||
|
||||
export async function reapEmptyRelayHusk(
|
||||
conn: SshConnection,
|
||||
incumbent: RelayEndpointIncumbent,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<RelayHuskReapResult> {
|
||||
const holder = incumbent.holders[0]
|
||||
if (!holder) {
|
||||
return 'retained-live-work'
|
||||
}
|
||||
try {
|
||||
const output = await execCommand(
|
||||
conn,
|
||||
reapEmptyRelayHuskCommand(holder.pid, incumbent.sockPath),
|
||||
{ wrapCommand: true, signal: options?.signal }
|
||||
)
|
||||
return interpretRelayHuskReapOutput(output)
|
||||
} catch (err) {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
throw err
|
||||
}
|
||||
return 'reap-unconfirmed'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when `--connect` to an existing socket failed and the caller is about to launch a
|
||||
* replacement at the same path. Resolves to nothing when the launch may proceed; throws
|
||||
* `RelayEndpointHeldError` when a live relay owns the path and holds work.
|
||||
*
|
||||
* `unverifiable` deliberately permits the launch: the daemon, not the client, performs the
|
||||
* takeover. `RelaySocketOwnership.listen` re-probes on EADDRINUSE, refuses a path that accepts
|
||||
* connections, and only unlinks an inode whose identity is unchanged — a check that is atomic
|
||||
* with the bind, which a client-side `rm -f` can never be.
|
||||
*/
|
||||
export async function resolveRelayEndpointBeforeRelaunch(
|
||||
conn: SshConnection,
|
||||
hostPlatform: RemoteHostPlatform,
|
||||
nodePath: string,
|
||||
sockPath: string,
|
||||
reconnectError: unknown,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<RelayEndpointIncumbent> {
|
||||
const probed = await probeRelayEndpointIncumbent(conn, hostPlatform, nodePath, sockPath, options)
|
||||
// A daemon that answered the handshake with its own version is live by positive host
|
||||
// evidence, even where nothing can enumerate socket holders.
|
||||
const incumbent = isRelayVersionMismatchError(reconnectError)
|
||||
? withHandshakeRefusalEvidence(probed)
|
||||
: probed
|
||||
console.warn(`[ssh-relay] Relay endpoint incumbent: ${describeRelayEndpointIncumbent(incumbent)}`)
|
||||
|
||||
if (mayLaunchOverRelayEndpoint(incumbent)) {
|
||||
return incumbent
|
||||
}
|
||||
if (!isReapableRelayHusk(incumbent)) {
|
||||
throw new RelayEndpointHeldError(incumbent)
|
||||
}
|
||||
const result = await reapEmptyRelayHusk(conn, incumbent, options)
|
||||
if (result !== 'reaped') {
|
||||
throw new RelayEndpointHeldError(incumbent)
|
||||
}
|
||||
console.log(`[ssh-relay] Reaped empty relay husk holding ${sockPath}`)
|
||||
return incumbent
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import type { BrowserWindow } from 'electron'
|
||||
import { deployAndLaunchRelay } from './ssh-relay-deploy'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error'
|
||||
import { isRelayEndpointHeldError } from './ssh-relay-endpoint-incumbent'
|
||||
import { replayPendingSshPtyKills } from './ssh-pending-pty-kill-replay'
|
||||
import { SshChannelMultiplexer } from './ssh-channel-multiplexer'
|
||||
import { SshPtyProvider } from '../providers/ssh-pty-provider'
|
||||
@@ -629,7 +630,13 @@ export class SshRelaySession {
|
||||
}
|
||||
// Why: terminal on first connect — a deployed binary against a still-running legacy daemon, or a
|
||||
// claim another connection holds. Notify the callback but still rethrow.
|
||||
if (isRelayVersionMismatchError(err) || isSshOwnerAdmissionBlockedError(err)) {
|
||||
// RelayEndpointHeldError is terminal for the same reason: a live incumbent owns the
|
||||
// socket path, and backoff cannot make it hand it over. The user resolves it.
|
||||
if (
|
||||
isRelayVersionMismatchError(err) ||
|
||||
isRelayEndpointHeldError(err) ||
|
||||
isSshOwnerAdmissionBlockedError(err)
|
||||
) {
|
||||
console.warn(
|
||||
`[ssh-relay-session] Terminal relay error on initial connect for ${this.targetId}: ${err.message}`
|
||||
)
|
||||
@@ -783,7 +790,13 @@ export class SshRelaySession {
|
||||
}
|
||||
// Why terminal: neither a version mismatch nor a blocked owner claim is reconcilable by backoff
|
||||
// retry, so fire the typed callback and drop out of 'reconnecting'.
|
||||
if (isRelayVersionMismatchError(err) || isSshOwnerAdmissionBlockedError(err)) {
|
||||
// RelayEndpointHeldError is terminal for the same reason: a live incumbent owns the
|
||||
// socket path, and backoff cannot make it hand it over. The user resolves it.
|
||||
if (
|
||||
isRelayVersionMismatchError(err) ||
|
||||
isRelayEndpointHeldError(err) ||
|
||||
isSshOwnerAdmissionBlockedError(err)
|
||||
) {
|
||||
console.warn(
|
||||
`[ssh-relay-session] Terminal relay error for ${this.targetId}: ${err.message}`
|
||||
)
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const execCommand = vi.fn()
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({
|
||||
execCommand: (...args: unknown[]) => execCommand(...args),
|
||||
isUnconfirmedSshCommandTermination: (error: unknown) =>
|
||||
(error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false
|
||||
}))
|
||||
|
||||
import { parseRelayEndpointIncumbentProbe } from './ssh-relay-endpoint-incumbent'
|
||||
import {
|
||||
classifySupersededRelay,
|
||||
supersededRelayEndpointListCommand,
|
||||
sweepSupersededRelayEndpoints
|
||||
} from './ssh-relay-superseded-endpoints'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const HOME = '/home/u'
|
||||
const SOCK_NAME = 'relay-deadbeef.sock'
|
||||
const CURRENT_DIR = `${HOME}/.orca-remote/relay-0.1.0+bd3ec370d21d`
|
||||
const OLD_SOCK = `${HOME}/.orca-remote/relay-0.1.0+7175e0a40ea7/${SOCK_NAME}`
|
||||
const HOST = getRemoteHostPlatform('linux-x64')
|
||||
const WINDOWS_HOST = getRemoteHostPlatform('win32-x64')
|
||||
const CONN = {} as SshConnection
|
||||
|
||||
const SWEEP = {
|
||||
remoteHome: HOME,
|
||||
currentRelayDir: CURRENT_DIR,
|
||||
sockName: SOCK_NAME,
|
||||
nodePath: '/usr/bin/node'
|
||||
}
|
||||
|
||||
function probe(lines: string[]): string {
|
||||
return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n')
|
||||
}
|
||||
|
||||
function incumbent(lines: string[]): ReturnType<typeof parseRelayEndpointIncumbentProbe> {
|
||||
return parseRelayEndpointIncumbentProbe(OLD_SOCK, probe(lines))
|
||||
}
|
||||
|
||||
function issuedCommands(): string[] {
|
||||
return execCommand.mock.calls.map((call) => String(call[1]))
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
execCommand.mockReset()
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
describe('supersededRelayEndpointListCommand', () => {
|
||||
it('globs sibling version dirs for this target socket and skips the current one', () => {
|
||||
const command = supersededRelayEndpointListCommand(SWEEP)
|
||||
expect(command).toContain('"$base"/relay-*/"$sock_name"')
|
||||
expect(command).toContain('[ "$dir" = "$current" ] && continue')
|
||||
expect(command).toContain(SOCK_NAME)
|
||||
expect(command).toContain(CURRENT_DIR)
|
||||
})
|
||||
})
|
||||
|
||||
describe('classifySupersededRelay', () => {
|
||||
it('retains a live relay that still owns PTYs', () => {
|
||||
expect(
|
||||
classifySupersededRelay(
|
||||
incumbent([
|
||||
'PRESENT=yes',
|
||||
'LISTEN=accepted',
|
||||
'HOLDERS_SOURCE=lsof',
|
||||
'HOLDER=3669803 yes 13'
|
||||
])
|
||||
)
|
||||
).toBe('retained-live-work')
|
||||
})
|
||||
|
||||
it('nominates only a proven empty relay for reaping', () => {
|
||||
expect(
|
||||
classifySupersededRelay(
|
||||
incumbent(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
|
||||
)
|
||||
).toBe('reap-candidate')
|
||||
})
|
||||
|
||||
it('removes only a socket proven to have no holder', () => {
|
||||
expect(
|
||||
classifySupersededRelay(incumbent(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof']))
|
||||
).toBe('stale-endpoint-removed')
|
||||
})
|
||||
|
||||
it('does nothing at all for an unverifiable endpoint', () => {
|
||||
expect(
|
||||
classifySupersededRelay(
|
||||
incumbent(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])
|
||||
)
|
||||
).toBe('unverifiable')
|
||||
})
|
||||
})
|
||||
|
||||
describe('sweepSupersededRelayEndpoints', () => {
|
||||
it('leaves an upgrade-orphaned relay that still owns terminals running, untouched', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
|
||||
.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13'])
|
||||
)
|
||||
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
|
||||
expect(findings).toHaveLength(1)
|
||||
expect(findings[0]).toMatchObject({ sockPath: OLD_SOCK, outcome: 'retained-live-work' })
|
||||
expect(issuedCommands().some((command) => /\bkill\b/.test(command))).toBe(false)
|
||||
expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false)
|
||||
})
|
||||
|
||||
it('reaps the empty husk an upgrade leaves behind, once the host confirms it is gone', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
|
||||
.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
|
||||
)
|
||||
.mockResolvedValueOnce('GONE\n')
|
||||
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
|
||||
expect(findings[0].outcome).toBe('reaped')
|
||||
expect(issuedCommands()[2]).toContain('kill -TERM "$pid"')
|
||||
})
|
||||
|
||||
it('reports reap-unconfirmed rather than reaped when the pid is still there', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
|
||||
.mockResolvedValueOnce(
|
||||
probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0'])
|
||||
)
|
||||
.mockResolvedValueOnce('LIVE\n')
|
||||
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
|
||||
expect(findings[0].outcome).toBe('reap-unconfirmed')
|
||||
})
|
||||
|
||||
it('unlinks an orphaned socket only once nothing holds it, unpinning the dir for GC', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
|
||||
.mockResolvedValueOnce(probe(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof']))
|
||||
.mockResolvedValueOnce('')
|
||||
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
|
||||
expect(findings[0].outcome).toBe('stale-endpoint-removed')
|
||||
expect(issuedCommands()[2]).toBe(`rm -f '${OLD_SOCK}'`)
|
||||
})
|
||||
|
||||
it('touches nothing on a host it cannot interrogate', async () => {
|
||||
execCommand
|
||||
.mockResolvedValueOnce(`${OLD_SOCK}\n`)
|
||||
.mockResolvedValueOnce(probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable']))
|
||||
const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)
|
||||
expect(findings[0].outcome).toBe('unverifiable')
|
||||
expect(issuedCommands()).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('is a no-op when the listing fails, and never guesses at what was there', async () => {
|
||||
execCommand.mockRejectedValueOnce(new Error('exec failed'))
|
||||
await expect(sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)).resolves.toEqual([])
|
||||
})
|
||||
|
||||
it('does not run against Windows hosts, whose endpoints are named pipes', async () => {
|
||||
await expect(sweepSupersededRelayEndpoints(CONN, WINDOWS_HOST, SWEEP)).resolves.toEqual([])
|
||||
expect(execCommand).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Relays this target left behind at a *different* version directory.
|
||||
*
|
||||
* Every relay build installs to `~/.orca-remote/relay-<fullVersion>/` and binds its socket
|
||||
* inside it, so the socket path moves on every app update even though the filename component
|
||||
* is stable. After an update the new client binds a path the previous relay's PTYs were never
|
||||
* associated with, and the previous relay is never contacted again (#13614, #13852). Nothing
|
||||
* signals it and nothing reclaims it: with `--grace-time 0` it keeps its shells and agents
|
||||
* alive forever.
|
||||
*
|
||||
* This sweep makes that population *visible and deliberate* rather than silent. It does not
|
||||
* make it recoverable — the daemon handshake compares the build's content hash exactly
|
||||
* (`relay-handshake.ts`), so a new client cannot speak to an old daemon at all. See the report
|
||||
* on this change for what a real cross-version handoff would require.
|
||||
*
|
||||
* The one thing it will terminate is a relay that provably holds nothing. Everything else is
|
||||
* retained, including everything it merely failed to reach.
|
||||
*/
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import {
|
||||
describeRelayEndpointIncumbent,
|
||||
isReapableRelayHusk,
|
||||
probeRelayEndpointIncumbent,
|
||||
type RelayEndpointIncumbent
|
||||
} from './ssh-relay-endpoint-incumbent'
|
||||
import { reapEmptyRelayHusk } from './ssh-relay-endpoint-takeover'
|
||||
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
/**
|
||||
* `reaped` is the only outcome that claims a process ended, and it is only reachable from a
|
||||
* post-signal `kill -0` that failed. A signal we sent but could not confirm is
|
||||
* `reap-unconfirmed`, which is `unverifiable` — not `exited` by another name.
|
||||
*/
|
||||
export type SupersededRelayOutcome =
|
||||
| 'reaped'
|
||||
| 'reap-unconfirmed'
|
||||
| 'retained-live-work'
|
||||
| 'stale-endpoint-removed'
|
||||
| 'unverifiable'
|
||||
|
||||
export type SupersededRelayFinding = {
|
||||
sockPath: string
|
||||
outcome: SupersededRelayOutcome
|
||||
incumbent: RelayEndpointIncumbent
|
||||
}
|
||||
|
||||
export type SupersededRelaySweepOptions = {
|
||||
remoteHome: string
|
||||
/** Absolute path of the version directory this client just launched into; never swept. */
|
||||
currentRelayDir: string
|
||||
/** Stable per-target socket filename, from `relaySocketNameForInstanceId`. */
|
||||
sockName: string
|
||||
nodePath: string
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
const MAX_SWEPT_ENDPOINTS = 32
|
||||
|
||||
export function supersededRelayEndpointListCommand(options: {
|
||||
remoteHome: string
|
||||
currentRelayDir: string
|
||||
sockName: string
|
||||
}): string {
|
||||
return [
|
||||
`base=${shellEscape(`${options.remoteHome}/${RELAY_REMOTE_DIR}`)}`,
|
||||
`sock_name=${shellEscape(options.sockName)}`,
|
||||
`current=${shellEscape(options.currentRelayDir)}`,
|
||||
'for sock in "$base"/relay-*/"$sock_name"; do',
|
||||
' [ -S "$sock" ] || continue',
|
||||
' dir=${sock%/*}',
|
||||
' [ "$dir" = "$current" ] && continue',
|
||||
' printf \'%s\\n\' "$sock"',
|
||||
'done'
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
/** Remove a socket inode proven to have no holder, so version-dir GC can reclaim the tree. */
|
||||
export function removeStaleRelayEndpointCommand(sockPath: string): string {
|
||||
return `rm -f ${shellEscape(sockPath)}`
|
||||
}
|
||||
|
||||
export function classifySupersededRelay(
|
||||
incumbent: RelayEndpointIncumbent
|
||||
): Exclude<SupersededRelayOutcome, 'reaped' | 'reap-unconfirmed'> | 'reap-candidate' {
|
||||
if (incumbent.verdict === 'exited') {
|
||||
return incumbent.socketPresent ? 'stale-endpoint-removed' : 'unverifiable'
|
||||
}
|
||||
if (incumbent.verdict !== 'live') {
|
||||
return 'unverifiable'
|
||||
}
|
||||
return isReapableRelayHusk(incumbent) ? 'reap-candidate' : 'retained-live-work'
|
||||
}
|
||||
|
||||
export async function sweepSupersededRelayEndpoints(
|
||||
conn: SshConnection,
|
||||
hostPlatform: RemoteHostPlatform,
|
||||
options: SupersededRelaySweepOptions
|
||||
): Promise<SupersededRelayFinding[]> {
|
||||
if (isWindowsRemoteHost(hostPlatform)) {
|
||||
return []
|
||||
}
|
||||
let listing: string
|
||||
try {
|
||||
listing = await execCommand(conn, supersededRelayEndpointListCommand(options), {
|
||||
wrapCommand: true,
|
||||
signal: options.signal
|
||||
})
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
const sockPaths = listing
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.startsWith('/'))
|
||||
.slice(0, MAX_SWEPT_ENDPOINTS)
|
||||
|
||||
const findings: SupersededRelayFinding[] = []
|
||||
for (const sockPath of sockPaths) {
|
||||
options.signal?.throwIfAborted()
|
||||
const incumbent = await probeRelayEndpointIncumbent(
|
||||
conn,
|
||||
hostPlatform,
|
||||
options.nodePath,
|
||||
sockPath,
|
||||
{ signal: options.signal }
|
||||
)
|
||||
findings.push({
|
||||
sockPath,
|
||||
outcome: await applySupersededRelayDecision(conn, incumbent, options),
|
||||
incumbent
|
||||
})
|
||||
}
|
||||
logSupersededRelayFindings(findings)
|
||||
return findings
|
||||
}
|
||||
|
||||
async function applySupersededRelayDecision(
|
||||
conn: SshConnection,
|
||||
incumbent: RelayEndpointIncumbent,
|
||||
options: SupersededRelaySweepOptions
|
||||
): Promise<SupersededRelayOutcome> {
|
||||
const decision = classifySupersededRelay(incumbent)
|
||||
if (decision === 'stale-endpoint-removed') {
|
||||
try {
|
||||
await execCommand(conn, removeStaleRelayEndpointCommand(incumbent.sockPath), {
|
||||
wrapCommand: true,
|
||||
signal: options.signal
|
||||
})
|
||||
return 'stale-endpoint-removed'
|
||||
} catch {
|
||||
return 'unverifiable'
|
||||
}
|
||||
}
|
||||
if (decision !== 'reap-candidate') {
|
||||
return decision
|
||||
}
|
||||
return reapEmptyRelayHusk(conn, incumbent, { signal: options.signal })
|
||||
}
|
||||
|
||||
function logSupersededRelayFindings(findings: SupersededRelayFinding[]): void {
|
||||
for (const finding of findings) {
|
||||
const detail = describeRelayEndpointIncumbent(finding.incumbent)
|
||||
if (finding.outcome === 'retained-live-work') {
|
||||
console.warn(
|
||||
`[ssh-relay] Superseded relay retained (holds live work; not signalled): ${detail}`
|
||||
)
|
||||
continue
|
||||
}
|
||||
if (finding.outcome === 'unverifiable' || finding.outcome === 'reap-unconfirmed') {
|
||||
console.warn(`[ssh-relay] Superseded relay ${finding.outcome}: ${detail}`)
|
||||
continue
|
||||
}
|
||||
console.log(`[ssh-relay] Superseded relay ${finding.outcome}: ${detail}`)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user