diff --git a/src/main/ssh/ssh-relay-deploy.test.ts b/src/main/ssh/ssh-relay-deploy.test.ts index 3134461fda1..fdd719cb91f 100644 --- a/src/main/ssh/ssh-relay-deploy.test.ts +++ b/src/main/ssh/ssh-relay-deploy.test.ts @@ -186,9 +186,9 @@ describe('deployAndLaunchRelay', () => { expect(progress).toContain('Starting relay...') }) - it('does not launch fresh after unconfirmed stale-socket cleanup', async () => { + it('does not launch fresh after an unconfirmed endpoint-incumbent probe', async () => { const conn = makeMockConnection() - const unconfirmedCleanup = Object.assign(new Error('socket cleanup still running'), { + const unconfirmedCleanup = Object.assign(new Error('endpoint probe still running'), { sshChannelCloseConfirmed: false }) vi.mocked(waitForSentinel).mockRejectedValueOnce(new Error('stale relay reconnect failed')) diff --git a/src/main/ssh/ssh-relay-deploy.ts b/src/main/ssh/ssh-relay-deploy.ts index 911d185b0fd..124fb68bbcc 100644 --- a/src/main/ssh/ssh-relay-deploy.ts +++ b/src/main/ssh/ssh-relay-deploy.ts @@ -77,6 +77,8 @@ import { import { detectRemoteHostPlatform } from './ssh-remote-platform-detection' import { powerShellCommand, powerShellLiteral, powerShellNativeArg } from './ssh-remote-powershell' import { relaySocketNameForInstanceId } from './ssh-relay-instance-id' +import { resolveRelayEndpointBeforeRelaunch } from './ssh-relay-endpoint-takeover' +import { sweepSupersededRelayEndpoints } from './ssh-relay-superseded-endpoints' import { isSshSessionLimitError } from './ssh-session-limit-error' import { isWindowsRelayPipePath, @@ -580,6 +582,17 @@ async function deployAndLaunchRelayAttempt( hostPlatform, recoverOneStaleRelayUploadStageCommand(hostPlatform, uploadStagePoolDir) ) + .catch(() => {}) + // Why before GC: a superseded relay pins its version dir via the live-socket probe, so the + // sweep has to settle first or GC keeps every orphan's tree forever. + .then(() => + sweepSupersededRelayEndpoints(conn, hostPlatform, { + remoteHome, + currentRelayDir: remoteRelayDir, + sockName: relaySocketNameForInstanceId(relayInstanceId), + nodePath: launched.nodePath + }) + ) .catch(() => {}) .then(() => gcOldRelayVersions(conn, remoteHome, remoteRelayDir, hostPlatform, { @@ -1457,17 +1470,15 @@ async function launchRelay( } catch (err) { signal?.throwIfAborted() console.warn( - '[ssh-relay] Socket reconnect failed, launching fresh relay:', + '[ssh-relay] Socket reconnect failed, establishing what owns the endpoint:', err instanceof Error ? err.message : String(err) ) - // Why: stale socket from a crashed relay — remove it so the fresh launch can bind at the same path. - await execCommand(conn, `rm -f ${shellEscape(sockFile)}`, { signal }).catch( - (cleanupErr) => { - if (isUnconfirmedSshCommandTermination(cleanupErr)) { - throw cleanupErr - } - } - ) + // Why not `rm -f`: unlinking does not close the listener the incumbent already holds, + // so a refused --connect (version mismatch, rotated credential) used to leave a live + // relay running forever with its PTYs while a replacement bound the same path (#8585). + await resolveRelayEndpointBeforeRelaunch(conn, hostPlatform, nodePath, sockFile, err, { + signal + }) signal?.throwIfAborted() } } diff --git a/src/main/ssh/ssh-relay-endpoint-incumbent-shell.integration.test.ts b/src/main/ssh/ssh-relay-endpoint-incumbent-shell.integration.test.ts new file mode 100644 index 00000000000..7ece0b6532e --- /dev/null +++ b/src/main/ssh/ssh-relay-endpoint-incumbent-shell.integration.test.ts @@ -0,0 +1,170 @@ +/** + * The probe and reap scripts run on someone else's machine and decide whether a process is + * signalled, so the shell itself is the part worth testing for real. These cases run the + * generated scripts through /bin/sh against real unix sockets and real processes. + */ +import { execFile, spawn, type ChildProcess } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' +import { + isReapableRelayHusk, + parseRelayEndpointIncumbentProbe, + relayEndpointIncumbentProbeCommand, + type RelayEndpointIncumbent +} from './ssh-relay-endpoint-incumbent' +import { reapEmptyRelayHuskCommand } from './ssh-relay-endpoint-takeover' + +const posixOnly = process.platform === 'win32' ? describe.skip : describe + +const FAKE_RELAY_SOURCE = ` +const net = require('net') +const sock = process.argv[process.argv.indexOf('--sock-path') + 1] +if (process.argv.includes('--with-child')) { + require('child_process').spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore' + }) +} +net.createServer(() => {}).listen(sock, () => process.stdout.write('READY\\n')) +process.on('SIGTERM', () => process.exit(0)) +` + +function sh(script: string): Promise { + return new Promise((resolve, reject) => { + execFile('/bin/sh', ['-c', script], { timeout: 20_000 }, (error, stdout) => { + if (error) { + reject(error) + return + } + resolve(stdout) + }) + }) +} + +let workDir: string +let hasLsof = false +const running: ChildProcess[] = [] + +function startFakeRelay(sockPath: string, withChild = false): Promise { + const args = [join(workDir, 'relay.js'), '--sock-path', sockPath] + if (withChild) { + args.push('--with-child') + } + const child = spawn(process.execPath, args, { stdio: ['ignore', 'pipe', 'ignore'] }) + running.push(child) + return new Promise((resolve, reject) => { + child.stdout.on('data', (chunk: Buffer) => { + if (chunk.toString().includes('READY')) { + resolve(child) + } + }) + child.on('exit', () => reject(new Error('fake relay exited before listening'))) + }) +} + +async function probe(sockPath: string): Promise { + const output = await sh(relayEndpointIncumbentProbeCommand(process.execPath, sockPath)) + return parseRelayEndpointIncumbentProbe(sockPath, output) +} + +beforeAll(async () => { + workDir = mkdtempSync(join(tmpdir(), 'orca-relay-incumbent-')) + writeFileSync(join(workDir, 'relay.js'), FAKE_RELAY_SOURCE) + hasLsof = await sh('command -v lsof >/dev/null 2>&1 && echo yes || echo no').then( + (out) => out.trim() === 'yes' + ) +}) + +afterEach(() => { + while (running.length > 0) { + running.pop()?.kill('SIGKILL') + } +}) + +afterAll(() => { + rmSync(workDir, { recursive: true, force: true }) +}) + +it('runs the holder-enumeration assertions on this machine', () => { + // Why asserted rather than assumed: the cases below degrade to verdict-only checks without + // lsof, and a silently degraded suite would stop covering the reap gate entirely. + expect(hasLsof).toBe(true) +}) + +posixOnly('relay endpoint probe against a real socket', () => { + it('reports live, and identifies the holding process, for a listening relay', async () => { + const sockPath = join(workDir, 'live.sock') + const relay = await startFakeRelay(sockPath) + const incumbent = await probe(sockPath) + + expect(incumbent.verdict).toBe('live') + expect(incumbent.evidence).toBe('accepted-connection') + expect(incumbent.socketPresent).toBe(true) + if (!hasLsof) { + return + } + expect(incumbent.holders.map((holder) => holder.pid)).toEqual([relay.pid]) + expect(incumbent.holders[0]).toMatchObject({ matchesRelayArgv: true, childCount: 0 }) + expect(isReapableRelayHusk(incumbent)).toBe(true) + }) + + it('refuses to call a relay with a live child an empty husk', async () => { + const sockPath = join(workDir, 'busy.sock') + await startFakeRelay(sockPath, true) + const incumbent = await probe(sockPath) + + expect(incumbent.verdict).toBe('live') + if (!hasLsof) { + return + } + expect(incumbent.holders[0].childCount).toBeGreaterThan(0) + expect(isReapableRelayHusk(incumbent)).toBe(false) + }) + + it('reports exited for a socket inode a SIGKILLed relay left behind', async () => { + const sockPath = join(workDir, 'stale.sock') + const relay = await startFakeRelay(sockPath) + relay.kill('SIGKILL') + await new Promise((resolve) => relay.on('exit', resolve)) + + const incumbent = await probe(sockPath) + expect(incumbent.socketPresent).toBe(true) + expect(incumbent.verdict).toBe(hasLsof ? 'exited' : 'unverifiable') + }) + + it('reports no listener for a path that was never bound', async () => { + const incumbent = await probe(join(workDir, 'never-existed.sock')) + expect(incumbent.socketPresent).toBe(false) + expect(incumbent.verdict).toBe(hasLsof ? 'exited' : 'unverifiable') + }) +}) + +posixOnly('empty relay husk reap against a real process', () => { + it('terminates a proven-empty relay and confirms the pid is gone', async () => { + const sockPath = join(workDir, 'husk.sock') + const relay = await startFakeRelay(sockPath) + const output = await sh(reapEmptyRelayHuskCommand(relay.pid!, sockPath)) + expect(output.trim()).toBe('GONE') + }) + + it('refuses to signal a relay that acquired a child after it was probed', async () => { + const sockPath = join(workDir, 'raced.sock') + const relay = await startFakeRelay(sockPath, true) + const output = await sh(reapEmptyRelayHuskCommand(relay.pid!, sockPath)) + expect(output.trim()).toBe('BUSY') + expect(relay.killed).toBe(false) + }) + + it('refuses to signal a pid whose argv is not this relay at this socket', async () => { + const sockPath = join(workDir, 'mismatch.sock') + await startFakeRelay(sockPath) + const bystander = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore' + }) + running.push(bystander) + const output = await sh(reapEmptyRelayHuskCommand(bystander.pid!, sockPath)) + expect(output.trim()).toBe('MISMATCH') + expect(bystander.killed).toBe(false) + }) +}) diff --git a/src/main/ssh/ssh-relay-endpoint-incumbent.test.ts b/src/main/ssh/ssh-relay-endpoint-incumbent.test.ts new file mode 100644 index 00000000000..a65cb33fc57 --- /dev/null +++ b/src/main/ssh/ssh-relay-endpoint-incumbent.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it, vi } from 'vitest' + +const execCommand = vi.fn() +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: (...args: unknown[]) => execCommand(...args), + isUnconfirmedSshCommandTermination: (error: unknown) => + (error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false +})) + +import { + describeRelayEndpointIncumbent, + isReapableRelayHusk, + mayLaunchOverRelayEndpoint, + parseRelayEndpointIncumbentProbe, + probeRelayEndpointIncumbent, + relayEndpointIncumbentProbeCommand, + withHandshakeRefusalEvidence, + type RelayEndpointIncumbent +} from './ssh-relay-endpoint-incumbent' +import type { SshConnection } from './ssh-connection' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const SOCK = '/home/u/.orca-remote/relay-0.1.0+aaaa/relay-deadbeef.sock' +const POSIX_HOST = getRemoteHostPlatform('linux-x64') +const WINDOWS_HOST = getRemoteHostPlatform('win32-x64') + +function probeOutput(lines: string[]): string { + return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n') +} + +describe('parseRelayEndpointIncumbentProbe', () => { + it('reports live when the socket accepted a connection', () => { + const incumbent = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=4242 yes 13']) + ) + expect(incumbent.verdict).toBe('live') + expect(incumbent.evidence).toBe('accepted-connection') + expect(incumbent.holders).toEqual([{ pid: 4242, matchesRelayArgv: true, childCount: 13 }]) + }) + + it('reports live when a process still holds an inode that refuses connections', () => { + const incumbent = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof', 'HOLDER=91 yes 2']) + ) + expect(incumbent.verdict).toBe('live') + expect(incumbent.evidence).toBe('holder-process') + }) + + it('reports exited only when the connect was refused AND nothing holds the socket', () => { + const incumbent = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof']) + ) + expect(incumbent.verdict).toBe('exited') + expect(incumbent.evidence).toBe('no-holder') + expect(incumbent.socketPresent).toBe(true) + }) + + it('reports unverifiable when the host cannot enumerate socket holders', () => { + const incumbent = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=unavailable']) + ) + expect(incumbent.verdict).toBe('unverifiable') + expect(incumbent.holdersEnumerable).toBe(false) + }) + + it('reports unverifiable when the connect probe timed out', () => { + const incumbent = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=lsof']) + ) + expect(incumbent.verdict).toBe('unverifiable') + }) + + it('reports unverifiable for truncated or garbled probe output', () => { + expect(parseRelayEndpointIncumbentProbe(SOCK, 'PRESENT=yes\nLISTEN=refused').verdict).toBe( + 'unverifiable' + ) + expect(parseRelayEndpointIncumbentProbe(SOCK, '').verdict).toBe('unverifiable') + }) + + it('drops holder lines that do not carry a usable pid', () => { + const incumbent = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof', 'HOLDER=- no unknown']) + ) + expect(incumbent.holders).toEqual([]) + expect(incumbent.verdict).toBe('exited') + }) + + it('keeps an unreadable child count as null rather than zero', () => { + const [holder] = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=7 yes unknown']) + ).holders + expect(holder.childCount).toBeNull() + }) +}) + +describe('probeRelayEndpointIncumbent', () => { + it('never asserts death when the probe itself could not run', async () => { + execCommand.mockRejectedValueOnce(new Error('channel closed')) + const incumbent = await probeRelayEndpointIncumbent( + {} as SshConnection, + POSIX_HOST, + '/usr/bin/node', + SOCK + ) + expect(incumbent.verdict).toBe('unverifiable') + expect(incumbent.holders).toEqual([]) + }) + + it('does not shell out on Windows hosts, where the endpoint is a named pipe', async () => { + execCommand.mockClear() + const incumbent = await probeRelayEndpointIncumbent( + {} as SshConnection, + WINDOWS_HOST, + 'node.exe', + SOCK + ) + expect(execCommand).not.toHaveBeenCalled() + expect(incumbent.verdict).toBe('unverifiable') + }) +}) + +describe('relayEndpointIncumbentProbeCommand', () => { + it('ANDs the lsof selectors so it cannot match unrelated unix-socket holders', () => { + expect(relayEndpointIncumbentProbeCommand('/usr/bin/node', SOCK)).toContain( + 'lsof -t -a -U "$sock"' + ) + }) + + it('never mutates the host: no unlink, no signal', () => { + const command = relayEndpointIncumbentProbeCommand('/usr/bin/node', SOCK) + expect(command).not.toMatch(/\brm\b/) + expect(command).not.toMatch(/\bkill\b/) + }) +}) + +describe('withHandshakeRefusalEvidence', () => { + it('upgrades an unenumerable endpoint to live when the daemon answered the handshake', () => { + const probed = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable']) + ) + const incumbent = withHandshakeRefusalEvidence(probed) + expect(incumbent.verdict).toBe('live') + expect(incumbent.evidence).toBe('handshake-refusal') + expect(mayLaunchOverRelayEndpoint(incumbent)).toBe(false) + }) + + it('leaves stronger evidence in place', () => { + const probed = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof']) + ) + expect(withHandshakeRefusalEvidence(probed).evidence).toBe('accepted-connection') + }) +}) + +describe('mayLaunchOverRelayEndpoint', () => { + const verdicts: RelayEndpointIncumbent['verdict'][] = ['live', 'unverifiable', 'exited'] + it.each(verdicts)('permits a relaunch for %s only when it is not live', (verdict) => { + const incumbent = { ...parseRelayEndpointIncumbentProbe(SOCK, ''), verdict } + expect(mayLaunchOverRelayEndpoint(incumbent)).toBe(verdict !== 'live') + }) +}) + +describe('isReapableRelayHusk', () => { + const husk = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=500 yes 0']) + ) + + it('accepts a single proven relay holder with zero children', () => { + expect(isReapableRelayHusk(husk)).toBe(true) + }) + + it('refuses a relay that still holds children', () => { + expect( + isReapableRelayHusk({ + ...husk, + holders: [{ pid: 500, matchesRelayArgv: true, childCount: 1 }] + }) + ).toBe(false) + }) + + it('refuses a holder whose child count could not be read', () => { + expect( + isReapableRelayHusk({ + ...husk, + holders: [{ pid: 500, matchesRelayArgv: true, childCount: null }] + }) + ).toBe(false) + }) + + it('refuses a holder whose argv is not this relay at this socket', () => { + expect( + isReapableRelayHusk({ + ...husk, + holders: [{ pid: 500, matchesRelayArgv: false, childCount: 0 }] + }) + ).toBe(false) + }) + + it('refuses when more than one process holds the socket', () => { + expect( + isReapableRelayHusk({ + ...husk, + holders: [ + { pid: 500, matchesRelayArgv: true, childCount: 0 }, + { pid: 501, matchesRelayArgv: true, childCount: 0 } + ] + }) + ).toBe(false) + }) + + it('refuses an unverifiable endpoint however empty it looks', () => { + expect(isReapableRelayHusk({ ...husk, verdict: 'unverifiable' })).toBe(false) + expect(isReapableRelayHusk({ ...husk, holdersEnumerable: false })).toBe(false) + }) +}) + +describe('describeRelayEndpointIncumbent', () => { + it('distinguishes "no holders" from "could not enumerate holders"', () => { + const none = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof']) + ) + const unknown = parseRelayEndpointIncumbentProbe( + SOCK, + probeOutput(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=unavailable']) + ) + expect(describeRelayEndpointIncumbent(none)).toContain('holders=none') + expect(describeRelayEndpointIncumbent(unknown)).toContain('holders=unenumerable') + }) +}) diff --git a/src/main/ssh/ssh-relay-endpoint-incumbent.ts b/src/main/ssh/ssh-relay-endpoint-incumbent.ts new file mode 100644 index 00000000000..2688267f4c7 --- /dev/null +++ b/src/main/ssh/ssh-relay-endpoint-incumbent.ts @@ -0,0 +1,285 @@ +/** + * Who currently owns a relay socket path, answered with host evidence. + * + * The client used to answer this by assumption: a failed `--connect` was read as "the relay + * crashed", the socket was `rm -f`'d, and a fresh relay bound the same path. Unlinking a unix + * socket does not close the listener the incumbent already holds, so an alive-but-refusing + * relay (the `RelayVersionMismatchError` case, and the credential-rotation case) was left + * running forever with its PTYs (#8585). + * + * The verdict vocabulary is fixed by docs/reference/ssh-execution-boundary.md — `live` / + * `unverifiable` / `exited`, with no synonyms and no collapsing. Two consequences are load + * bearing here: + * + * - `exited` is a claim about **this endpoint**, not about every relay on the host. It means + * nothing holds this socket path, established positively (a connect that was refused *and* + * an enumeration that found no holder). A relay whose socket was already unlinked is + * invisible to this probe by construction — that is what the superseded sweep is for. + * - a probe that could not run, a host without `lsof`, or a connect that failed for any other + * reason is `unverifiable`. It never authorizes unlinking, rebinding over, or signalling. + */ +import type { SshConnection } from './ssh-connection' +import { shellEscape } from './ssh-connection-utils' +import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +export type RelayEndpointVerdict = 'live' | 'unverifiable' | 'exited' + +export type RelayEndpointEvidence = + | 'accepted-connection' + | 'handshake-refusal' + | 'holder-process' + | 'no-holder' + | 'inconclusive' + +export type RelayEndpointHolder = { + pid: number + /** The holder's argv names relay.js AND this exact socket path. */ + matchesRelayArgv: boolean + /** Direct children, or null when `pgrep` could not answer. Never guessed. */ + childCount: number | null +} + +export type RelayEndpointIncumbent = { + sockPath: string + verdict: RelayEndpointVerdict + evidence: RelayEndpointEvidence + socketPresent: boolean + /** Pids proven to hold this exact socket. Empty when the host could not enumerate them. */ + holders: RelayEndpointHolder[] + /** False when no enumeration tool was available — an empty `holders` then proves nothing. */ + holdersEnumerable: boolean +} + +const PROBE_BEGIN = 'ORCA-INCUMBENT-BEGIN' +const PROBE_END = 'ORCA-INCUMBENT-END' +const CONNECT_PROBE_TIMEOUT_MS = 1000 + +// Why ES5 syntax: nodePath may be a host-resolved system node, not the bundled one. +const CONNECT_PROBE_JS = [ + 'var s=require("net").connect(process.argv[1]);', + 'var done=false;', + 'function say(v){if(done)return;done=true;try{s.destroy()}catch(e){};', + 'process.stdout.write(v);process.exit(0)}', + 's.on("connect",function(){say("accepted")});', + 's.on("error",function(e){', + 'say(e.code==="ECONNREFUSED"?"refused":e.code==="ENOENT"?"absent":"unknown")});', + `setTimeout(function(){say("unknown")},${CONNECT_PROBE_TIMEOUT_MS})` +].join('') + +/** + * A POSIX probe that reports only what the host actually observed. Every field has an + * explicit "could not tell" value; nothing is inferred from a missing tool. + */ +export function relayEndpointIncumbentProbeCommand(nodePath: string, sockPath: string): string { + const sock = shellEscape(sockPath) + const node = shellEscape(nodePath) + return [ + `sock=${sock}`, + `node=${node}`, + `printf '%s\\n' ${shellEscape(PROBE_BEGIN)}`, + 'if [ -S "$sock" ]; then', + " printf 'PRESENT=yes\\n'", + ` listen=$("$node" -e ${shellEscape(CONNECT_PROBE_JS)} "$sock" 2>/dev/null) || listen=unknown`, + ' [ -n "$listen" ] || listen=unknown', + 'else', + " printf 'PRESENT=no\\n'", + ' listen=absent', + 'fi', + 'printf \'LISTEN=%s\\n\' "$listen"', + 'if command -v lsof >/dev/null 2>&1; then', + " printf 'HOLDERS_SOURCE=lsof\\n'", + // Why -a: lsof ORs its selectors, so without it every unix-socket holder on the box + // would be reported as holding this path (#8762). + ' for pid in $(lsof -t -a -U "$sock" 2>/dev/null); do', + ' args=$(ps -o args= -p "$pid" 2>/dev/null | tr "\\n" " ")', + ' match=no', + ' case "$args" in *relay.js*"$sock"*) match=yes ;; esac', + ' kids=unknown', + ' if command -v pgrep >/dev/null 2>&1; then', + ' kids=$(pgrep -P "$pid" 2>/dev/null | grep -c .)', + ' fi', + ' printf \'HOLDER=%s %s %s\\n\' "$pid" "$match" "$kids"', + ' done', + 'else', + " printf 'HOLDERS_SOURCE=unavailable\\n'", + 'fi', + `printf '%s\\n' ${shellEscape(PROBE_END)}` + ].join('\n') +} + +export function parseRelayEndpointIncumbentProbe( + sockPath: string, + output: string +): RelayEndpointIncumbent { + const lines = output.split('\n').map((line) => line.trim()) + if (!lines.includes(PROBE_BEGIN) || !lines.includes(PROBE_END)) { + return unverifiableEndpoint(sockPath) + } + const socketPresent = lines.includes('PRESENT=yes') + const listen = lines.find((line) => line.startsWith('LISTEN='))?.slice('LISTEN='.length) ?? '' + const holdersEnumerable = lines.includes('HOLDERS_SOURCE=lsof') + const holders = lines + .filter((line) => line.startsWith('HOLDER=')) + .map((line) => parseHolder(line.slice('HOLDER='.length))) + .filter((holder): holder is RelayEndpointHolder => holder !== null) + + if (listen === 'accepted') { + return { + sockPath, + verdict: 'live', + evidence: 'accepted-connection', + socketPresent, + holders, + holdersEnumerable + } + } + if (holders.length > 0) { + // The inode is held by a running process that is not accepting — wedged, not gone. + return { + sockPath, + verdict: 'live', + evidence: 'holder-process', + socketPresent, + holders, + holdersEnumerable + } + } + if (holdersEnumerable && (listen === 'refused' || listen === 'absent')) { + return { + sockPath, + verdict: 'exited', + evidence: 'no-holder', + socketPresent, + holders, + holdersEnumerable + } + } + return { ...unverifiableEndpoint(sockPath), socketPresent, holders, holdersEnumerable } +} + +function parseHolder(value: string): RelayEndpointHolder | null { + const [rawPid, rawMatch, rawKids] = value.split(/\s+/) + const pid = Number.parseInt(rawPid ?? '', 10) + if (!Number.isInteger(pid) || pid <= 0) { + return null + } + const childCount = Number.parseInt(rawKids ?? '', 10) + return { + pid, + matchesRelayArgv: rawMatch === 'yes', + childCount: Number.isInteger(childCount) && childCount >= 0 ? childCount : null + } +} + +function unverifiableEndpoint(sockPath: string): RelayEndpointIncumbent { + return { + sockPath, + verdict: 'unverifiable', + evidence: 'inconclusive', + socketPresent: false, + holders: [], + holdersEnumerable: false + } +} + +export async function probeRelayEndpointIncumbent( + conn: SshConnection, + hostPlatform: RemoteHostPlatform, + nodePath: string, + sockPath: string, + options?: { signal?: AbortSignal } +): Promise { + // Windows relays are named pipes: there is no inode to unlink and no `lsof`, so the + // orphan-by-unlink mechanism this probe defends against cannot occur there. + if (isWindowsRemoteHost(hostPlatform)) { + return unverifiableEndpoint(sockPath) + } + try { + const output = await execCommand(conn, relayEndpointIncumbentProbeCommand(nodePath, sockPath), { + wrapCommand: true, + signal: options?.signal + }) + return parseRelayEndpointIncumbentProbe(sockPath, output) + } catch (err) { + // An exec whose channel never confirmed close may still be running remotely; the caller + // must not race a detached launch against it. + if (isUnconfirmedSshCommandTermination(err)) { + throw err + } + // Any other unanswered probe observes nothing. It is never evidence of death. + return unverifiableEndpoint(sockPath) + } +} + +/** + * A relay that told us its version over the wire is `live` by positive host evidence, even on + * a host where nothing can enumerate socket holders. + */ +export function withHandshakeRefusalEvidence( + incumbent: RelayEndpointIncumbent +): RelayEndpointIncumbent { + if (incumbent.verdict === 'live') { + return incumbent + } + return { ...incumbent, verdict: 'live', evidence: 'handshake-refusal' } +} + +/** + * May a fresh relay be launched onto this path? + * + * Only `live` forbids it. `unverifiable` is permitted because the *daemon* — not the client — + * performs the takeover: `RelaySocketOwnership.listen` re-probes on EADDRINUSE, refuses to + * steal a path that accepts connections, and only unlinks an inode whose identity is + * unchanged. That check is atomic with the bind, which a client-side `rm -f` can never be. + */ +export function mayLaunchOverRelayEndpoint(incumbent: RelayEndpointIncumbent): boolean { + return incumbent.verdict !== 'live' +} + +/** + * A live relay that provably holds nothing: identity confirmed against its argv, exactly one + * holder, and zero children. Reaping it destroys no user work. Anything less is retained — + * killing the wrong pid on someone's remote host is the worst outcome available here. + */ +export function isReapableRelayHusk(incumbent: RelayEndpointIncumbent): boolean { + if (incumbent.verdict !== 'live' || !incumbent.holdersEnumerable) { + return false + } + if (incumbent.holders.length !== 1) { + return false + } + const [holder] = incumbent.holders + return holder.matchesRelayArgv && holder.childCount === 0 +} + +export function describeRelayEndpointIncumbent(incumbent: RelayEndpointIncumbent): string { + const holders = incumbent.holders + .map((holder) => `${holder.pid}(children=${holder.childCount ?? 'unknown'})`) + .join(',') + return ( + `${incumbent.sockPath} verdict=${incumbent.verdict} evidence=${incumbent.evidence} ` + + `holders=${incumbent.holdersEnumerable ? holders || 'none' : 'unenumerable'}` + ) +} + +/** + * Thrown instead of orphaning: a live relay owns the endpoint and refused us, so the path is + * not ours to rebind. Terminal for this attempt — the user resolves it with Reset Relay, + * which signals the incumbent deliberately and with consent. + */ +export class RelayEndpointHeldError extends Error { + readonly name = 'RelayEndpointHeldError' + constructor(readonly incumbent: RelayEndpointIncumbent) { + super( + `A live relay still owns ${incumbent.sockPath} and refused this connection ` + + `(${describeRelayEndpointIncumbent(incumbent)}). Orca will not replace it, because ` + + 'unlinking its socket would strand its terminals. Use Reset Relay for this host to ' + + 'stop it, then reconnect.' + ) + } +} + +export function isRelayEndpointHeldError(err: unknown): err is RelayEndpointHeldError { + return err instanceof RelayEndpointHeldError +} diff --git a/src/main/ssh/ssh-relay-endpoint-takeover.test.ts b/src/main/ssh/ssh-relay-endpoint-takeover.test.ts new file mode 100644 index 00000000000..687d633b92b --- /dev/null +++ b/src/main/ssh/ssh-relay-endpoint-takeover.test.ts @@ -0,0 +1,159 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const execCommand = vi.fn() +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: (...args: unknown[]) => execCommand(...args), + isUnconfirmedSshCommandTermination: (error: unknown) => + (error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false +})) + +import { isRelayEndpointHeldError } from './ssh-relay-endpoint-incumbent' +import { + interpretRelayHuskReapOutput, + reapEmptyRelayHuskCommand, + resolveRelayEndpointBeforeRelaunch +} from './ssh-relay-endpoint-takeover' +import { RelayVersionMismatchError } from './ssh-relay-version-mismatch-error' +import type { SshConnection } from './ssh-connection' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const SOCK = '/home/u/.orca-remote/relay-0.1.0+aaaa/relay-deadbeef.sock' +const HOST = getRemoteHostPlatform('linux-x64') +const CONN = {} as SshConnection + +function probe(lines: string[]): string { + return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n') +} + +function issuedCommands(): string[] { + return execCommand.mock.calls.map((call) => String(call[1])) +} + +function resolve(reconnectError: unknown = new Error('connect failed')): Promise { + return resolveRelayEndpointBeforeRelaunch(CONN, HOST, '/usr/bin/node', SOCK, reconnectError) +} + +beforeEach(() => { + execCommand.mockReset() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'log').mockImplementation(() => {}) +}) + +describe('incumbent alive and refusing', () => { + it('refuses to rebind a live relay holding PTYs, and signals nothing', async () => { + execCommand.mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13']) + ) + await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError) + // The whole point of #8585: the incumbent's socket must survive so it is not orphaned. + expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false) + expect(issuedCommands().some((command) => /\bkill\b/.test(command))).toBe(false) + }) + + it('names the incumbent pid and the Reset Relay escape hatch in the error', async () => { + execCommand.mockResolvedValue( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13']) + ) + await expect(resolve()).rejects.toThrow(/3669803\(children=13\)/) + await expect(resolve()).rejects.toThrow(/Reset Relay/) + }) + + it('treats a version mismatch as live even where holders cannot be enumerated', async () => { + execCommand.mockResolvedValue( + probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable']) + ) + const mismatch = new RelayVersionMismatchError('0.1.0+new', '0.1.0+old', '') + await expect(resolve(mismatch)).rejects.toSatisfy(isRelayEndpointHeldError) + expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false) + }) + + it('reaps a live relay only when it provably holds nothing, and confirms it is gone', async () => { + execCommand + .mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0']) + ) + .mockResolvedValueOnce('GONE\n') + await expect(resolve()).resolves.toMatchObject({ verdict: 'live' }) + expect(issuedCommands()[1]).toContain('kill -TERM "$pid"') + }) + + it('does not launch over an empty relay whose death could not be confirmed', async () => { + execCommand + .mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0']) + ) + .mockResolvedValueOnce('LIVE\n') + await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError) + }) + + it('does not launch over a relay the host refused to signal on its own re-check', async () => { + execCommand + .mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0']) + ) + .mockResolvedValueOnce('BUSY\n') + await expect(resolve()).rejects.toSatisfy(isRelayEndpointHeldError) + }) +}) + +describe('incumbent genuinely gone', () => { + it('permits the relaunch without unlinking anything itself', async () => { + execCommand.mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof']) + ) + await expect(resolve()).resolves.toMatchObject({ verdict: 'exited', evidence: 'no-holder' }) + // The daemon unlinks under an identity check that is atomic with its bind; the client + // cannot be, which is what created the orphan in the first place. + expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false) + }) +}) + +describe('incumbent unverifiable', () => { + it('permits the relaunch but never claims the incumbent exited', async () => { + execCommand.mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable']) + ) + await expect(resolve()).resolves.toMatchObject({ verdict: 'unverifiable' }) + expect(issuedCommands()).toHaveLength(1) + }) + + it('stays unverifiable when the probe command itself fails', async () => { + execCommand.mockRejectedValueOnce(new Error('exec timeout')) + await expect(resolve()).resolves.toMatchObject({ verdict: 'unverifiable' }) + }) +}) + +describe('reapEmptyRelayHuskCommand', () => { + it('re-verifies argv and emptiness on the host immediately before signalling', () => { + const command = reapEmptyRelayHuskCommand(4242, SOCK) + expect(command.indexOf('MISMATCH')).toBeLessThan(command.indexOf('kill -TERM')) + expect(command.indexOf('BUSY')).toBeLessThan(command.indexOf('kill -TERM')) + }) + + it('sends SIGTERM only, so the relay runs its own socket cleanup', () => { + const command = reapEmptyRelayHuskCommand(4242, SOCK) + expect(command).toContain('kill -TERM') + expect(command).not.toContain('kill -KILL') + expect(command).not.toContain('-9') + }) + + it('aborts without signalling when the host cannot count children', () => { + expect(reapEmptyRelayHuskCommand(4242, SOCK)).toContain( + "command -v pgrep >/dev/null 2>&1 || { printf 'BUSY\\n'; exit 0; }" + ) + }) +}) + +describe('interpretRelayHuskReapOutput', () => { + it('claims reaped only for a post-signal liveness check that failed', () => { + expect(interpretRelayHuskReapOutput('GONE\n')).toBe('reaped') + expect(interpretRelayHuskReapOutput('LIVE\n')).toBe('reap-unconfirmed') + expect(interpretRelayHuskReapOutput('')).toBe('reap-unconfirmed') + expect(interpretRelayHuskReapOutput('unexpected noise')).toBe('reap-unconfirmed') + }) + + it('reports a host-side refusal as retained rather than as a failed kill', () => { + expect(interpretRelayHuskReapOutput('MISMATCH\n')).toBe('retained-live-work') + expect(interpretRelayHuskReapOutput('BUSY\n')).toBe('retained-live-work') + }) +}) diff --git a/src/main/ssh/ssh-relay-endpoint-takeover.ts b/src/main/ssh/ssh-relay-endpoint-takeover.ts new file mode 100644 index 00000000000..f104aab5256 --- /dev/null +++ b/src/main/ssh/ssh-relay-endpoint-takeover.ts @@ -0,0 +1,133 @@ +/** + * Deciding whether a relay socket path is ours to take, and acting on the answer. + * + * The only destructive action available here is a SIGTERM to a relay that has been proven — + * by argv, by socket-holder enumeration, and by a zero child count re-checked on the host + * immediately before the signal — to hold nothing at all. Everything else is left running. + * Per docs/reference/ssh-execution-boundary.md, a relay we merely failed to reach is + * `unverifiable`, and `unverifiable` never authorizes a kill or a rebind. + */ +import type { SshConnection } from './ssh-connection' +import { shellEscape } from './ssh-connection-utils' +import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { + describeRelayEndpointIncumbent, + isReapableRelayHusk, + mayLaunchOverRelayEndpoint, + probeRelayEndpointIncumbent, + RelayEndpointHeldError, + withHandshakeRefusalEvidence, + type RelayEndpointIncumbent +} from './ssh-relay-endpoint-incumbent' +import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +/** `reaped` is only reachable from a post-signal `kill -0` that failed. Nothing else claims it. */ +export type RelayHuskReapResult = 'reaped' | 'reap-unconfirmed' | 'retained-live-work' + +const REAP_CONFIRM_ATTEMPTS = 15 + +/** + * Signal one relay, re-verifying identity and emptiness inside the same command. + * + * The re-verification is not belt-and-braces: a client can attach and spawn a PTY between the + * probe and the signal, and pids are reused. `MISMATCH`/`BUSY` abort without signalling. + */ +export function reapEmptyRelayHuskCommand(pid: number, sockPath: string): string { + return [ + `pid=${shellEscape(String(pid))}`, + `sock=${shellEscape(sockPath)}`, + 'args=$(ps -o args= -p "$pid" 2>/dev/null | tr "\\n" " ")', + 'case "$args" in *relay.js*"$sock"*) ;; *) printf \'MISMATCH\\n\'; exit 0 ;; esac', + "command -v pgrep >/dev/null 2>&1 || { printf 'BUSY\\n'; exit 0; }", + 'kids=$(pgrep -P "$pid" 2>/dev/null | grep -c .)', + '[ "$kids" = "0" ] || { printf \'BUSY\\n\'; exit 0; }', + // SIGTERM only: the relay's own handler disposes and unlinks. SIGKILL would leave the + // socket inode behind and skip that shutdown path for no gain on an empty daemon. + 'kill -TERM "$pid" 2>/dev/null || true', + 'i=0', + `while [ $i -lt ${REAP_CONFIRM_ATTEMPTS} ]; do`, + ' kill -0 "$pid" 2>/dev/null || { printf \'GONE\\n\'; exit 0; }', + ' sleep 0.2', + ' i=$((i+1))', + 'done', + "printf 'LIVE\\n'" + ].join('\n') +} + +export function interpretRelayHuskReapOutput(output: string): RelayHuskReapResult { + const state = output.trim().split('\n').pop()?.trim() + if (state === 'GONE') { + return 'reaped' + } + // The host refused on its own re-check: what is there is not the empty relay we probed, so + // nothing was signalled and nothing is claimed about it. + if (state === 'MISMATCH' || state === 'BUSY') { + return 'retained-live-work' + } + return 'reap-unconfirmed' +} + +export async function reapEmptyRelayHusk( + conn: SshConnection, + incumbent: RelayEndpointIncumbent, + options?: { signal?: AbortSignal } +): Promise { + const holder = incumbent.holders[0] + if (!holder) { + return 'retained-live-work' + } + try { + const output = await execCommand( + conn, + reapEmptyRelayHuskCommand(holder.pid, incumbent.sockPath), + { wrapCommand: true, signal: options?.signal } + ) + return interpretRelayHuskReapOutput(output) + } catch (err) { + if (isUnconfirmedSshCommandTermination(err)) { + throw err + } + return 'reap-unconfirmed' + } +} + +/** + * Called when `--connect` to an existing socket failed and the caller is about to launch a + * replacement at the same path. Resolves to nothing when the launch may proceed; throws + * `RelayEndpointHeldError` when a live relay owns the path and holds work. + * + * `unverifiable` deliberately permits the launch: the daemon, not the client, performs the + * takeover. `RelaySocketOwnership.listen` re-probes on EADDRINUSE, refuses a path that accepts + * connections, and only unlinks an inode whose identity is unchanged — a check that is atomic + * with the bind, which a client-side `rm -f` can never be. + */ +export async function resolveRelayEndpointBeforeRelaunch( + conn: SshConnection, + hostPlatform: RemoteHostPlatform, + nodePath: string, + sockPath: string, + reconnectError: unknown, + options?: { signal?: AbortSignal } +): Promise { + const probed = await probeRelayEndpointIncumbent(conn, hostPlatform, nodePath, sockPath, options) + // A daemon that answered the handshake with its own version is live by positive host + // evidence, even where nothing can enumerate socket holders. + const incumbent = isRelayVersionMismatchError(reconnectError) + ? withHandshakeRefusalEvidence(probed) + : probed + console.warn(`[ssh-relay] Relay endpoint incumbent: ${describeRelayEndpointIncumbent(incumbent)}`) + + if (mayLaunchOverRelayEndpoint(incumbent)) { + return incumbent + } + if (!isReapableRelayHusk(incumbent)) { + throw new RelayEndpointHeldError(incumbent) + } + const result = await reapEmptyRelayHusk(conn, incumbent, options) + if (result !== 'reaped') { + throw new RelayEndpointHeldError(incumbent) + } + console.log(`[ssh-relay] Reaped empty relay husk holding ${sockPath}`) + return incumbent +} diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index bdc7c4370f2..ed5ae3f9cfd 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -6,6 +6,7 @@ import type { BrowserWindow } from 'electron' import { deployAndLaunchRelay } from './ssh-relay-deploy' import { execCommand } from './ssh-relay-deploy-helpers' import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error' +import { isRelayEndpointHeldError } from './ssh-relay-endpoint-incumbent' import { replayPendingSshPtyKills } from './ssh-pending-pty-kill-replay' import { SshChannelMultiplexer } from './ssh-channel-multiplexer' import { SshPtyProvider } from '../providers/ssh-pty-provider' @@ -629,7 +630,13 @@ export class SshRelaySession { } // Why: terminal on first connect — a deployed binary against a still-running legacy daemon, or a // claim another connection holds. Notify the callback but still rethrow. - if (isRelayVersionMismatchError(err) || isSshOwnerAdmissionBlockedError(err)) { + // RelayEndpointHeldError is terminal for the same reason: a live incumbent owns the + // socket path, and backoff cannot make it hand it over. The user resolves it. + if ( + isRelayVersionMismatchError(err) || + isRelayEndpointHeldError(err) || + isSshOwnerAdmissionBlockedError(err) + ) { console.warn( `[ssh-relay-session] Terminal relay error on initial connect for ${this.targetId}: ${err.message}` ) @@ -783,7 +790,13 @@ export class SshRelaySession { } // Why terminal: neither a version mismatch nor a blocked owner claim is reconcilable by backoff // retry, so fire the typed callback and drop out of 'reconnecting'. - if (isRelayVersionMismatchError(err) || isSshOwnerAdmissionBlockedError(err)) { + // RelayEndpointHeldError is terminal for the same reason: a live incumbent owns the + // socket path, and backoff cannot make it hand it over. The user resolves it. + if ( + isRelayVersionMismatchError(err) || + isRelayEndpointHeldError(err) || + isSshOwnerAdmissionBlockedError(err) + ) { console.warn( `[ssh-relay-session] Terminal relay error for ${this.targetId}: ${err.message}` ) diff --git a/src/main/ssh/ssh-relay-superseded-endpoints.test.ts b/src/main/ssh/ssh-relay-superseded-endpoints.test.ts new file mode 100644 index 00000000000..874d9aae3fe --- /dev/null +++ b/src/main/ssh/ssh-relay-superseded-endpoints.test.ts @@ -0,0 +1,164 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const execCommand = vi.fn() +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: (...args: unknown[]) => execCommand(...args), + isUnconfirmedSshCommandTermination: (error: unknown) => + (error as { sshChannelCloseConfirmed?: boolean } | null)?.sshChannelCloseConfirmed === false +})) + +import { parseRelayEndpointIncumbentProbe } from './ssh-relay-endpoint-incumbent' +import { + classifySupersededRelay, + supersededRelayEndpointListCommand, + sweepSupersededRelayEndpoints +} from './ssh-relay-superseded-endpoints' +import type { SshConnection } from './ssh-connection' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const HOME = '/home/u' +const SOCK_NAME = 'relay-deadbeef.sock' +const CURRENT_DIR = `${HOME}/.orca-remote/relay-0.1.0+bd3ec370d21d` +const OLD_SOCK = `${HOME}/.orca-remote/relay-0.1.0+7175e0a40ea7/${SOCK_NAME}` +const HOST = getRemoteHostPlatform('linux-x64') +const WINDOWS_HOST = getRemoteHostPlatform('win32-x64') +const CONN = {} as SshConnection + +const SWEEP = { + remoteHome: HOME, + currentRelayDir: CURRENT_DIR, + sockName: SOCK_NAME, + nodePath: '/usr/bin/node' +} + +function probe(lines: string[]): string { + return ['ORCA-INCUMBENT-BEGIN', ...lines, 'ORCA-INCUMBENT-END'].join('\n') +} + +function incumbent(lines: string[]): ReturnType { + return parseRelayEndpointIncumbentProbe(OLD_SOCK, probe(lines)) +} + +function issuedCommands(): string[] { + return execCommand.mock.calls.map((call) => String(call[1])) +} + +beforeEach(() => { + execCommand.mockReset() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'log').mockImplementation(() => {}) +}) + +describe('supersededRelayEndpointListCommand', () => { + it('globs sibling version dirs for this target socket and skips the current one', () => { + const command = supersededRelayEndpointListCommand(SWEEP) + expect(command).toContain('"$base"/relay-*/"$sock_name"') + expect(command).toContain('[ "$dir" = "$current" ] && continue') + expect(command).toContain(SOCK_NAME) + expect(command).toContain(CURRENT_DIR) + }) +}) + +describe('classifySupersededRelay', () => { + it('retains a live relay that still owns PTYs', () => { + expect( + classifySupersededRelay( + incumbent([ + 'PRESENT=yes', + 'LISTEN=accepted', + 'HOLDERS_SOURCE=lsof', + 'HOLDER=3669803 yes 13' + ]) + ) + ).toBe('retained-live-work') + }) + + it('nominates only a proven empty relay for reaping', () => { + expect( + classifySupersededRelay( + incumbent(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0']) + ) + ).toBe('reap-candidate') + }) + + it('removes only a socket proven to have no holder', () => { + expect( + classifySupersededRelay(incumbent(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])) + ).toBe('stale-endpoint-removed') + }) + + it('does nothing at all for an unverifiable endpoint', () => { + expect( + classifySupersededRelay( + incumbent(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable']) + ) + ).toBe('unverifiable') + }) +}) + +describe('sweepSupersededRelayEndpoints', () => { + it('leaves an upgrade-orphaned relay that still owns terminals running, untouched', async () => { + execCommand + .mockResolvedValueOnce(`${OLD_SOCK}\n`) + .mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=3669803 yes 13']) + ) + const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP) + expect(findings).toHaveLength(1) + expect(findings[0]).toMatchObject({ sockPath: OLD_SOCK, outcome: 'retained-live-work' }) + expect(issuedCommands().some((command) => /\bkill\b/.test(command))).toBe(false) + expect(issuedCommands().some((command) => /\brm -f\b/.test(command))).toBe(false) + }) + + it('reaps the empty husk an upgrade leaves behind, once the host confirms it is gone', async () => { + execCommand + .mockResolvedValueOnce(`${OLD_SOCK}\n`) + .mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0']) + ) + .mockResolvedValueOnce('GONE\n') + const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP) + expect(findings[0].outcome).toBe('reaped') + expect(issuedCommands()[2]).toContain('kill -TERM "$pid"') + }) + + it('reports reap-unconfirmed rather than reaped when the pid is still there', async () => { + execCommand + .mockResolvedValueOnce(`${OLD_SOCK}\n`) + .mockResolvedValueOnce( + probe(['PRESENT=yes', 'LISTEN=accepted', 'HOLDERS_SOURCE=lsof', 'HOLDER=80583 yes 0']) + ) + .mockResolvedValueOnce('LIVE\n') + const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP) + expect(findings[0].outcome).toBe('reap-unconfirmed') + }) + + it('unlinks an orphaned socket only once nothing holds it, unpinning the dir for GC', async () => { + execCommand + .mockResolvedValueOnce(`${OLD_SOCK}\n`) + .mockResolvedValueOnce(probe(['PRESENT=yes', 'LISTEN=refused', 'HOLDERS_SOURCE=lsof'])) + .mockResolvedValueOnce('') + const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP) + expect(findings[0].outcome).toBe('stale-endpoint-removed') + expect(issuedCommands()[2]).toBe(`rm -f '${OLD_SOCK}'`) + }) + + it('touches nothing on a host it cannot interrogate', async () => { + execCommand + .mockResolvedValueOnce(`${OLD_SOCK}\n`) + .mockResolvedValueOnce(probe(['PRESENT=yes', 'LISTEN=unknown', 'HOLDERS_SOURCE=unavailable'])) + const findings = await sweepSupersededRelayEndpoints(CONN, HOST, SWEEP) + expect(findings[0].outcome).toBe('unverifiable') + expect(issuedCommands()).toHaveLength(2) + }) + + it('is a no-op when the listing fails, and never guesses at what was there', async () => { + execCommand.mockRejectedValueOnce(new Error('exec failed')) + await expect(sweepSupersededRelayEndpoints(CONN, HOST, SWEEP)).resolves.toEqual([]) + }) + + it('does not run against Windows hosts, whose endpoints are named pipes', async () => { + await expect(sweepSupersededRelayEndpoints(CONN, WINDOWS_HOST, SWEEP)).resolves.toEqual([]) + expect(execCommand).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/ssh-relay-superseded-endpoints.ts b/src/main/ssh/ssh-relay-superseded-endpoints.ts new file mode 100644 index 00000000000..1a9554f7b82 --- /dev/null +++ b/src/main/ssh/ssh-relay-superseded-endpoints.ts @@ -0,0 +1,178 @@ +/** + * Relays this target left behind at a *different* version directory. + * + * Every relay build installs to `~/.orca-remote/relay-/` and binds its socket + * inside it, so the socket path moves on every app update even though the filename component + * is stable. After an update the new client binds a path the previous relay's PTYs were never + * associated with, and the previous relay is never contacted again (#13614, #13852). Nothing + * signals it and nothing reclaims it: with `--grace-time 0` it keeps its shells and agents + * alive forever. + * + * This sweep makes that population *visible and deliberate* rather than silent. It does not + * make it recoverable — the daemon handshake compares the build's content hash exactly + * (`relay-handshake.ts`), so a new client cannot speak to an old daemon at all. See the report + * on this change for what a real cross-version handoff would require. + * + * The one thing it will terminate is a relay that provably holds nothing. Everything else is + * retained, including everything it merely failed to reach. + */ +import type { SshConnection } from './ssh-connection' +import { shellEscape } from './ssh-connection-utils' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { execCommand } from './ssh-relay-deploy-helpers' +import { + describeRelayEndpointIncumbent, + isReapableRelayHusk, + probeRelayEndpointIncumbent, + type RelayEndpointIncumbent +} from './ssh-relay-endpoint-incumbent' +import { reapEmptyRelayHusk } from './ssh-relay-endpoint-takeover' +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +/** + * `reaped` is the only outcome that claims a process ended, and it is only reachable from a + * post-signal `kill -0` that failed. A signal we sent but could not confirm is + * `reap-unconfirmed`, which is `unverifiable` — not `exited` by another name. + */ +export type SupersededRelayOutcome = + | 'reaped' + | 'reap-unconfirmed' + | 'retained-live-work' + | 'stale-endpoint-removed' + | 'unverifiable' + +export type SupersededRelayFinding = { + sockPath: string + outcome: SupersededRelayOutcome + incumbent: RelayEndpointIncumbent +} + +export type SupersededRelaySweepOptions = { + remoteHome: string + /** Absolute path of the version directory this client just launched into; never swept. */ + currentRelayDir: string + /** Stable per-target socket filename, from `relaySocketNameForInstanceId`. */ + sockName: string + nodePath: string + signal?: AbortSignal +} + +const MAX_SWEPT_ENDPOINTS = 32 + +export function supersededRelayEndpointListCommand(options: { + remoteHome: string + currentRelayDir: string + sockName: string +}): string { + return [ + `base=${shellEscape(`${options.remoteHome}/${RELAY_REMOTE_DIR}`)}`, + `sock_name=${shellEscape(options.sockName)}`, + `current=${shellEscape(options.currentRelayDir)}`, + 'for sock in "$base"/relay-*/"$sock_name"; do', + ' [ -S "$sock" ] || continue', + ' dir=${sock%/*}', + ' [ "$dir" = "$current" ] && continue', + ' printf \'%s\\n\' "$sock"', + 'done' + ].join('\n') +} + +/** Remove a socket inode proven to have no holder, so version-dir GC can reclaim the tree. */ +export function removeStaleRelayEndpointCommand(sockPath: string): string { + return `rm -f ${shellEscape(sockPath)}` +} + +export function classifySupersededRelay( + incumbent: RelayEndpointIncumbent +): Exclude | 'reap-candidate' { + if (incumbent.verdict === 'exited') { + return incumbent.socketPresent ? 'stale-endpoint-removed' : 'unverifiable' + } + if (incumbent.verdict !== 'live') { + return 'unverifiable' + } + return isReapableRelayHusk(incumbent) ? 'reap-candidate' : 'retained-live-work' +} + +export async function sweepSupersededRelayEndpoints( + conn: SshConnection, + hostPlatform: RemoteHostPlatform, + options: SupersededRelaySweepOptions +): Promise { + if (isWindowsRemoteHost(hostPlatform)) { + return [] + } + let listing: string + try { + listing = await execCommand(conn, supersededRelayEndpointListCommand(options), { + wrapCommand: true, + signal: options.signal + }) + } catch { + return [] + } + const sockPaths = listing + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith('/')) + .slice(0, MAX_SWEPT_ENDPOINTS) + + const findings: SupersededRelayFinding[] = [] + for (const sockPath of sockPaths) { + options.signal?.throwIfAborted() + const incumbent = await probeRelayEndpointIncumbent( + conn, + hostPlatform, + options.nodePath, + sockPath, + { signal: options.signal } + ) + findings.push({ + sockPath, + outcome: await applySupersededRelayDecision(conn, incumbent, options), + incumbent + }) + } + logSupersededRelayFindings(findings) + return findings +} + +async function applySupersededRelayDecision( + conn: SshConnection, + incumbent: RelayEndpointIncumbent, + options: SupersededRelaySweepOptions +): Promise { + const decision = classifySupersededRelay(incumbent) + if (decision === 'stale-endpoint-removed') { + try { + await execCommand(conn, removeStaleRelayEndpointCommand(incumbent.sockPath), { + wrapCommand: true, + signal: options.signal + }) + return 'stale-endpoint-removed' + } catch { + return 'unverifiable' + } + } + if (decision !== 'reap-candidate') { + return decision + } + return reapEmptyRelayHusk(conn, incumbent, { signal: options.signal }) +} + +function logSupersededRelayFindings(findings: SupersededRelayFinding[]): void { + for (const finding of findings) { + const detail = describeRelayEndpointIncumbent(finding.incumbent) + if (finding.outcome === 'retained-live-work') { + console.warn( + `[ssh-relay] Superseded relay retained (holds live work; not signalled): ${detail}` + ) + continue + } + if (finding.outcome === 'unverifiable' || finding.outcome === 'reap-unconfirmed') { + console.warn(`[ssh-relay] Superseded relay ${finding.outcome}: ${detail}`) + continue + } + console.log(`[ssh-relay] Superseded relay ${finding.outcome}: ${detail}`) + } +}