fix(ssh): cancel pending loopback routes and guard process cleanup

This commit is contained in:
Neil
2026-10-23 09:30:11 -07:00
parent a85624222a
commit b301b6ac7b
6 changed files with 397 additions and 43 deletions
@@ -102,7 +102,6 @@ src/main/ssh/ssh-proxy-command.ts
src/main/ssh/system-transport/system-ssh-command.ts
src/main/ssh/system-transport/system-ssh-dynamic-forward-process.ts
src/main/ssh/system-transport/system-ssh-file-transfer.ts
src/main/ssh/system-transport/system-ssh-forward-process.ts
src/main/ssh/system-transport/system-ssh-operation-lifecycle.ts
src/main/startup/ensure-virtual-display.ts
src/main/windows-process-tree-kill.ts
@@ -0,0 +1,116 @@
import { EventEmitter } from 'node:events'
import { PassThrough } from 'node:stream'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { resetSshConnectionMocks } from './ssh-connection-test-harness'
import { createCallbacks, createTarget } from './ssh-connection-test-fixtures'
import { SshConnection } from './ssh-connection'
const { startForwardMock, waitForwardMock } = vi.hoisted(() => ({
startForwardMock: vi.fn(),
waitForwardMock: vi.fn()
}))
vi.mock('ssh2', async () =>
(await import('../__tests__/ssh-connection-test-client')).createSsh2Module()
)
vi.mock('../system-transport/system-ssh-binary', async () =>
(await import('./ssh-connection-test-harness')).createSystemSshBinaryModule()
)
vi.mock('../system-transport/system-ssh-args', async (importOriginal) =>
(await import('./ssh-connection-test-harness')).withSystemSshArgsMocks(await importOriginal())
)
vi.mock('../system-transport/system-ssh-command', async (importOriginal) =>
(await import('./ssh-connection-test-harness')).withSystemSshCommandMocks(await importOriginal())
)
vi.mock('../ssh-control-socket', async () =>
(await import('./ssh-connection-test-harness')).createControlSocketModule()
)
vi.mock('../config/ssh-config-parser', async () =>
(await import('./ssh-connection-test-harness')).createSshConfigParserModule()
)
vi.mock('../config/ssh-g-config-resolution', async (importOriginal) =>
(await import('./ssh-connection-test-harness')).withSshGConfigResolutionMocks(
await importOriginal()
)
)
vi.mock('../system-transport/system-ssh-loopback-route', async (importOriginal) => ({
...(await importOriginal()),
shouldUseSystemSshLoopbackRoute: () => true
}))
vi.mock('../system-transport/system-ssh-forward-process', () => ({
startSystemSshPortForwardProcess: startForwardMock,
waitForSystemSshForwardStartup: waitForwardMock
}))
describe('pending real loopback route startup ownership', () => {
beforeEach(() => {
resetSshConnectionMocks()
startForwardMock.mockReset()
waitForwardMock.mockReset()
})
it('disconnect stops the already spawned forward before startup completes', async () => {
resetSshConnectionMocks()
const dispose = vi.fn()
const process = Object.assign(new EventEmitter(), {
pid: undefined,
exitCode: null,
signalCode: null,
stderr: new PassThrough()
})
startForwardMock.mockResolvedValue({ process, localPort: 50001, dispose })
let failStartup = (_error: Error): void => {}
waitForwardMock.mockImplementation(
() =>
new Promise((_resolve, reject) => {
failStartup = reject
})
)
const conn = new SshConnection(
createTarget({ proxyCommand: 'ssh -W %h:%p bastion.example.com' }),
createCallbacks()
)
await conn.connect()
await vi.waitFor(() => expect(waitForwardMock).toHaveBeenCalled())
await conn.disconnect()
const stoppedAtDisconnect = dispose.mock.calls.length
failStartup(new Error('review cleanup: interrupted slow handshake'))
await vi.waitFor(() => expect(dispose).toHaveBeenCalled())
expect(stoppedAtDisconnect).toBeGreaterThan(0)
})
it('cancels the superseded forward while keeping the new startup owned', async () => {
const createForward = () => ({
process: Object.assign(new EventEmitter(), {
pid: undefined,
exitCode: null,
signalCode: null
}),
localPort: 50001,
dispose: vi.fn()
})
const stale = createForward()
const current = createForward()
startForwardMock.mockResolvedValueOnce(stale).mockResolvedValueOnce(current)
const failures: ((error: Error) => void)[] = []
waitForwardMock.mockImplementation(
() => new Promise((_resolve, reject) => failures.push(reject))
)
const conn = new SshConnection(
createTarget({ proxyCommand: 'ssh -W %h:%p bastion.example.com' }),
createCallbacks()
)
await conn.connect()
await vi.waitFor(() => expect(waitForwardMock).toHaveBeenCalledTimes(1))
await conn.reconnect()
await vi.waitFor(() => expect(waitForwardMock).toHaveBeenCalledTimes(2))
expect(stale.dispose).toHaveBeenCalledTimes(1)
expect(current.dispose).not.toHaveBeenCalled()
failures[0]?.(new Error('superseded handshake'))
await Promise.resolve()
await Promise.resolve()
await conn.disconnect()
expect(current.dispose).toHaveBeenCalledTimes(1)
failures[1]?.(new Error('disconnected handshake'))
await Promise.resolve()
expect(conn.getSystemSshBuildArgsOptions().loopbackRoute).toBeUndefined()
})
})
+20 -3
View File
@@ -107,6 +107,7 @@ export class SshConnection {
private systemSshGssapiOnlyForSession = false
private systemSshLoopbackRoute: SystemSshLoopbackRoute | null = null
private systemSshLoopbackRouteStartingGeneration: number | null = null
private systemSshLoopbackRouteAbortController: AbortController | null = null
private useSystemSshTransport = false
private credentialAbortController = new AbortController()
private readonly pendingSsh2Clients = new Set<SshClient>()
@@ -681,15 +682,28 @@ export class SshConnection {
}
// Why per generation: a start a reconnect superseded can still be in flight, and must not block the new one.
this.systemSshLoopbackRouteStartingGeneration = connectGeneration
const controller = new AbortController()
this.systemSshLoopbackRouteAbortController = controller
const settle = (): void => {
if (this.systemSshLoopbackRouteStartingGeneration === connectGeneration) {
if (this.systemSshLoopbackRouteAbortController === controller) {
this.systemSshLoopbackRouteStartingGeneration = null
this.systemSshLoopbackRouteAbortController = null
}
}
startSystemSshLoopbackRoute(this.target, this.systemSshResolvedConfig, options).then(
startSystemSshLoopbackRoute(
this.target,
this.systemSshResolvedConfig,
options,
process.platform,
controller.signal
).then(
(route) => {
settle()
if (!this.isCurrentConnectAttempt(connectGeneration) || !this.useSystemSshTransport) {
if (
controller.signal.aborted ||
!this.isCurrentConnectAttempt(connectGeneration) ||
!this.useSystemSshTransport
) {
route.dispose()
return
}
@@ -847,6 +861,9 @@ export class SshConnection {
}
private resetSystemTransport(): void {
this.systemSshLoopbackRouteAbortController?.abort()
this.systemSshLoopbackRouteAbortController = null
this.systemSshLoopbackRouteStartingGeneration = null
this.systemSshLoopbackRoute?.dispose()
this.systemSshLoopbackRoute = null
this.systemSshResolvedConfig = null
@@ -1,5 +1,7 @@
import { spawn, type ChildProcess } from 'node:child_process'
import type { ChildProcessHandle as ChildProcess } from '@orca/process-host/process-spec'
import { spawnProcess } from '@orca/process-host'
import { connect } from 'node:net'
import { createSshOperationAbortError } from '../connection/ssh-exec-options'
import { buildForwardProcessSshArgs, type SystemSshBuildArgsOptions } from './system-ssh-args'
import { findSystemSsh } from './system-ssh-binary'
import { reserveLocalForwardPort } from './system-ssh-forward-port-reservation'
@@ -37,9 +39,10 @@ export function spawnSystemSshPortForward(
const args = buildSystemSshPortForwardArgs(target, localPort, remoteHost, remotePort, options)
// Why: port-forward ssh processes are not wired to Orca credential prompts;
// system SSH forwards must authenticate via OpenSSH config, agent, or control socket.
return spawn(sshPath, args, {
stdio: ['ignore', 'ignore', 'pipe'],
windowsHide: true
return spawnProcess({
program: sshPath,
args,
stdio: ['ignore', 'ignore', 'pipe']
})
}
@@ -81,13 +84,16 @@ export function startSystemSshPortForwardProcess(
localPort: number,
remoteHost: string,
remotePort: number,
options?: SystemSshBuildArgsOptions
options?: SystemSshBuildArgsOptions,
signal?: AbortSignal
): Promise<SystemSshPortForwardProcess> {
signal?.throwIfAborted()
const orphanArgs =
localPort === 0
? undefined
: buildSystemSshPortForwardArgs(target, localPort, remoteHost, remotePort, options)
return reserveLocalForwardPort(localPort, orphanArgs).then((boundPort) => {
signal?.throwIfAborted()
const process = spawnSystemSshPortForward(target, boundPort, remoteHost, remotePort, options)
stopOnOwnListenFailure(process, boundPort)
return {
@@ -139,7 +145,8 @@ function stopOnOwnListenFailure(process: ChildProcess, localPort: number): void
export function waitForSystemSshForwardStartup(
process: ChildProcess,
localPort: number,
listenerTimeoutMs?: number
listenerTimeoutMs?: number,
signal?: AbortSignal
): Promise<void> {
return new Promise((resolve, reject) => {
let stderr = ''
@@ -153,6 +160,7 @@ export function waitForSystemSshForwardStartup(
if (graceTimer) {
clearTimeout(graceTimer)
}
signal?.removeEventListener('abort', onAbort)
process.off('error', onError)
process.off('exit', onExit)
process.stderr?.off('data', onStderr)
@@ -165,6 +173,7 @@ export function waitForSystemSshForwardStartup(
cleanup()
callback()
}
const onAbort = (): void => finish(() => reject(createSshOperationAbortError()))
const onStderr = (chunk: Buffer): void => {
stderr += chunk.toString('utf-8')
}
@@ -176,7 +185,7 @@ export function waitForSystemSshForwardStartup(
}
const scheduleProbe = (): void => {
probeTimer = setTimeout(() => {
probeLocalForward(localPort).then(
probeLocalForward(localPort, signal).then(
() => finish(resolve),
() => {
if (!settled) {
@@ -203,6 +212,11 @@ export function waitForSystemSshForwardStartup(
)
)
}, listenerTimeoutMs ?? SYSTEM_SSH_FORWARD_STARTUP_GRACE_MS)
signal?.addEventListener('abort', onAbort, { once: true })
if (signal?.aborted) {
onAbort()
return
}
scheduleProbe()
})
}
@@ -285,14 +299,16 @@ function bestErrorLine(stderr: string): string {
)
}
function probeLocalForward(localPort: number): Promise<void> {
function probeLocalForward(localPort: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const socket = connect({ host: '127.0.0.1', port: localPort })
const cleanup = (): void => {
signal?.removeEventListener('abort', onAbort)
socket.off('connect', onConnect)
socket.off('error', onError)
socket.off('timeout', onTimeout)
}
const onAbort = (): void => onError(createSshOperationAbortError())
const onConnect = (): void => {
cleanup()
socket.destroy()
@@ -312,5 +328,9 @@ function probeLocalForward(localPort: number): Promise<void> {
socket.once('connect', onConnect)
socket.once('error', onError)
socket.once('timeout', onTimeout)
signal?.addEventListener('abort', onAbort, { once: true })
if (signal?.aborted) {
onAbort()
}
})
}
@@ -77,7 +77,7 @@ function createResolved(overrides?: Partial<SshResolvedConfig>): SshResolvedConf
}
type FakeProcess = EventEmitter & {
pid: number
pid: number | undefined
stdin: PassThrough
stdout: PassThrough
stderr: PassThrough
@@ -99,7 +99,7 @@ function createFakeProcess(): FakeProcess {
return proc
}
function createFakeServer(): EventEmitter {
function createFakeServer() {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fields are assigned on the next lines, before the server is returned.
const server = new EventEmitter() as EventEmitter & {
listen: ReturnType<typeof vi.fn>
@@ -354,4 +354,157 @@ describe('startSystemSshLoopbackRoute', () => {
expect(onExit).toHaveBeenCalledTimes(1)
})
it.each(['exit', 'signal'] as const)('does not kill a forward after %s', async (reason) => {
const forward = createFakeProcess()
const probe = createFakeProcess()
spawnMock.mockReturnValueOnce(forward).mockImplementationOnce(() => {
answerProbe(probe, 0)
return probe
})
const route = await startSystemSshLoopbackRoute(createTarget(), createResolved(), {}, 'win32')
if (reason === 'exit') {
forward.exitCode = 255
} else {
forward.signalCode = 'SIGTERM'
}
forward.emit('exit', forward.exitCode, forward.signalCode)
route.dispose()
route.dispose()
expect(treeKillMock).not.toHaveBeenCalled()
expect(forward.kill).not.toHaveBeenCalled()
})
it('cancels listener polling and closes its socket without waiting for child exit', async () => {
vi.useFakeTimers()
try {
const forward = createFakeProcess()
const socket = Object.assign(new EventEmitter(), { setTimeout: vi.fn(), destroy: vi.fn() })
spawnMock.mockReturnValueOnce(forward)
connectMock.mockReturnValue(socket)
const controller = new AbortController()
const pending = startSystemSshLoopbackRoute(
createTarget(),
createResolved(),
{},
'linux',
controller.signal
)
const rejected = expect(pending).rejects.toMatchObject({ name: 'AbortError' })
await vi.advanceTimersByTimeAsync(50)
expect(connectMock).toHaveBeenCalledTimes(1)
controller.abort()
await rejected
expect(forward.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM')
expect(socket.destroy).toHaveBeenCalledTimes(1)
expect(socket.eventNames()).toEqual([])
expect(forward.listenerCount('error')).toBe(0)
expect(vi.getTimerCount()).toBe(0)
await vi.advanceTimersByTimeAsync(60_000)
expect(connectMock).toHaveBeenCalledTimes(1)
expect(spawnMock).toHaveBeenCalledTimes(1)
} finally {
vi.useRealTimers()
}
})
it('cancels the login probe and forward without waiting for probe timeout', async () => {
vi.useFakeTimers()
try {
const forward = createFakeProcess()
const probe = createFakeProcess()
spawnMock.mockReturnValueOnce(forward).mockReturnValueOnce(probe)
const controller = new AbortController()
const pending = startSystemSshLoopbackRoute(
createTarget(),
createResolved(),
{},
'linux',
controller.signal
)
const rejected = expect(pending).rejects.toMatchObject({ name: 'AbortError' })
await vi.advanceTimersByTimeAsync(50)
expect(spawnMock).toHaveBeenCalledTimes(2)
controller.abort()
await rejected
expect(forward.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM')
expect(probe.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM')
expect(vi.getTimerCount()).toBe(0)
} finally {
vi.useRealTimers()
}
})
it('does not spawn a forward if cancelled during port reservation', async () => {
const server = createFakeServer()
createServerMock.mockReturnValueOnce(server)
server.listen.mockImplementationOnce(() => server)
const controller = new AbortController()
const pending = startSystemSshLoopbackRoute(
createTarget(),
createResolved(),
{},
'linux',
controller.signal
)
controller.abort()
server.emit('listening')
await expect(pending).rejects.toMatchObject({ name: 'AbortError' })
expect(spawnMock).not.toHaveBeenCalled()
})
it('disposes a live Windows route once and avoids fallback after child exit', async () => {
const forward = createFakeProcess()
const probe = createFakeProcess()
spawnMock.mockReturnValueOnce(forward).mockImplementationOnce(() => {
answerProbe(probe, 0)
return probe
})
const route = await startSystemSshLoopbackRoute(createTarget(), createResolved(), {}, 'win32')
let finishKill = (_stopped: boolean): void => {}
treeKillMock.mockImplementationOnce(
() =>
new Promise<boolean>((resolve) => {
finishKill = resolve
})
)
route.dispose()
route.dispose()
expect(treeKillMock).toHaveBeenCalledTimes(1)
forward.exitCode = 255
forward.emit('exit', 255)
finishKill(false)
await Promise.resolve()
expect(forward.kill).not.toHaveBeenCalled()
})
it('uses direct cleanup for a live forward with no PID', async () => {
const forward = createFakeProcess()
forward.pid = undefined
const probe = createFakeProcess()
spawnMock.mockReturnValueOnce(forward).mockImplementationOnce(() => {
answerProbe(probe, 0)
return probe
})
const route = await startSystemSshLoopbackRoute(createTarget(), createResolved(), {}, 'win32')
route.dispose()
route.dispose()
expect(treeKillMock).not.toHaveBeenCalled()
expect(forward.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM')
})
it('closes a live login probe when its output stream errors', async () => {
vi.useFakeTimers()
try {
const forward = createFakeProcess()
const probe = createFakeProcess()
spawnMock.mockReturnValueOnce(forward).mockReturnValueOnce(probe)
const pending = startSystemSshLoopbackRoute(createTarget(), createResolved(), {}, 'linux')
const rejected = expect(pending).rejects.toThrow('probe output failed')
await vi.advanceTimersByTimeAsync(50)
probe.stdout.emit('error', new Error('probe output failed'))
await rejected
expect(probe.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM')
expect(forward.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM')
expect(vi.getTimerCount()).toBe(0)
} finally {
vi.useRealTimers()
}
})
})
@@ -3,7 +3,8 @@
* One held `ssh -N -L` pays the ProxyCommand/ProxyJump handshake once; later commands log in over
* 127.0.0.1 through it instead of re-running the proxy for every exec.
*/
import type { ChildProcess } from 'node:child_process'
import type { ChildProcessHandle as ChildProcess } from '@orca/process-host/process-spec'
import { createSshOperationAbortError } from '../connection/ssh-exec-options'
import type { SshTarget } from '../../../shared/ssh-types'
import { terminateWindowsProcessTree } from '../../windows-process-tree-kill'
import type { SshResolvedConfig } from '../config/ssh-g-config-resolution'
@@ -64,27 +65,44 @@ export async function startSystemSshLoopbackRoute(
target: SshTarget,
resolved: SshResolvedConfig | null,
options: SystemSshBuildArgsOptions,
platform: NodeJS.Platform = process.platform
platform: NodeJS.Platform = process.platform,
signal?: AbortSignal
): Promise<SystemSshLoopbackRoute> {
signal?.throwIfAborted()
const remotePort = resolved?.port || target.port || 22
// Why: the forward itself must take the real proxy route, never an existing loopback one.
const forward = await startSystemSshPortForwardProcess(target, 0, 'localhost', remotePort, {
...options,
loopbackRoute: undefined
})
const dispose = (): void => stopLoopbackForward(forward.process, forward.dispose, platform)
const forward = await startSystemSshPortForwardProcess(
target,
0,
'localhost',
remotePort,
{ ...options, loopbackRoute: undefined },
signal
)
let disposed = false
const dispose = (): void => {
if (disposed) {
return
}
disposed = true
stopLoopbackForward(forward.process, forward.dispose, platform)
}
signal?.addEventListener('abort', dispose, { once: true })
try {
signal?.throwIfAborted()
await waitForSystemSshForwardStartup(
forward.process,
forward.localPort,
SYSTEM_SSH_LOOPBACK_ROUTE_LISTEN_TIMEOUT_MS
SYSTEM_SSH_LOOPBACK_ROUTE_LISTEN_TIMEOUT_MS,
signal
)
const args = {
localPort: forward.localPort,
hostKeyAlias: getLoopbackRouteHostKeyAlias(target, resolved)
}
// Why: a listening forward does not prove sshd allows it (AllowTcpForwarding) or that auth works over it.
await probeLoopbackRoute(target, { ...options, loopbackRoute: args })
await probeLoopbackRoute(target, { ...options, loopbackRoute: args }, signal)
signal?.throwIfAborted()
return {
args,
onExit: (callback) => {
@@ -100,6 +118,8 @@ export async function startSystemSshLoopbackRoute(
} catch (error) {
dispose()
throw error
} finally {
signal?.removeEventListener('abort', dispose)
}
}
@@ -108,6 +128,9 @@ function stopLoopbackForward(
killForward: () => void,
platform: NodeJS.Platform
): void {
if (forwardProcess.exitCode !== null || forwardProcess.signalCode !== null) {
return
}
const pid = forwardProcess.pid
if (platform !== 'win32' || !pid) {
killForward()
@@ -115,13 +138,18 @@ function stopLoopbackForward(
}
// Why: Windows does not end a killed ssh's ProxyCommand child, which would keep the proxy tunnel open.
void terminateWindowsProcessTree(pid, { site: 'system-ssh-loopback-route' }).then((stopped) => {
if (!stopped) {
if (!stopped && forwardProcess.exitCode === null && forwardProcess.signalCode === null) {
killForward()
}
})
}
function probeLoopbackRoute(target: SshTarget, options: SystemSshBuildArgsOptions): Promise<void> {
function probeLoopbackRoute(
target: SshTarget,
options: SystemSshBuildArgsOptions,
signal?: AbortSignal
): Promise<void> {
signal?.throwIfAborted()
const channel = spawnSystemSshCommand(target, `echo ${LOOPBACK_ROUTE_PROBE_MARKER}`, {
...options,
nonInteractive: true,
@@ -129,25 +157,46 @@ function probeLoopbackRoute(target: SshTarget, options: SystemSshBuildArgsOption
})
return new Promise((resolve, reject) => {
let stdout = ''
const timeout = setTimeout(() => {
channel.close()
reject(new Error('System SSH loopback route probe timed out'))
}, SYSTEM_SSH_LOOPBACK_ROUTE_PROBE_TIMEOUT_MS)
channel.on('data', (data: Buffer) => {
stdout += data.toString('utf-8')
})
channel.stderr.resume()
channel.once('error', (error: Error) => {
clearTimeout(timeout)
reject(error)
})
channel.once('close', (code: number | null) => {
clearTimeout(timeout)
if (code === 0 && stdout.includes(LOOPBACK_ROUTE_PROBE_MARKER)) {
resolve()
let settled = false
const finish = (error?: Error, closeChannel = false): void => {
if (settled) {
return
}
reject(new Error(`System SSH loopback route probe failed (exit ${code ?? 'unknown'})`))
})
settled = true
clearTimeout(timeout)
signal?.removeEventListener('abort', onAbort)
channel.off('data', onData)
channel.off('close', onClose)
if (error) {
if (closeChannel) {
channel.close()
}
reject(error)
} else {
resolve()
}
}
const onAbort = (): void => finish(createSshOperationAbortError(), true)
const onData = (data: Buffer): void => {
stdout += data.toString('utf-8')
}
const onClose = (code: number | null): void => {
finish(
code === 0 && stdout.includes(LOOPBACK_ROUTE_PROBE_MARKER)
? undefined
: new Error(`System SSH loopback route probe failed (exit ${code ?? 'unknown'})`)
)
}
const timeout = setTimeout(() => {
finish(new Error('System SSH loopback route probe timed out'), true)
}, SYSTEM_SSH_LOOPBACK_ROUTE_PROBE_TIMEOUT_MS)
channel.on('data', onData)
channel.stderr.resume()
channel.once('error', (error: Error) => finish(error, true))
channel.once('close', onClose)
signal?.addEventListener('abort', onAbort, { once: true })
if (signal?.aborted) {
onAbort()
}
})
}