diff --git a/src/main/ssh/legacy-relay/legacy-relay-bridge-connections.test.ts b/src/main/ssh/legacy-relay/legacy-relay-bridge-connections.test.ts new file mode 100644 index 00000000000..13b5ec7b66c --- /dev/null +++ b/src/main/ssh/legacy-relay/legacy-relay-bridge-connections.test.ts @@ -0,0 +1,169 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' +import type { ClientChannel } from 'ssh2' +import type { SshConnection } from '../connection/ssh-connection' +import { + createLegacyRelayBridgeConnections, + LEGACY_RELAY_CHANNEL_BUDGET_EXHAUSTED, + RELAY_CHANNELS_ON_PRIMARY, + RELAY_CHANNELS_PER_SIDE_CONNECTION +} from './legacy-relay-bridge-connections' + +type FakeChannel = EventEmitter & { close: () => void } + +function fakeConnection( + options: { + systemSsh?: boolean + controlMaster?: boolean + connectError?: Error + maxSessions?: number + } = {} +) { + const channels: FakeChannel[] = [] + const conn = { + usesSystemSshTransport: () => options.systemSsh === true, + canRunConcurrentExecCommands: () => + options.systemSsh !== true || options.controlMaster === true, + connect: vi.fn(async () => { + if (options.connectError) { + throw options.connectError + } + }), + disconnect: vi.fn(async () => {}), + exec: vi.fn(async () => { + if (channels.length >= (options.maxSessions ?? Infinity)) { + throw Object.assign(new Error('(SSH) Channel open failure: open failed'), { reason: 2 }) + } + const channel = Object.assign(new EventEmitter(), { + close: () => channel.emit('close') + }) + channels.push(channel) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pool only listens for 'close' and calls close(). + return channel as unknown as ClientChannel + }) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pool calls only the methods this fake defines. + return { conn, channels, asConnection: conn as unknown as SshConnection } +} + +describe('legacy relay bridge connections', () => { + it('keeps relay channels off the target connection, a bounded number per side connection', async () => { + const primary = fakeConnection() + const sides: ReturnType[] = [] + const pool = createLegacyRelayBridgeConnections({ + primary: () => primary.asConnection, + createSide: () => { + const side = fakeConnection() + sides.push(side) + return side.asConnection + } + }) + + for (let i = 0; i < 20; i++) { + await pool.open(`relay ${i}`) + } + + expect(primary.conn.exec).not.toHaveBeenCalled() + expect(sides.map((side) => side.channels.length)).toEqual([ + RELAY_CHANNELS_PER_SIDE_CONNECTION, + RELAY_CHANNELS_PER_SIDE_CONNECTION, + 20 - 2 * RELAY_CHANNELS_PER_SIDE_CONNECTION + ]) + }) + + it('closes a side connection once its last relay channel closes, and reuses freed room', async () => { + const sides: ReturnType[] = [] + const pool = createLegacyRelayBridgeConnections({ + primary: () => fakeConnection().asConnection, + createSide: () => { + const side = fakeConnection() + sides.push(side) + return side.asConnection + } + }) + await pool.open('a') + await pool.open('b') + sides[0].channels[0].close() + expect(sides[0].conn.disconnect).not.toHaveBeenCalled() + sides[0].channels[1].close() + expect(sides[0].conn.disconnect).toHaveBeenCalledTimes(1) + + await pool.open('c') + expect(sides).toHaveLength(2) + }) + + it('falls back to the target connection, within a budget, when a side cannot sign in unprompted', async () => { + const primary = fakeConnection() + const createSide = vi.fn(() => fakeConnection({ connectError: new Error('auth') }).asConnection) + const pool = createLegacyRelayBridgeConnections({ + primary: () => primary.asConnection, + createSide + }) + + for (let i = 0; i < RELAY_CHANNELS_ON_PRIMARY; i++) { + await pool.open(`relay ${i}`) + } + await expect(pool.open('one too many')).rejects.toThrow(LEGACY_RELAY_CHANNEL_BUDGET_EXHAUSTED) + + // One refused sign-in is not retried for every relay. + expect(createSide).toHaveBeenCalledTimes(1) + expect(primary.conn.exec).toHaveBeenCalledTimes(RELAY_CHANNELS_ON_PRIMARY) + + primary.channels[0].close() + await expect(pool.open('room again')).resolves.toBeDefined() + }) + + it('keeps system ssh on the target connection, budgeted only when a ControlMaster shares one session', async () => { + const createSide = vi.fn() + const shared = fakeConnection({ systemSsh: true, controlMaster: true }) + const sharedPool = createLegacyRelayBridgeConnections({ + primary: () => shared.asConnection, + createSide + }) + for (let i = 0; i < RELAY_CHANNELS_ON_PRIMARY; i++) { + await sharedPool.open(`relay ${i}`) + } + await expect(sharedPool.open('over')).rejects.toThrow(LEGACY_RELAY_CHANNEL_BUDGET_EXHAUSTED) + + const separate = fakeConnection({ systemSsh: true }) + const separatePool = createLegacyRelayBridgeConnections({ + primary: () => separate.asConnection, + createSide + }) + for (let i = 0; i < 20; i++) { + await separatePool.open(`relay ${i}`) + } + expect(separate.conn.exec).toHaveBeenCalledTimes(20) + expect(createSide).not.toHaveBeenCalled() + }) + + it('opens on another side connection when the host allows fewer sessions than the shard size', async () => { + const primary = fakeConnection() + const sides: ReturnType[] = [] + const pool = createLegacyRelayBridgeConnections({ + primary: () => primary.asConnection, + createSide: () => { + const side = fakeConnection({ maxSessions: 3 }) + sides.push(side) + return side.asConnection + } + }) + for (let i = 0; i < 7; i++) { + await pool.open(`relay ${i}`) + } + expect(primary.conn.exec).not.toHaveBeenCalled() + expect(sides.map((side) => side.channels.length)).toEqual([3, 3, 1]) + }) + + it('disconnects side connections on dispose and opens nothing after', async () => { + const side = fakeConnection() + const pool = createLegacyRelayBridgeConnections({ + primary: () => fakeConnection().asConnection, + createSide: () => side.asConnection + }) + await pool.open('relay') + pool.dispose() + expect(side.conn.disconnect).toHaveBeenCalledTimes(1) + await expect(pool.open('late')).rejects.toThrow('legacy_relay_bridge_disposed') + }) +}) diff --git a/src/main/ssh/legacy-relay/legacy-relay-bridge-connections.ts b/src/main/ssh/legacy-relay/legacy-relay-bridge-connections.ts new file mode 100644 index 00000000000..f4d0ce04380 --- /dev/null +++ b/src/main/ssh/legacy-relay/legacy-relay-bridge-connections.ts @@ -0,0 +1,211 @@ +/** + * Where the bridge opens each old relay's long-lived `--connect` channel. + * sshd's MaxSessions (OpenSSH default 10) caps sessions per connection, and one host can keep a relay per + * desktop update it outlived; on the target's own connection they starved every later exec ("open failed"). + * COMPAT(legacy-relay-bridge): revisit 2027-01-15. + */ +import type { ClientChannel } from 'ssh2' +import { SshConnection } from '../connection/ssh-connection' +import type { SshExecOptions } from '../connection/ssh-exec-options' +import { isSshSessionLimitError } from '../ssh-session-limit-error' + +/** Below OpenSSH's default 10, leaving each side connection room for its own short execs. */ +export const RELAY_CHANNELS_PER_SIDE_CONNECTION = 8 +/** Fallback onto the target's connection keeps 4 of the default 10 for its own work. */ +export const RELAY_CHANNELS_ON_PRIMARY = 6 + +export const LEGACY_RELAY_CHANNEL_BUDGET_EXHAUSTED = 'legacy_relay_channel_budget_exhausted' + +type Side = { + conn: SshConnection + channels: number + /** Lowered when the host's own MaxSessions is below ours. */ + limit: number + ready: Promise + failed: boolean +} + +export type LegacyRelayBridgeConnections = { + open: (command: string, options?: SshExecOptions) => Promise + /** Close side connections; relay daemons and their shells keep running. */ + dispose: () => void +} + +export function createLegacyRelayBridgeConnections(deps: { + primary: () => SshConnection | null + /** Tests swap the connection; production signs in like the target without ever prompting. */ + createSide?: (primary: SshConnection) => SshConnection +}): LegacyRelayBridgeConnections { + const createSide = + deps.createSide ?? + ((primary: SshConnection) => + new SshConnection( + primary.getTarget(), + // No credential callback: a side connection never prompts, it falls back instead. + { onStateChange: () => {} }, + primary.getReusableCredentials() ?? undefined + )) + const sides: Side[] = [] + let primaryChannels = 0 + // A sign-in a side could not repeat stays unrepeatable for this primary; retry after it reconnects. + let sidesRefusedFor: SshConnection | null = null + let disposed = false + + const closeSide = (side: Side): void => { + const index = sides.indexOf(side) + if (index !== -1) { + sides.splice(index, 1) + } + void side.conn.disconnect({ quiet: true }).catch(() => {}) + } + + const releaseOnClose = (channel: ClientChannel, release: () => void): void => { + channel.once('close', release) + } + + const openOnSide = async ( + primary: SshConnection, + command: string, + options: SshExecOptions | undefined + ): Promise => { + let side = sides.find((entry) => !entry.failed && entry.channels < entry.limit) + if (!side) { + const conn = createSide(primary) + const created: Side = { + conn, + channels: 0, + limit: RELAY_CHANNELS_PER_SIDE_CONNECTION, + ready: conn.connect(), + failed: false + } + sides.push(created) + side = created + } + // Reserve before awaiting so concurrent opens cannot overfill one connection. + side.channels += 1 + try { + await side.ready + } catch (error) { + if (!side.failed) { + console.warn( + '[legacy-relay] side SSH connection failed; relay channels share the target connection', + error instanceof Error ? error.message : error + ) + } + side.failed = true + sidesRefusedFor = primary + side.channels -= 1 + closeSide(side) + return null + } + let channel: ClientChannel + try { + channel = await side.conn.exec(command, options) + } catch (error) { + side.channels -= 1 + const full = isSshSessionLimitError(error) && side.channels > 0 + if (full) { + // This host allows fewer sessions per connection; cap this one and open the relay elsewhere. + side.limit = side.channels + } + if (side.channels === 0) { + closeSide(side) + } + if (full && !disposed) { + return openOnSide(primary, command, options) + } + throw error + } + const owner = side + releaseOnClose(channel, () => { + owner.channels -= 1 + if (owner.channels === 0) { + closeSide(owner) + } + }) + if (disposed) { + channel.close() + } + return channel + } + + const openOnPrimary = async ( + primary: SshConnection, + command: string, + options: SshExecOptions | undefined + ): Promise => { + // An over-budget relay stays unverifiable and its shell keeps running; the host stays usable. + if (primaryChannels >= RELAY_CHANNELS_ON_PRIMARY) { + console.warn('[legacy-relay] relay channel budget on the target connection is full') + throw new Error(LEGACY_RELAY_CHANNEL_BUDGET_EXHAUSTED) + } + primaryChannels += 1 + let channel: ClientChannel + try { + channel = await primary.exec(command, options) + } catch (error) { + primaryChannels -= 1 + throw error + } + releaseOnClose(channel, () => { + primaryChannels -= 1 + }) + return channel + } + + return { + open: async (command, options) => { + if (disposed) { + throw new Error('legacy_relay_bridge_disposed') + } + const primary = deps.primary() + if (!primary) { + throw new Error('ssh_connection_unavailable') + } + if (primary.usesSystemSshTransport()) { + // Without a ControlMaster every exec is its own ssh process and sshd connection: no shared cap. + // With one, a side connection would join the same sshd session, so only the budget helps. + return primary.canRunConcurrentExecCommands() + ? openOnPrimary(primary, command, options) + : primary.exec(command, options) + } + if (sidesRefusedFor !== primary) { + const channel = await openOnSide(primary, command, options) + if (channel) { + return channel + } + } + return openOnPrimary(primary, command, options) + }, + dispose: () => { + disposed = true + for (const side of sides.splice(0)) { + void side.conn.disconnect({ quiet: true }).catch(() => {}) + } + } + } +} + +const byTarget = new Map() + +/** Each target's relay channels: one set per bridge, created on first use. */ +export const legacyRelayChannels = { + for(targetId: string, primary: () => SshConnection | null): LegacyRelayBridgeConnections { + const existing = byTarget.get(targetId) + if (existing) { + return existing + } + const created = createLegacyRelayBridgeConnections({ primary }) + byTarget.set(targetId, created) + return created + }, + /** One target's, or with no id every target's; relay daemons and their shells keep running. */ + dispose(targetId?: string): void { + for (const [id, channels] of byTarget) { + if (targetId === undefined || id === targetId) { + byTarget.delete(id) + channels.dispose() + } + } + } +} diff --git a/src/main/ssh/legacy-relay/legacy-relay-bridge-registry.ts b/src/main/ssh/legacy-relay/legacy-relay-bridge-registry.ts index a5149eacd10..e8812a3e99f 100644 --- a/src/main/ssh/legacy-relay/legacy-relay-bridge-registry.ts +++ b/src/main/ssh/legacy-relay/legacy-relay-bridge-registry.ts @@ -28,6 +28,7 @@ import { listedLeasesByRelayId } from './legacy-relay-bridge-leases' import { LegacyRelayPaneOwners } from './legacy-relay-pane-owners' +import { legacyRelayChannels } from './legacy-relay-bridge-connections' import { PtyExitProof, type PtyRetirementEvidence @@ -74,6 +75,7 @@ export function installLegacyRelayBridge(next: LegacyRelayBridgeHost | null): vo bridge.dispose() } bridges.clear() + legacyRelayChannels.dispose() heldPanes.clear() paneOwners.clear() paneStops.forEach((stops) => stops.dispose()) @@ -110,6 +112,7 @@ function bridgeFor(targetId: string): LegacyRelayTargetBridge | null { } return conn } + const channels = legacyRelayChannels.for(targetId, () => installed.connection(targetId)) const remoteCli = installed.remoteCli const cliServer = remoteCli ? createLegacyRelayCliServer({ @@ -135,7 +138,7 @@ function bridgeFor(targetId: string): LegacyRelayTargetBridge | null { cliServer, openTransport: async () => waitForSentinel( - await connection().exec(endpoint.connectCommand ?? '', { + await channels.open(endpoint.connectCommand ?? '', { wrapCommand: endpoint.wrapCommand }) ) @@ -217,6 +220,7 @@ export function stopLegacyRelayBridge(targetId: string): void { // Disposing publishes the list, now with the held panes instead of the bridge's. bridge.dispose() } + legacyRelayChannels.dispose(targetId) } /** The target and its leases are gone, so nothing it served is held any longer. */ diff --git a/tests/e2e/cross-version-wire/legacy-relay-bridge.unit.test.ts b/tests/e2e/cross-version-wire/legacy-relay-bridge.unit.test.ts index a4712001c8f..8ae1f7f8639 100644 --- a/tests/e2e/cross-version-wire/legacy-relay-bridge.unit.test.ts +++ b/tests/e2e/cross-version-wire/legacy-relay-bridge.unit.test.ts @@ -1,10 +1,11 @@ // Reach live v1.4.224 terminals through the old relay’s --connect after upgrade, without displacing an attached old client. // COMPAT(legacy-relay-bridge): revisit 2027-01-15, with src/main/ssh/legacy-relay/. import type { ChildProcess } from 'node:child_process' -import { mkdtemp, rm } from 'node:fs/promises' +import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import type { SshRemotePtyLease } from '../../../src/shared/ssh-types' +import type { SshRemotePtyLease, SshTarget } from '../../../src/shared/ssh-types' +import { SshConnection } from '../../../src/main/ssh/connection/ssh-connection' import { toAppSshPtyId } from '../../../src/shared/ssh-pty-id' import { LegacyRelayClient, @@ -12,10 +13,23 @@ import { } from '../../../src/main/ssh/legacy-relay/legacy-relay-client' import { LegacyRelayTargetBridge } from '../../../src/main/ssh/legacy-relay/legacy-relay-target-bridge' import { LegacyRelayPaneOwners } from '../../../src/main/ssh/legacy-relay/legacy-relay-pane-owners' -import type { LegacyRelayEndpoint } from '../../../src/main/ssh/legacy-relay/legacy-relay-endpoints' +import { + listLegacyRelayEndpoints, + posixEndpointExited, + type LegacyRelayEndpoint +} from '../../../src/main/ssh/legacy-relay/legacy-relay-endpoints' +import { createLegacyRelayBridgeConnections } from '../../../src/main/ssh/legacy-relay/legacy-relay-bridge-connections' +import { waitForSentinel } from '../../../src/main/ssh/legacy-relay/ssh-relay-sentinel' +import { relaySocketNameForInstanceId } from '../../../src/main/ssh/legacy-relay/ssh-relay-instance-id' +import { detectRemoteHostPlatform } from '../../../src/main/ssh/ssh-remote-platform-detection' +import { + startRealOpenSshFixture, + type RealOpenSshFixture +} from '../../../src/main/ssh/system-transport/real-openssh-sshd-fixture' import { getRemoteHostPlatform } from '../../../src/main/ssh/ssh-remote-platform' import { materializeReleaseCheckout } from './release-checkout' import { + buildReleasedRelayBundle, connectAsReleasedApp, installReleasedRelay, leaveShellInReleasedRelay, @@ -234,3 +248,132 @@ describe.skipIf(process.platform === 'win32')( ) } ) + +// One SSH host with many old relays, each still running a shell, must stay connectable. +// The bridge holds a relay's --connect channel per endpoint; sshd's MaxSessions (default 10) caps sessions per connection. +const MANY_RELAYS_TARGET_ID = 'ssh-many-relays' +// Budget for the release build plus one real SSH session and sign-in per relay. +const MANY_RELAYS_TIMEOUT_MS = 300_000 +// One per desktop update that left a relay running a shell; the reporter updated Orca many times. +const RELAY_COUNT = Number(process.env.ORCA_REPRO_RELAY_COUNT ?? 10) +// OpenSSH's default; lower values cover hosts stricter than the bridge's per-connection shard. +const MAX_SESSIONS = Number(process.env.ORCA_REPRO_MAX_SESSIONS ?? 10) + +describe.skipIf(process.platform === 'win32' || process.env.ORCA_REAL_OPENSSH_TEST !== '1')( + 'an SSH host where many old relays still run a shell', + () => { + let fixture: RealOpenSshFixture + const daemons: ChildProcess[] = [] + const leases: SshRemotePtyLease[] = [] + let conn: SshConnection + + beforeAll(async () => { + fixture = await startRealOpenSshFixture({ sshd: [`MaxSessions ${MAX_SESSIONS}`] }) + const checkout = await materializeReleaseCheckout(PREVIOUS_RELAY_REF) + const { bundleDir, version } = await buildReleasedRelayBundle(checkout) + const sockName = relaySocketNameForInstanceId(MANY_RELAYS_TARGET_ID) + for (let i = 0; i < RELAY_COUNT; i++) { + const v = `${version}-${i}` + const install: ReleasedRelayInstall = { + dir: join(fixture.remoteHome, '.orca-remote', `relay-${v}`), + version: v + } + await mkdir(install.dir, { recursive: true }) + await symlink(join(bundleDir, 'relay.js'), join(install.dir, 'relay.js')) + await writeFile(join(install.dir, '.version'), v) + const sockPath = join(install.dir, sockName) + daemons.push(await startReleasedRelayDaemon(install, sockPath, { SHELL: '/bin/sh' })) + const relayPtyId = await leaveShellInReleasedRelay( + () => openShellRelayTransport(releasedRelayConnectCommand(install, sockPath)), + `${v}-desktop` + ) + leases.push({ + targetId: MANY_RELAYS_TARGET_ID, + ptyId: relayPtyId, + tabId: `tab-${i}`, + leafId: `leaf-${i}`, + state: 'detached', + createdAt: 1, + updatedAt: 1 + }) + } + const sshdPort = fixture.buildOptions().resolvedConfig?.port ?? 0 + const target: SshTarget = { + id: MANY_RELAYS_TARGET_ID, + label: 'orca-many-relays-repro', + host: '127.0.0.1', + port: sshdPort, + username: process.env.USER ?? '', + identityFile: join(fixture.dir, 'clientkey'), + identitiesOnly: true, + source: 'manual' + } + conn = new SshConnection(target, { onStateChange: () => {} }) + await conn.connect() + }, MANY_RELAYS_TIMEOUT_MS) + + afterAll(async () => { + await conn?.disconnect().catch(() => {}) + for (const daemon of daemons) { + if (daemon.exitCode === null) { + daemon.kill('SIGKILL') + } + } + await fixture?.close() + }) + + // Bridging every relay opens a real SSH session per relay, plus sign-ins for side connections. + it( + 'still runs commands on the host while the bridge serves every old relay', + async () => { + // As legacy-relay-bridge-registry.ts opens relay channels. + const channels = createLegacyRelayBridgeConnections({ primary: () => conn }) + const bridge = new LegacyRelayTargetBridge({ + targetId: MANY_RELAYS_TARGET_ID, + leases: () => leases.filter((lease) => lease.state !== 'terminated'), + retireLease: () => {}, + listEndpoints: (signal) => listLegacyRelayEndpoints(conn, MANY_RELAYS_TARGET_ID, signal), + openClient: (endpoint, resume) => + LegacyRelayClient.open({ + clientInstanceId: `orca-legacy-relay-bridge:${MANY_RELAYS_TARGET_ID}`, + resume, + openTransport: async () => + waitForSentinel( + await channels.open(endpoint.connectCommand ?? '', { + wrapCommand: endpoint.wrapCommand + }) + ) + }), + endpointExited: (endpoint) => posixEndpointExited(conn, endpoint), + onPanesChanged: () => {}, + onData: () => {}, + onExit: () => {}, + onTerminalExited: () => {}, + owners: new LegacyRelayPaneOwners() + }) + try { + await bridge.refresh() + const panes = bridge.snapshot() + console.log( + `[repro] relays=${RELAY_COUNT} bridged panes=${panes.length} live=${ + panes.filter((pane) => pane.state === 'live').length + }` + ) + // The move's first step on connect: detect the host's platform over a fresh channel. + const probe = await detectRemoteHostPlatform(conn).then( + (host) => `ok:${host?.relayPlatform ?? 'null'}`, + (error: unknown) => `error:${error instanceof Error ? error.message : String(error)}` + ) + console.log(`[repro] platform probe -> ${probe}`) + expect(probe).toMatch(/^ok:/) + // Every old relay's shell is still served, none dropped to make room. + expect(panes.filter((pane) => pane.state === 'live')).toHaveLength(RELAY_COUNT) + } finally { + bridge.dispose() + channels.dispose() + } + }, + MANY_RELAYS_TIMEOUT_MS + ) + } +)