From bf13d318de38580dae0c595accca4e87129caa3f Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Sun, 10 May 2026 21:39:59 -0400 Subject: [PATCH] fix(ssh): bound relay-lost reconnect with exponential backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the relay exec channel keeps dying (e.g. a remote-side bug closes every fresh --connect channel right after handshake, or a stale bridge keeps being replaced), the unguarded _onRelayLost handler reconnects as fast as the network allows — spawning relay deploy attempts in a tight loop until the user force-quits. Each iteration spawns a fresh ssh2 exec channel, hammers sshd's MaxSessions counter, and floods the renderer with state churn. Add per-target exponential backoff (500ms → 15s, capped at 6 attempts) so the loop terminates instead of running forever. After the cap the session goes to 'error' state with a 'Relay channel kept dropping. Please reconnect.' message — visible in the renderer instead of an invisible failure where typing in remote terminals just stops working. Successful 'ready' resets the attempt counter only if the session stabilized for >= 5s; faster flaps preserve the counter so a flaky remote backs off rather than retrying indefinitely on every brief ready→lost cycle. Backoff state is cleared on explicit disconnect, on session replacement during reconnect, and on connect failures, so a real reconnect attempt after backoff exhaustion always starts from zero. Co-authored-by: Orca --- src/main/ipc/ssh.ts | 112 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/src/main/ipc/ssh.ts b/src/main/ipc/ssh.ts index edb556697b1..130204744de 100644 --- a/src/main/ipc/ssh.ts +++ b/src/main/ipc/ssh.ts @@ -41,6 +41,39 @@ const connectInFlight = new Map>() // avoids that visual glitch. const testingTargets = new Set() +// Why: when a relay channel keeps dying (e.g. a stale --connect bridge keeps +// being replaced, a remote-side bug closes the channel right after handshake, +// or a mismatched relay binary refuses every handshake), the unguarded +// _onRelayLost handler reconnects as fast as the network allows, hammering +// both the local main process and the remote sshd in a tight loop. Track +// per-target reconnect attempts and apply exponential backoff so the loop +// terminates with a recoverable error instead of running forever. Successful +// `ready` resets the attempt counter for the next genuine drop. +type RelayLostBackoffState = { + attempts: number + lastAttemptStartedAt: number + pendingTimer: ReturnType | null +} +const relayLostBackoff = new Map() +const RELAY_LOST_MAX_ATTEMPTS = 6 +const RELAY_LOST_BASE_DELAY_MS = 500 +const RELAY_LOST_MAX_DELAY_MS = 15_000 +// Why: if a fresh reconnect's mux dies within this window, the new session +// never stabilized (a flap, not a real recovery). Without this clamp the +// attempt counter would be reset prematurely by a mux that "reached ready" +// only on paper. 5 seconds covers normal post-deploy provider registration +// and PTY reattach without being so generous that a real long-lived session +// looks like a flap. +const RELAY_LOST_STABILIZED_MS = 5_000 + +function clearRelayLostBackoff(targetId: string): void { + const state = relayLostBackoff.get(targetId) + if (state?.pendingTimer) { + clearTimeout(state.pendingTimer) + } + relayLostBackoff.delete(targetId) +} + function broadcastPortForwards(getMainWindow: () => BrowserWindow | null, targetId: string): void { const win = getMainWindow() if (!win || win.isDestroyed()) { @@ -291,6 +324,7 @@ export function registerSshHandlers( await portForwardManager!.removeAllForwards(targetId) existingSession.dispose() activeSessions.delete(targetId) + clearRelayLostBackoff(targetId) } // Why: create the session early so onStateChange sees it in 'deploying' @@ -322,6 +356,7 @@ export function registerSshHandlers( // needs a passphrase. credentialRequestedForTarget.delete(targetId) activeSessions.delete(targetId) + clearRelayLostBackoff(targetId) const win = getMainWindow() if (win && !win.isDestroyed()) { win.webContents.send('ssh:state-changed', { @@ -359,16 +394,83 @@ export function registerSshHandlers( return } const c = connectionManager?.getConnection(tid) - const t = sshStore?.getTarget(tid) - if (c) { - void s.reconnect(c, t?.relayGracePeriodSeconds) + if (!c) { + return } + const t = sshStore?.getTarget(tid) + + // Why: bounded exponential backoff. Without this, a remote-side bug + // that closes every fresh --connect channel turns into an infinite + // tight loop spawning relay deploys until the user force-quits. + const state = relayLostBackoff.get(tid) ?? { + attempts: 0, + lastAttemptStartedAt: 0, + pendingTimer: null + } + if (state.pendingTimer) { + // A retry is already scheduled — coalesce this burst. + return + } + if (state.attempts >= RELAY_LOST_MAX_ATTEMPTS) { + console.warn( + `[ssh] Relay channel for ${tid} kept dying across ${state.attempts} attempts; giving up. User must reconnect manually.` + ) + relayLostBackoff.delete(tid) + // Why: surface the failure so the renderer can prompt the user. + // A still-live SSH connection with a dead relay is otherwise an + // invisible failure — typing in remote terminals just stops working. + const win = getMainWindow() + if (win && !win.isDestroyed()) { + win.webContents.send('ssh:state-changed', { + targetId: tid, + state: { + targetId: tid, + status: 'error', + error: 'Relay channel kept dropping. Please reconnect.', + reconnectAttempt: 0 + } + }) + } + return + } + const delay = Math.min( + RELAY_LOST_BASE_DELAY_MS * 2 ** state.attempts, + RELAY_LOST_MAX_DELAY_MS + ) + state.attempts += 1 + state.pendingTimer = setTimeout(() => { + state.pendingTimer = null + state.lastAttemptStartedAt = Date.now() + relayLostBackoff.set(tid, state) + const liveConn = connectionManager?.getConnection(tid) + if (!liveConn || !activeSessions.has(tid)) { + return + } + void s.reconnect(liveConn, t?.relayGracePeriodSeconds) + }, delay) + relayLostBackoff.set(tid, state) + console.warn( + `[ssh] Relay channel for ${tid} lost; reconnect attempt ${state.attempts}/${RELAY_LOST_MAX_ATTEMPTS} in ${delay}ms` + ) }) // Why: fires after both establish() and reconnect() reach 'ready'. // Re-creates persisted port forwards so they survive app restarts - // and network blips without manual re-configuration. + // and network blips without manual re-configuration. We also clear + // the relay-lost backoff state so a subsequent genuine drop starts + // from a fresh attempt counter — but only if the session had a chance + // to stabilize, otherwise rapid `ready → lost → ready → lost` flaps + // would silently keep retrying forever. session.setOnReady((tid) => { + const state = relayLostBackoff.get(tid) + if (state) { + const stabilized = + state.lastAttemptStartedAt === 0 || + Date.now() - state.lastAttemptStartedAt >= RELAY_LOST_STABILIZED_MS + if (stabilized) { + relayLostBackoff.delete(tid) + } + } void restorePortForwards(tid, getMainWindow) }) @@ -393,6 +495,7 @@ export function registerSshHandlers( } catch (err) { // Relay deployment failed — disconnect SSH activeSessions.delete(targetId) + clearRelayLostBackoff(targetId) await connectionManager!.disconnect(targetId) throw err } @@ -417,6 +520,7 @@ export function registerSshHandlers( await portForwardManager!.removeAllForwards(args.targetId) session.dispose() activeSessions.delete(args.targetId) + clearRelayLostBackoff(args.targetId) } await connectionManager!.disconnect(args.targetId) })