fix(ssh): bound relay-lost reconnect with exponential backoff

When the relay exec channel keeps dying (e.g. a remote-side bug closes
every fresh --connect channel right after handshake, or a stale bridge
keeps being replaced), the unguarded _onRelayLost handler reconnects as
fast as the network allows — spawning relay deploy attempts in a tight
loop until the user force-quits. Each iteration spawns a fresh ssh2 exec
channel, hammers sshd's MaxSessions counter, and floods the renderer
with state churn.

Add per-target exponential backoff (500ms → 15s, capped at 6 attempts)
so the loop terminates instead of running forever. After the cap the
session goes to 'error' state with a 'Relay channel kept dropping.
Please reconnect.' message — visible in the renderer instead of an
invisible failure where typing in remote terminals just stops working.

Successful 'ready' resets the attempt counter only if the session
stabilized for >= 5s; faster flaps preserve the counter so a flaky
remote backs off rather than retrying indefinitely on every brief
ready→lost cycle.

Backoff state is cleared on explicit disconnect, on session replacement
during reconnect, and on connect failures, so a real reconnect attempt
after backoff exhaustion always starts from zero.

Co-authored-by: Orca <help@stably.ai>

src/main/ipc/ssh.ts

@@ -41,6 +41,39 @@ const connectInFlight = new Map<string, Promise<SshConnectionState>>()
 // avoids that visual glitch.
 const testingTargets = new Set<string>()
 
+// Why: when a relay channel keeps dying (e.g. a stale --connect bridge keeps
+// being replaced, a remote-side bug closes the channel right after handshake,
+// or a mismatched relay binary refuses every handshake), the unguarded
+// _onRelayLost handler reconnects as fast as the network allows, hammering
+// both the local main process and the remote sshd in a tight loop. Track
+// per-target reconnect attempts and apply exponential backoff so the loop
+// terminates with a recoverable error instead of running forever. Successful
+// `ready` resets the attempt counter for the next genuine drop.
+type RelayLostBackoffState = {
+  attempts: number
+  lastAttemptStartedAt: number
+  pendingTimer: ReturnType<typeof setTimeout> | null
+}
+const relayLostBackoff = new Map<string, RelayLostBackoffState>()
+const RELAY_LOST_MAX_ATTEMPTS = 6
+const RELAY_LOST_BASE_DELAY_MS = 500
+const RELAY_LOST_MAX_DELAY_MS = 15_000
+// Why: if a fresh reconnect's mux dies within this window, the new session
+// never stabilized (a flap, not a real recovery). Without this clamp the
+// attempt counter would be reset prematurely by a mux that "reached ready"
+// only on paper. 5 seconds covers normal post-deploy provider registration
+// and PTY reattach without being so generous that a real long-lived session
+// looks like a flap.
+const RELAY_LOST_STABILIZED_MS = 5_000
+
+function clearRelayLostBackoff(targetId: string): void {
+  const state = relayLostBackoff.get(targetId)
+  if (state?.pendingTimer) {
+    clearTimeout(state.pendingTimer)
+  }
+  relayLostBackoff.delete(targetId)
+}
+
 function broadcastPortForwards(getMainWindow: () => BrowserWindow | null, targetId: string): void {
   const win = getMainWindow()
   if (!win || win.isDestroyed()) {
@@ -291,6 +324,7 @@ export function registerSshHandlers(
       await portForwardManager!.removeAllForwards(targetId)
       existingSession.dispose()
       activeSessions.delete(targetId)
+      clearRelayLostBackoff(targetId)
     }
 
     // Why: create the session early so onStateChange sees it in 'deploying'
@@ -322,6 +356,7 @@ export function registerSshHandlers(
       // needs a passphrase.
       credentialRequestedForTarget.delete(targetId)
       activeSessions.delete(targetId)
+      clearRelayLostBackoff(targetId)
       const win = getMainWindow()
       if (win && !win.isDestroyed()) {
         win.webContents.send('ssh:state-changed', {
@@ -359,16 +394,83 @@ export function registerSshHandlers(
           return
         }
         const c = connectionManager?.getConnection(tid)
-        const t = sshStore?.getTarget(tid)
-        if (c) {
-          void s.reconnect(c, t?.relayGracePeriodSeconds)
+        if (!c) {
+          return
         }
+        const t = sshStore?.getTarget(tid)
+
+        // Why: bounded exponential backoff. Without this, a remote-side bug
+        // that closes every fresh --connect channel turns into an infinite
+        // tight loop spawning relay deploys until the user force-quits.
+        const state = relayLostBackoff.get(tid) ?? {
+          attempts: 0,
+          lastAttemptStartedAt: 0,
+          pendingTimer: null
+        }
+        if (state.pendingTimer) {
+          // A retry is already scheduled — coalesce this burst.
+          return
+        }
+        if (state.attempts >= RELAY_LOST_MAX_ATTEMPTS) {
+          console.warn(
+            `[ssh] Relay channel for ${tid} kept dying across ${state.attempts} attempts; giving up. User must reconnect manually.`
+          )
+          relayLostBackoff.delete(tid)
+          // Why: surface the failure so the renderer can prompt the user.
+          // A still-live SSH connection with a dead relay is otherwise an
+          // invisible failure — typing in remote terminals just stops working.
+          const win = getMainWindow()
+          if (win && !win.isDestroyed()) {
+            win.webContents.send('ssh:state-changed', {
+              targetId: tid,
+              state: {
+                targetId: tid,
+                status: 'error',
+                error: 'Relay channel kept dropping. Please reconnect.',
+                reconnectAttempt: 0
+              }
+            })
+          }
+          return
+        }
+        const delay = Math.min(
+          RELAY_LOST_BASE_DELAY_MS * 2 ** state.attempts,
+          RELAY_LOST_MAX_DELAY_MS
+        )
+        state.attempts += 1
+        state.pendingTimer = setTimeout(() => {
+          state.pendingTimer = null
+          state.lastAttemptStartedAt = Date.now()
+          relayLostBackoff.set(tid, state)
+          const liveConn = connectionManager?.getConnection(tid)
+          if (!liveConn || !activeSessions.has(tid)) {
+            return
+          }
+          void s.reconnect(liveConn, t?.relayGracePeriodSeconds)
+        }, delay)
+        relayLostBackoff.set(tid, state)
+        console.warn(
+          `[ssh] Relay channel for ${tid} lost; reconnect attempt ${state.attempts}/${RELAY_LOST_MAX_ATTEMPTS} in ${delay}ms`
+        )
       })
 
       // Why: fires after both establish() and reconnect() reach 'ready'.
       // Re-creates persisted port forwards so they survive app restarts
-      // and network blips without manual re-configuration.
+      // and network blips without manual re-configuration. We also clear
+      // the relay-lost backoff state so a subsequent genuine drop starts
+      // from a fresh attempt counter — but only if the session had a chance
+      // to stabilize, otherwise rapid `ready → lost → ready → lost` flaps
+      // would silently keep retrying forever.
       session.setOnReady((tid) => {
+        const state = relayLostBackoff.get(tid)
+        if (state) {
+          const stabilized =
+            state.lastAttemptStartedAt === 0 ||
+            Date.now() - state.lastAttemptStartedAt >= RELAY_LOST_STABILIZED_MS
+          if (stabilized) {
+            relayLostBackoff.delete(tid)
+          }
+        }
         void restorePortForwards(tid, getMainWindow)
       })
 
@@ -393,6 +495,7 @@ export function registerSshHandlers(
     } catch (err) {
       // Relay deployment failed — disconnect SSH
       activeSessions.delete(targetId)
+      clearRelayLostBackoff(targetId)
       await connectionManager!.disconnect(targetId)
       throw err
     }
@@ -417,6 +520,7 @@ export function registerSshHandlers(
       await portForwardManager!.removeAllForwards(args.targetId)
       session.dispose()
       activeSessions.delete(args.targetId)
+      clearRelayLostBackoff(args.targetId)
     }
     await connectionManager!.disconnect(args.targetId)
   })