diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 77cdbd710d6..2a4b489894f 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -27,7 +27,6 @@ inline src/main/index.ts inline src/main/ipc/filesystem-watcher.ts inline src/main/ipc/filesystem.ts inline src/main/ipc/repos.ts -inline src/main/ipc/ssh.ts inline src/main/ipc/worktree-remote.ts inline src/main/keybindings/keybinding-file.ts inline src/main/linear/issues.ts diff --git a/src/main/ephemeral-vm-runtime-ssh.ts b/src/main/ephemeral-vm-runtime-ssh.ts index 07c579860f9..770a43cbf8f 100644 --- a/src/main/ephemeral-vm-runtime-ssh.ts +++ b/src/main/ephemeral-vm-runtime-ssh.ts @@ -1,11 +1,10 @@ import { getSshFilesystemProvider } from './providers/ssh-filesystem-dispatch' import { getSshGitProvider } from './providers/ssh-git-dispatch' +import { connectRegisteredSshTarget, getSshConnectionStore } from './ipc/ssh' import { - connectRegisteredSshTarget, disconnectRegisteredSshTarget, - getSshConnectionStore, removeRegisteredSshTarget -} from './ipc/ssh' +} from './ipc/ssh-session-teardown' import type { EphemeralVmRecipeConnection } from '../shared/ephemeral-vm-recipes' import type { SshTarget } from '../shared/ssh-types' diff --git a/src/main/index.ts b/src/main/index.ts index 0a47efb9335..067711e3247 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -318,7 +318,7 @@ import { normalizeComputerAwakeMode } from '../shared/computer-awake-mode' import { registerSystemResumeBroadcast } from './system-resume-broadcast' import { settleTeardownWithinDeadline } from './quit-teardown-deadline' import { quitTeardownStartGate } from './quit-teardown-start-gate' -import { beginSshShutdown } from './ipc/ssh' +import { beginSshShutdown } from './ipc/ssh-shutdown-drain' import { PluginService } from './plugins/plugin-service' import { PluginKillListService } from './plugins/plugin-kill-list-service' import { getPluginsDataDir } from './plugins/plugin-discovery' diff --git a/src/main/ipc/ssh-active-relay-sessions.ts b/src/main/ipc/ssh-active-relay-sessions.ts new file mode 100644 index 00000000000..74b7f385806 --- /dev/null +++ b/src/main/ipc/ssh-active-relay-sessions.ts @@ -0,0 +1,11 @@ +import type { SshRelaySession } from '../ssh/ssh-relay-session' +import { setSshActiveMultiplexerResolver } from '../ssh/ssh-target-registry' + +// One session per SSH target owns the whole relay lifecycle (mux, providers, abort controller, state machine). +export const activeSessions = new Map() + +// Why at module scope: this resolver is pure state lookup with no handler lifecycle, so +// installing it on import keeps it correct even before registerSshHandlers runs. +setSshActiveMultiplexerResolver( + (connectionId) => activeSessions.get(connectionId)?.getMux() ?? undefined +) diff --git a/src/main/ipc/ssh-advertised-url-refresh.ts b/src/main/ipc/ssh-advertised-url-refresh.ts new file mode 100644 index 00000000000..927b7017ba3 --- /dev/null +++ b/src/main/ipc/ssh-advertised-url-refresh.ts @@ -0,0 +1,37 @@ +import type { BrowserWindow } from 'electron' +import { advertisedUrlWatcher } from '../ports/advertised-url-watcher' +import { getConnectionIdsForWorktree } from '../ports/ssh-advertised-url-enrichment' +import { activeSessions } from './ssh-active-relay-sessions' +import { persistedStore } from './ssh-ipc-context' +import { broadcastDetectedPorts, broadcastPortForwards } from './ssh-renderer-broadcast' + +let advertisedUrlWatcherUnsubscribe: (() => void) | null = null + +export function registerAdvertisedUrlRefresh(getMainWindow: () => BrowserWindow | null): void { + advertisedUrlWatcherUnsubscribe?.() + // Why: SSH port scans only emit on raw host/port/PID changes, but a terminal can print the advertised URL later, so the watcher must also refresh the renderer. + advertisedUrlWatcherUnsubscribe = advertisedUrlWatcher.onDidChange(({ worktreeId }) => { + if (!persistedStore) { + return + } + for (const targetId of getConnectionIdsForWorktree(persistedStore, worktreeId)) { + const session = activeSessions.get(targetId) + if (!session) { + continue + } + const scanner = session.getPortScanner() + if (scanner) { + // Why: watcher changes can arrive before the next SSH scan refreshes listener PIDs, so don't validate PIDs against cached scanner rows. + broadcastDetectedPorts(getMainWindow, targetId, scanner.getDetectedPorts(targetId), { + validatePid: false + }) + } + broadcastPortForwards(getMainWindow, targetId) + } + }) +} + +export function unregisterAdvertisedUrlRefresh(): void { + advertisedUrlWatcherUnsubscribe?.() + advertisedUrlWatcherUnsubscribe = null +} diff --git a/src/main/ipc/ssh-app-shutdown.test.ts b/src/main/ipc/ssh-app-shutdown.test.ts index 47c819c054a..b587d666a6c 100644 --- a/src/main/ipc/ssh-app-shutdown.test.ts +++ b/src/main/ipc/ssh-app-shutdown.test.ts @@ -22,12 +22,12 @@ vi.mock('../providers/ssh-git-dispatch', () => mocks.sshGitDispatch) vi.mock('../ssh/ssh-port-forward', () => mocks.sshPortForward) vi.mock('../ssh/ssh-port-scanner', () => mocks.sshPortScanner) +import { getActiveMultiplexer } from './ssh' import { beginSshShutdown, SSH_SHUTDOWN_BUDGET_MS, - getActiveMultiplexer, type SshShutdownResult -} from './ssh' +} from './ssh-shutdown-drain' import type { SshTarget } from '../../shared/ssh-types' import { quitTeardownStartGate } from '../quit-teardown-start-gate' import { createSshIpcHarness } from './ssh-ipc-test-harness' diff --git a/src/main/ipc/ssh-connect-attempt-registry.ts b/src/main/ipc/ssh-connect-attempt-registry.ts new file mode 100644 index 00000000000..5a1e3632982 --- /dev/null +++ b/src/main/ipc/ssh-connect-attempt-registry.ts @@ -0,0 +1,46 @@ +import type { DirectSshAuthority, SshConnectionState } from '../../shared/ssh-types' +import { quitTeardownStartGate } from '../quit-teardown-start-gate' +import { + isCurrentSshProviderAuthority, + rotateSshProviderAuthority +} from '../ssh/ssh-provider-authority' + +// Why: keep this outside registerSshHandlers so a BrowserWindow recreation mid-connect doesn't split credential tracking. +export const credentialRequestedForTarget = new Set() + +// Why: tabs must share one connect, while a disconnect must invalidate that +// attempt so its late continuation cannot clobber a replacement. +export type ConnectAttempt = { + authority: DirectSshAuthority + promise: Promise +} + +export const connectInFlight = new Map() +export const pendingTransportReconnects = new Set() + +// Why the quit gate rather than a local latch: "the committed quit has begun" already has an owner, +// and a private copy could be set by something that is not actually quitting — leaving SSH connects +// refused for the rest of the process lifetime. +export function assertSshConnectsNotFenced(): void { + if (quitTeardownStartGate.hasStarted()) { + throw new Error('SSH connects are closed for app shutdown') + } +} + +export function invalidateConnectAttempt(targetId: string): void { + rotateSshProviderAuthority(targetId) + pendingTransportReconnects.delete(targetId) + connectInFlight.delete(targetId) + credentialRequestedForTarget.delete(targetId) +} + +export function isCurrentConnectAttempt(targetId: string, authority: DirectSshAuthority): boolean { + return authority.targetId === targetId && isCurrentSshProviderAuthority(authority) +} + +// Why: publish reset's teardown/force-stop/disconnect lifecycle so new connects and duplicate resets can't race it. +export const resetRelayInFlight = new Map>() + +// Why: ssh:testConnection connects then disconnects; suppressing broadcasts during the test avoids worktree cards flashing connected → disconnected. +export const testingTargets = new Set() +export const testConnectionProbes = new Set>() diff --git a/src/main/ipc/ssh-connect-flow.ts b/src/main/ipc/ssh-connect-flow.ts new file mode 100644 index 00000000000..9a0b0c25f05 --- /dev/null +++ b/src/main/ipc/ssh-connect-flow.ts @@ -0,0 +1,264 @@ +import { appendFileSync } from 'node:fs' +import type { SshConnection } from '../ssh/ssh-connection' +import { SshRelaySession } from '../ssh/ssh-relay-session' +import type { SshConnectionState, SshConnectionStatus } from '../../shared/ssh-types' +import { createCancelledConnectAttemptError } from '../ssh/ssh-connect-attempt-cancellation' +import { isAuthError } from '../ssh/ssh-connection-utils' +import { + getSshProviderAuthority, + isCurrentSshProviderAuthority, + rotateSshProviderAuthority +} from '../ssh/ssh-provider-authority' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { activeSessions } from './ssh-active-relay-sessions' +import { + assertSshConnectsNotFenced, + connectInFlight, + credentialRequestedForTarget, + isCurrentConnectAttempt, + pendingTransportReconnects, + resetRelayInFlight +} from './ssh-connect-attempt-registry' +import { + handleSshConnectionStateChange, + relayGracePeriodForTarget +} from './ssh-connection-state-callbacks' +import { + connectionManager, + currentRuntime, + getCurrentMainWindow, + persistedStore, + portForwardManager +} from './ssh-ipc-context' +import { clearRelayLostBackoff, relayLostBackoff } from './ssh-relay-lost-backoff' +import { configureRelaySessionCallbacks } from './ssh-relay-session-callbacks' +import { + broadcastDetectedPortsFromCurrentWindow, + broadcastSshState, + clearRelayStateOverride, + getPublicSshState, + relayStateOverrides +} from './ssh-renderer-broadcast' +import { abandonCancelledConnectAttempt, abandonFailedSshSession } from './ssh-session-teardown' +import { awaitTargetLifecycle } from './ssh-target-lifecycle-queue' + +export async function connectTarget(targetId: string): Promise { + const e2eProbePath = process.env.ORCA_E2E_FORBID_LOCAL_SSH_CONNECT_PROBE + if (e2eProbePath) { + appendFileSync(e2eProbePath, `${JSON.stringify(targetId)}\n`) + throw new Error('e2e_forbidden_local_ssh_connect') + } + // Why: fence callers that entered before a same-turn disconnect/reset but resume after its cleanup. + const admissionAuthority = getSshProviderAuthority(targetId) + await awaitTargetLifecycle(targetId) + const reset = resetRelayInFlight.get(targetId) + if (reset) { + await reset + } + + // Why: serialize concurrent ssh:connect for the same target; interleaved connects otherwise leak the first session. + const existing = connectInFlight.get(targetId) + let replacePendingTransport = false + if (existing) { + if (isCurrentConnectAttempt(targetId, existing.authority)) { + return existing.promise + } + } + if (!isCurrentConnectAttempt(targetId, admissionAuthority)) { + throw createCancelledConnectAttemptError() + } + const observedAuthority = admissionAuthority + if (existing) { + if (connectInFlight.get(targetId) === existing) { + connectInFlight.delete(targetId) + replacePendingTransport = true + } + } + if (!isCurrentSshProviderAuthority(observedAuthority)) { + throw createCancelledConnectAttemptError() + } + // Why: the shutdown drain fences and snapshots synchronously, so a connect either registers in + // connectInFlight below (and gets joined) or fails here — it can never slip between the two. + assertSshConnectsNotFenced() + + pendingTransportReconnects.delete(targetId) + const promise = doConnect(targetId, replacePendingTransport) + const attempt = { authority: getSshProviderAuthority(targetId), promise } + connectInFlight.set(targetId, attempt) + try { + return await promise + } finally { + if (connectInFlight.get(targetId) === attempt) { + connectInFlight.delete(targetId) + } + } +} + +async function doConnect( + targetId: string, + replacePendingTransport = false +): Promise { + const target = getSshTargetRegistryStore()!.getTarget(targetId) + if (!target) { + throw new Error(`SSH target "${targetId}" not found`) + } + + const existingSession = activeSessions.get(targetId) + const existingState = connectionManager!.getState(targetId) + const existingMux = existingSession?.getMux() + if ( + existingSession?.getState() === 'ready' && + existingState?.status === 'connected' && + connectionManager!.getConnection(targetId) && + existingMux && + !existingMux.isDisposed() && + !relayStateOverrides.has(targetId) && + !relayLostBackoff.has(targetId) + ) { + // Why: BrowserWindow reactivation re-fires ssh:connect for already-live targets; treat as a refresh instead of tearing down the relay and its forwards. + broadcastSshState(getCurrentMainWindow, targetId, existingState) + return getPublicSshState(targetId)! + } + + const authority = rotateSshProviderAuthority(targetId) + clearRelayStateOverride(targetId) + const pendingTransportDisconnect = replacePendingTransport + ? connectionManager!.disconnect(targetId).then( + () => ({ ok: true }) as const, + (error: unknown) => ({ ok: false, error }) as const + ) + : null + let conn + // Why: tear down any existing session first to avoid leaking its multiplexer, providers, and timers (double-connect / reconnect-after-error). + if (existingSession) { + // Why: await port teardown before disposing, else the new session's restorePortForwards can hit EADDRINUSE on not-yet-released ports. + await portForwardManager!.removeAllForwards(targetId) + if (!isCurrentConnectAttempt(targetId, authority)) { + throw createCancelledConnectAttemptError() + } + try { + await existingSession.detachAndPersist() + } finally { + // Why finally: detachAndPersist runs its in-memory half synchronously, so the session is + // dead even when the lease write rejects — keeping it in activeSessions would strand every + // later connect on the same dead session. Why still after the await, not before it: the + // write has settled by now, so it can no longer clobber the replacement's 'attached' write. + if (activeSessions.get(targetId) === existingSession) { + activeSessions.delete(targetId) + clearRelayLostBackoff(targetId) + clearRelayStateOverride(targetId) + } + } + } + + if (pendingTransportDisconnect) { + const disconnectResult = await pendingTransportDisconnect + if (!disconnectResult.ok) { + throw disconnectResult.error + } + if (!isCurrentConnectAttempt(targetId, authority)) { + throw createCancelledConnectAttemptError() + } + } + + // Why here and not only at entry: this is the publication point, and it is the last statement + // before the transport opens. Checking it in the same synchronous block as activeSessions.set + // means a connect either registers before the shutdown drain snapshots, or registers never and + // owns nothing to clean up. + assertSshConnectsNotFenced() + // Why: create the session early so onStateChange sees it in 'deploying' and skips reconnect logic. + const session = new SshRelaySession( + targetId, + getCurrentMainWindow, + persistedStore!, + portForwardManager!, + currentRuntime, + broadcastDetectedPortsFromCurrentWindow + ) + configureRelaySessionCallbacks(session) + activeSessions.set(targetId, session) + const ownsSession = (): boolean => + isCurrentConnectAttempt(targetId, authority) && activeSessions.get(targetId) === session + + // Why captured here and not with existingState: connect() reuses an already-connected transport, + // and only a transport this attempt opened is this attempt's to close when it loses the race. + const priorConnection = connectionManager!.getConnection(targetId) + const mintedConnection = (): SshConnection | null => + conn && conn !== priorConnection ? conn : null + + try { + conn = await connectionManager!.connect(target) + if (!ownsSession()) { + throw createCancelledConnectAttemptError() + } + } catch (err) { + // Why: connect()'s internal state may not have reached the renderer; broadcast explicitly so the UI leaves 'connecting'. + const errObj = err instanceof Error ? err : new Error(String(err)) + const status: SshConnectionStatus = isAuthError(errObj) ? 'auth-failed' : 'error' + if (!ownsSession()) { + await abandonCancelledConnectAttempt(targetId, session, mintedConnection()) + throw createCancelledConnectAttemptError() + } + // Why: clear this failed connect's flag so a later non-prompting connect isn't deferred. + credentialRequestedForTarget.delete(targetId) + await abandonFailedSshSession(targetId, session) + clearRelayLostBackoff(targetId) + clearRelayStateOverride(targetId) + broadcastSshState(getCurrentMainWindow, targetId, { + targetId, + status, + error: errObj.message, + reconnectAttempt: 0 + }) + throw err + } + + try { + handleSshConnectionStateChange(targetId, { + targetId, + status: 'deploying-relay', + error: null, + reconnectAttempt: 0 + }) + + await session.establish(conn, relayGracePeriodForTarget(target)) + if (!ownsSession()) { + throw createCancelledConnectAttemptError() + } + + // Why: we manually pushed `deploying-relay`, so send `connected` straight to the renderer — routing through onStateChange would trigger reconnect logic. + clearRelayStateOverride(targetId) + broadcastSshState(getCurrentMainWindow, targetId, { + targetId, + status: 'connected', + error: null, + reconnectAttempt: 0, + supportsFolderDownload: conn.usesSystemSshTransport?.() !== true + }) + } catch (err) { + if (!ownsSession()) { + await abandonCancelledConnectAttempt(targetId, session, mintedConnection()) + throw createCancelledConnectAttemptError() + } + await abandonFailedSshSession(targetId, session) + clearRelayLostBackoff(targetId) + try { + await connectionManager!.disconnect(targetId) + } catch (disconnectError) { + // Why: the establish failure is the actionable error; a teardown throw must not replace it. + console.warn( + `[ssh] Failed to disconnect transport after failed establish for ${targetId}: ${disconnectError instanceof Error ? disconnectError.message : String(disconnectError)}` + ) + } + throw err + } + + // Why: persist whether this connect needed a credential so startup can partition targets into eager vs deferred without re-probing keys. + const requiredPassphrase = credentialRequestedForTarget.has(targetId) + credentialRequestedForTarget.delete(targetId) + getSshTargetRegistryStore()!.updateTarget(targetId, { + lastRequiredPassphrase: requiredPassphrase + }) + + return getPublicSshState(targetId)! +} diff --git a/src/main/ipc/ssh-connection-handlers.ts b/src/main/ipc/ssh-connection-handlers.ts new file mode 100644 index 00000000000..8c3bf4bbc6d --- /dev/null +++ b/src/main/ipc/ssh-connection-handlers.ts @@ -0,0 +1,262 @@ +import { ipcMain } from 'electron' +import type { SshTarget } from '../../shared/ssh-types' +import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' +import { isSshPtyNotFoundError } from '../providers/ssh-pty-errors' +import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' +import { rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' +import { forceStopRelayForTarget } from '../ssh/ssh-relay-reset' +import { setSshTargetRegistryHandlers, getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { + clearProviderPtyState, + deletePtyOwnership, + getPtyIdsForConnection, + getSshPtyProvider +} from './pty' +import { activeSessions } from './ssh-active-relay-sessions' +import { + assertSshConnectsNotFenced, + connectInFlight, + credentialRequestedForTarget, + invalidateConnectAttempt, + resetRelayInFlight, + testConnectionProbes, + testingTargets +} from './ssh-connect-attempt-registry' +import { connectTarget } from './ssh-connect-flow' +import { connectionManager, persistedStore } from './ssh-ipc-context' +import { getPublicSshState } from './ssh-renderer-broadcast' +import { + disconnectRegisteredSshTarget, + teardownActiveSshSession, + teardownSshTargetTransport +} from './ssh-session-teardown' +import { runTargetLifecycle } from './ssh-target-lifecycle-queue' + +async function doResetRelay(targetId: string, target: SshTarget): Promise { + const inFlightConnect = connectInFlight.get(targetId) + if (inFlightConnect) { + try { + // Why: resetting activeSessions mid-deploy would dispose the session doConnect will use. + await inFlightConnect.promise + } catch { + // The reset can still recover a stale remote relay after a failed connect. + } + } + + rotateSshProviderAuthority(targetId) + const session = activeSessions.get(targetId) + if (session) { + // Why: detach() not dispose() — reset has its own stale-lease semantics below that dispose()'s clean-termination recording would hide. + await teardownActiveSshSession(targetId, (capturedSession) => + capturedSession.detachAndPersist() + ) + } + + const existingConn = connectionManager!.getConnection(targetId) + let conn = existingConn + if (!conn) { + // Why re-check: admission fenced this reset before it parked on the in-flight connect, so shutdown + // may have started (and drained) while we waited — opening a transport now would outlive the drain. + assertSshConnectsNotFenced() + conn = await connectionManager!.connect(target) + } + try { + await forceStopRelayForTarget(conn, targetId) + } finally { + const ptyIds = new Set(getPtyIdsForConnection(targetId)) + for (const lease of persistedStore!.getSshRemotePtyLeases(targetId)) { + if (lease.state !== 'terminated' && lease.state !== 'expired') { + ptyIds.add(lease.ptyId) + persistedStore!.markSshRemotePtyLease(targetId, lease.ptyId, 'expired') + } + } + // Why: reset force-kills the remote relay, so every local PTY handle it owned is stale even if the reset command failed after SIGTERM. + for (const ptyId of ptyIds) { + const appPtyId = toAppSshPtyId(targetId, ptyId) + clearProviderPtyState(appPtyId) + deletePtyOwnership(appPtyId) + } + // Why: reset's connect() may trip onCredentialRequest; clear so a later non-prompting doConnect doesn't persist lastRequiredPassphrase=true. + credentialRequestedForTarget.delete(targetId) + await connectionManager!.disconnect(targetId) + } +} + +export function registerSshConnectionHandlers(): void { + setSshTargetRegistryHandlers({ + connect: connectTarget, + getState: (targetId: string) => getPublicSshState(targetId) + }) + + ipcMain.handle('ssh:connect', async (_event, args: { targetId: string }) => { + return connectTarget(args.targetId) + }) + + ipcMain.handle('ssh:disconnect', async (_event, args: { targetId: string }) => { + await disconnectRegisteredSshTarget(args.targetId) + }) + + ipcMain.handle('ssh:terminateSessions', async (_event, args: { targetId: string }) => { + invalidateConnectAttempt(args.targetId) + await runTargetLifecycle(args.targetId, async () => { + const provider = getSshPtyProvider(args.targetId) + const leases = persistedStore!.getSshRemotePtyLeases(args.targetId) + const ptyIdsByRelayId = new Map() + // Why: only leases the app still believes it owns may force a reconnect; 'expired' ones are + // swept opportunistically because they can name a host that is gone for good (issue #2626). + const ownedRelayIds = new Set() + const trackPtyId = (ptyId: string, owned: boolean): void => { + const relayPtyId = toRelaySshPtyId(args.targetId, ptyId) + if (!ptyIdsByRelayId.has(relayPtyId)) { + ptyIdsByRelayId.set(relayPtyId, toAppSshPtyId(args.targetId, ptyId)) + } + if (owned) { + ownedRelayIds.add(relayPtyId) + } + } + for (const ptyId of getPtyIdsForConnection(args.targetId)) { + trackPtyId(ptyId, true) + } + for (const lease of leases) { + if (lease.state === 'terminated') { + continue + } + // Why: 'expired' records that reattach gave up, never that the remote shell died — those are + // precisely the orphans, so the user's terminate action has to be able to reach them. + trackPtyId(lease.ptyId, lease.state !== 'expired') + } + const ptyIds = Array.from(ptyIdsByRelayId, ([relayPtyId, appPtyId]) => ({ + relayPtyId, + appPtyId + })) + + if (ownedRelayIds.size > 0 && !provider) { + throw new Error( + `${SSH_TERMINATE_RECONNECT_REQUIRED}: SSH relay is not connected; reconnect before terminating remote sessions.` + ) + } + const shutdownResults = provider + ? await Promise.allSettled( + ptyIds.map(({ appPtyId }) => + provider.shutdown(appPtyId, { immediate: true, keepHistory: false }) + ) + ) + : [] + const shutdownFailures: string[] = [] + for (const [index, result] of shutdownResults.entries()) { + const { appPtyId, relayPtyId } = ptyIds[index] + if (result.status !== 'fulfilled' && !isSshPtyNotFoundError(result.reason)) { + shutdownFailures.push( + `${relayPtyId}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}` + ) + continue + } + clearProviderPtyState(appPtyId) + deletePtyOwnership(appPtyId) + persistedStore!.markSshRemotePtyLease(args.targetId, relayPtyId, 'terminated') + } + if (shutdownFailures.length > 0) { + // Why: a failed relay shutdown can leave the remote process alive in the grace window; keep the lease/session so the user can retry. + throw new Error(`Failed to terminate SSH host sessions: ${shutdownFailures.join('; ')}`) + } + await teardownSshTargetTransport(args.targetId, (session) => session.disposeAndPersist()) + }) + }) + + ipcMain.handle('ssh:resetRelay', (_event, args: { targetId: string }) => { + const existingReset = resetRelayInFlight.get(args.targetId) + if (existingReset) { + return existingReset + } + + const target = getSshTargetRegistryStore()!.getTarget(args.targetId) + if (!target) { + throw new Error(`SSH target "${args.targetId}" not found`) + } + // Why: reset opens its own transport, so it must be fenced by shutdown the same way connect is. + assertSshConnectsNotFenced() + + let resetPromise: Promise + resetPromise = runTargetLifecycle(args.targetId, () => + doResetRelay(args.targetId, target) + ).finally(() => { + if (resetRelayInFlight.get(args.targetId) === resetPromise) { + resetRelayInFlight.delete(args.targetId) + } + }) + resetRelayInFlight.set(args.targetId, resetPromise) + return resetPromise + }) + + ipcMain.handle('ssh:getState', (_event, args: { targetId: string }) => { + return getPublicSshState(args.targetId) + }) + + // Why: auto-connect callers need to know whether connecting will prompt; true when the last connect required a credential and no live conn has it cached. + ipcMain.handle('ssh:needsPassphrasePrompt', (_event, args: { targetId: string }) => { + const target = getSshTargetRegistryStore()!.getTarget(args.targetId) + if (!target?.lastRequiredPassphrase) { + return false + } + const conn = connectionManager!.getConnection(args.targetId) + return !conn?.hasCachedCredential() + }) + + ipcMain.handle('ssh:testConnection', async (_event, args: { targetId: string }) => { + const target = getSshTargetRegistryStore()!.getTarget(args.targetId) + if (!target) { + throw new Error(`SSH target "${args.targetId}" not found`) + } + + // Why: with a live/reconnecting session, testConnection's disconnect() would tear down the relay stack (PTYs, watchers), so skip. + const existingSession = activeSessions.get(args.targetId) + const sessionState = existingSession?.getState() + if ( + sessionState === 'ready' || + sessionState === 'deploying' || + sessionState === 'reconnecting' + ) { + return { success: true, state: connectionManager!.getState(args.targetId) } + } + + // Why: testConnection's disconnect() would tear down an in-flight connect's relay deployment; await it instead. + const inFlight = connectInFlight.get(args.targetId) + if (inFlight) { + try { + const state = await inFlight.promise + return { success: true, state } + } catch (err) { + return { + success: false, + error: err instanceof Error ? err.message : String(err) + } + } + } + + testingTargets.add(args.targetId) + // Why a tracked promise and not just the id: a probe holds a real transport that no session owns, + // so shutdown has to be able to join it before the final drain disconnects what is left. + const probe = (async () => { + // Why: a probe transport opened after the shutdown drain would outlive orderly teardown. + assertSshConnectsNotFenced() + const conn = await connectionManager!.connect(target) + const state = conn.getState() + await connectionManager!.disconnect(args.targetId) + return state + })() + testConnectionProbes.add(probe) + try { + return { success: true, state: await probe } + } catch (err) { + return { + success: false, + error: err instanceof Error ? err.message : String(err) + } + } finally { + testConnectionProbes.delete(probe) + testingTargets.delete(args.targetId) + // Why: clear so a test's credential prompt doesn't leave lastRequiredPassphrase=true and defer this target at startup. + credentialRequestedForTarget.delete(args.targetId) + } + }) +} diff --git a/src/main/ipc/ssh-connection-state-callbacks.ts b/src/main/ipc/ssh-connection-state-callbacks.ts new file mode 100644 index 00000000000..9c79f7eb9b4 --- /dev/null +++ b/src/main/ipc/ssh-connection-state-callbacks.ts @@ -0,0 +1,129 @@ +import type { SshConnectionCallbacks } from '../ssh/ssh-connection' +import type { SshConnectionState, SshTarget } from '../../shared/ssh-types' +import { rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { activeSessions } from './ssh-active-relay-sessions' +import { + connectInFlight, + credentialRequestedForTarget, + pendingTransportReconnects, + testingTargets +} from './ssh-connect-attempt-registry' +import { connectionManager, getCurrentMainWindow } from './ssh-ipc-context' +import { requestCredential } from './ssh-passphrase' +import { clearRelayLostBackoff } from './ssh-relay-lost-backoff' +import { + broadcastSshState, + clearRelayStateOverride, + publishRelayOverride, + relayStateOverrides +} from './ssh-renderer-broadcast' + +export function relayGracePeriodForTarget( + target: SshTarget | null | undefined +): number | undefined { + return target?.relayGracePeriodSeconds +} + +// Why extracted from the callbacks object: an explicit connect pushes its own 'deploying-relay' +// through this exact path, so both callers share one implementation. +export function handleSshConnectionStateChange(targetId: string, state: SshConnectionState): void { + if (testingTargets.has(targetId)) { + return + } + + // Why: an SSH reconnect must re-deploy the relay and rebuild providers; the guard below fires only for real reconnects, not an explicit connect's 'deploying'. + const session = activeSessions.get(targetId) + const sessionState = session?.getState() + const transportReconnectStarted = + state.status === 'reconnecting' && + (sessionState === 'ready' || sessionState === 'reconnecting') && + !pendingTransportReconnects.has(targetId) + if (transportReconnectStarted) { + rotateSshProviderAuthority(targetId) + pendingTransportReconnects.add(targetId) + } else if ( + state.status === 'disconnected' || + state.status === 'auth-failed' || + state.status === 'reconnection-failed' || + state.status === 'error' + ) { + pendingTransportReconnects.delete(targetId) + } + const completedTransportReconnect = + state.status === 'connected' && pendingTransportReconnects.delete(targetId) + const shouldReconnectRelay = + session !== undefined && + completedTransportReconnect && + state.reconnectAttempt === 0 && + (sessionState === 'ready' || sessionState === 'reconnecting') + const relayReconnectAlreadyInFlight = + !completedTransportReconnect && + state.status === 'connected' && + sessionState === 'reconnecting' && + relayStateOverrides.has(targetId) + + if (shouldReconnectRelay) { + // Why: this branch redeploys the relay itself over a fresh transport, so any pending relay-lost retry is stale — dropping it also gives the new transport generation a full attempt budget. + clearRelayLostBackoff(targetId) + // Why: SSH connects before the relay providers rebuild; keep renderer actions gated until SshRelaySession reaches ready again. + publishRelayOverride( + getCurrentMainWindow, + targetId, + 'reconnecting', + 'Relay channel reconnecting...', + state.reconnectAttempt + ) + } else if (relayReconnectAlreadyInFlight) { + // Why: duplicate connected notifications belong to the same socket generation and must not expose providers before relay recovery finishes. + return + } else if ( + state.status === 'connected' && + session !== undefined && + sessionState !== 'ready' && + !completedTransportReconnect && + connectInFlight.has(targetId) + ) { + // Why: the raw SSH transport reaches 'connected' before the relay session establishes during an + // explicit connect. Forwarding it makes the renderer treat the host as fully up — it remounts + // SSH panes (-> window.api.ssh.connect) and fires connected-gated data reads before any provider + // exists. On a permanent relay-deploy failure that premature 'connected' drives an unbounded + // reconnect loop. Hold it at 'deploying-relay'; the in-flight doConnect broadcasts the + // authoritative 'connected' directly (bypassing this callback) after establish() succeeds, or a + // terminal state on failure. The connectInFlight gate keeps this scoped to a live connect, so a + // stray raw 'connected' with no follow-up (e.g. a transport blip on a session left 'idle' by a + // relay version mismatch) is never wedged at 'deploying-relay'. + clearRelayStateOverride(targetId) + broadcastSshState(getCurrentMainWindow, targetId, { + targetId, + status: 'deploying-relay', + error: state.error, + reconnectAttempt: state.reconnectAttempt + }) + } else { + clearRelayStateOverride(targetId) + broadcastSshState(getCurrentMainWindow, targetId, state) + } + + if (!session) { + return + } + // Why: allow reconnect from both 'ready' and 'reconnecting'; without the latter, a failed relay deploy would permanently brick the session. + if (shouldReconnectRelay) { + const target = getSshTargetRegistryStore()?.getTarget(targetId) + const conn = connectionManager?.getConnection(targetId) + if (conn) { + void session.reconnect(conn, relayGracePeriodForTarget(target)) + } + } +} + +export function createSshConnectionCallbacks(): SshConnectionCallbacks { + return { + onCredentialRequest: (targetId, kind, detail) => { + credentialRequestedForTarget.add(targetId) + return requestCredential(getCurrentMainWindow, targetId, kind, detail) + }, + onStateChange: handleSshConnectionStateChange + } +} diff --git a/src/main/ipc/ssh-host-sleep-reconnect.ts b/src/main/ipc/ssh-host-sleep-reconnect.ts new file mode 100644 index 00000000000..fcc33db7cab --- /dev/null +++ b/src/main/ipc/ssh-host-sleep-reconnect.ts @@ -0,0 +1,71 @@ +import { powerMonitor } from 'electron' +import type { SshRelaySession } from '../ssh/ssh-relay-session' +import { activeSessions } from './ssh-active-relay-sessions' +import { connectionManager } from './ssh-ipc-context' + +let powerMonitorUnsubscribe: (() => void) | null = null + +// Why: macOS can resume before the network is back, so a failed first probe gets one retry before the link is declared dead (#7773). +const RESUME_PROBE_TIMEOUT_MS = 5_000 +const RESUME_PROBE_ATTEMPTS = 2 + +async function isRelayLinkAliveAfterResume(session: SshRelaySession): Promise { + const mux = session.getMux() + if (!mux || mux.isDisposed()) { + return false + } + for (let attempt = 0; attempt < RESUME_PROBE_ATTEMPTS; attempt++) { + if (await mux.probeLiveness(RESUME_PROBE_TIMEOUT_MS)) { + return true + } + } + return false +} + +export function registerPowerMonitorReconnect(): void { + powerMonitorUnsubscribe?.() + const onSuspend = (): void => { + for (const session of activeSessions.values()) { + session.prepareForHostSleep() + } + } + const onResume = (): void => { + for (const [targetId, session] of activeSessions) { + const manager = connectionManager + const conn = manager?.getConnection(targetId) + if (!conn) { + continue + } + void (async () => { + // Why: unconditional reconnect on wake tore down live sessions and flashed the overlay (#7773); only reconnect if the relay link actually died during sleep. + if (await isRelayLinkAliveAfterResume(session)) { + return + } + // Why: the probe can take ~10s; bail if the session/connection was replaced or torn down meanwhile, else we'd resurrect it. + if (activeSessions.get(targetId) !== session || manager?.getConnection(targetId) !== conn) { + return + } + try { + await manager?.reconnect(targetId) + } catch (err) { + console.warn( + `[ssh] Failed to reconnect ${targetId} after system resume: ${ + err instanceof Error ? err.message : String(err) + }` + ) + } + })() + } + } + powerMonitor.on('suspend', onSuspend) + powerMonitor.on('resume', onResume) + powerMonitorUnsubscribe = () => { + powerMonitor.off('suspend', onSuspend) + powerMonitor.off('resume', onResume) + } +} + +export function unregisterPowerMonitorReconnect(): void { + powerMonitorUnsubscribe?.() + powerMonitorUnsubscribe = null +} diff --git a/src/main/ipc/ssh-ipc-context.ts b/src/main/ipc/ssh-ipc-context.ts new file mode 100644 index 00000000000..f17cfb6b1f1 --- /dev/null +++ b/src/main/ipc/ssh-ipc-context.ts @@ -0,0 +1,37 @@ +import type { BrowserWindow } from 'electron' +import type { Store } from '../persistence' +import type { SshConnectionManager } from '../ssh/ssh-connection-manager' +import type { SshPortForwardManager } from '../ssh/ssh-port-forward' +import type { OrcaRuntimeService } from '../runtime/orca-runtime' + +// Why live bindings rather than getters: every SSH IPC module reads these singletons on the hot +// path, and only registerSshHandlers/the test reset write them (through the setters below). +export let connectionManager: SshConnectionManager | null = null +export let portForwardManager: SshPortForwardManager | null = null +export let persistedStore: Store | null = null +let currentGetMainWindow: () => BrowserWindow | null = () => null +export let currentRuntime: OrcaRuntimeService | undefined + +export function setConnectionManager(next: SshConnectionManager | null): void { + connectionManager = next +} + +export function setPortForwardManager(next: SshPortForwardManager | null): void { + portForwardManager = next +} + +export function setPersistedStore(next: Store | null): void { + persistedStore = next +} + +export function setCurrentGetMainWindow(next: () => BrowserWindow | null): void { + currentGetMainWindow = next +} + +export function setCurrentRuntime(next: OrcaRuntimeService | undefined): void { + currentRuntime = next +} + +export function getCurrentMainWindow(): BrowserWindow | null { + return currentGetMainWindow() +} diff --git a/src/main/ipc/ssh-port-forward-handlers.ts b/src/main/ipc/ssh-port-forward-handlers.ts new file mode 100644 index 00000000000..5b936fff5e3 --- /dev/null +++ b/src/main/ipc/ssh-port-forward-handlers.ts @@ -0,0 +1,108 @@ +import { ipcMain } from 'electron' +import { + enrichSshForwardEntries, + getWorktreeIdsForConnection +} from '../ports/ssh-advertised-url-enrichment' +import { activeSessions } from './ssh-active-relay-sessions' +import { + connectionManager, + getCurrentMainWindow, + persistedStore, + portForwardManager +} from './ssh-ipc-context' +import { persistPortForwards } from './ssh-port-forward-persistence' +import { broadcastPortForwards, enrichDetected } from './ssh-renderer-broadcast' + +export function registerSshPortForwardHandlers(): void { + ipcMain.handle( + 'ssh:addPortForward', + async ( + _event, + args: { + targetId: string + localPort: number + remoteHost: string + remotePort: number + label?: string + } + ) => { + const conn = connectionManager!.getConnection(args.targetId) + if (!conn) { + throw new Error(`SSH connection "${args.targetId}" not found`) + } + const entry = await portForwardManager!.addForward( + args.targetId, + conn, + args.localPort, + args.remoteHost, + args.remotePort, + args.label + ) + persistPortForwards(args.targetId) + broadcastPortForwards(getCurrentMainWindow, args.targetId) + return entry + } + ) + + ipcMain.handle( + 'ssh:updatePortForward', + async ( + _event, + args: { + id: string + targetId: string + localPort: number + remoteHost: string + remotePort: number + label?: string + } + ) => { + const conn = connectionManager!.getConnection(args.targetId) + if (!conn) { + throw new Error(`SSH connection "${args.targetId}" not found`) + } + try { + const entry = await portForwardManager!.updateForward( + args.id, + conn, + args.localPort, + args.remoteHost, + args.remotePort, + args.label + ) + persistPortForwards(entry.connectionId) + broadcastPortForwards(getCurrentMainWindow, entry.connectionId) + return entry + } catch (err) { + // Why: edit/rollback may have failed, so resync renderer to actual runtime state. + persistPortForwards(args.targetId) + broadcastPortForwards(getCurrentMainWindow, args.targetId) + throw err + } + } + ) + + ipcMain.handle('ssh:removePortForward', async (_event, args: { id: string }) => { + const removed = await portForwardManager!.removeForwardAndWait(args.id) + if (removed) { + persistPortForwards(removed.connectionId) + broadcastPortForwards(getCurrentMainWindow, removed.connectionId) + } + return removed + }) + + ipcMain.handle('ssh:listPortForwards', (_event, args?: { targetId?: string }) => { + const all = portForwardManager!.listForwards(args?.targetId) + if (!persistedStore || !args?.targetId) { + // Why: cross-target entries can't be mapped to worktrees in one call, so serve the raw list. + return all + } + return enrichSshForwardEntries(all, getWorktreeIdsForConnection(persistedStore, args.targetId)) + }) + + ipcMain.handle('ssh:listDetectedPorts', (_event, args: { targetId: string }) => { + const session = activeSessions.get(args.targetId) + const ports = session?.getPortScanner()?.getDetectedPorts(args.targetId) ?? [] + return enrichDetected(args.targetId, ports) + }) +} diff --git a/src/main/ipc/ssh-port-forward-persistence.ts b/src/main/ipc/ssh-port-forward-persistence.ts new file mode 100644 index 00000000000..e0407f859f5 --- /dev/null +++ b/src/main/ipc/ssh-port-forward-persistence.ts @@ -0,0 +1,82 @@ +import type { BrowserWindow } from 'electron' +import type { SavedPortForward } from '../../shared/ssh-types' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { connectionManager, portForwardManager } from './ssh-ipc-context' +import { broadcastPortForwards } from './ssh-renderer-broadcast' + +// Why: after user add/remove/update the runtime manager is the source of truth — persist exactly its entries (unrestored ones handled by a separate helper). +export function persistPortForwards(targetId: string): void { + const active = portForwardManager!.listForwards(targetId) + const saved: SavedPortForward[] = active.map((f) => ({ + localPort: f.localPort, + remoteHost: f.remoteHost, + remotePort: f.remotePort, + label: f.label + })) + getSshTargetRegistryStore()!.updateTarget(targetId, { + portForwards: saved.length > 0 ? saved : undefined + }) +} + +// Why: keep forwards that failed to restore in the persisted list so they retry on next reconnect instead of being silently dropped. +export function persistPortForwardsWithUnrestored(targetId: string): void { + const active = portForwardManager!.listForwards(targetId) + const activeKeys = new Set(active.map((f) => `${f.localPort}:${f.remoteHost}:${f.remotePort}`)) + + const existing = getSshTargetRegistryStore()!.getTarget(targetId)?.portForwards ?? [] + const unrestored = existing.filter( + (pf) => !activeKeys.has(`${pf.localPort}:${pf.remoteHost}:${pf.remotePort}`) + ) + + const saved: SavedPortForward[] = [ + ...active.map((f) => ({ + localPort: f.localPort, + remoteHost: f.remoteHost, + remotePort: f.remotePort, + label: f.label + })), + ...unrestored + ] + getSshTargetRegistryStore()!.updateTarget(targetId, { + portForwards: saved.length > 0 ? saved : undefined + }) +} + +export async function restorePortForwards( + targetId: string, + getMainWindow: () => BrowserWindow | null +): Promise { + const target = getSshTargetRegistryStore()!.getTarget(targetId) + if (!target?.portForwards?.length) { + return + } + const conn = connectionManager!.getConnection(targetId) + if (!conn) { + return + } + + // Why: keep failed restores in persisted state — a failure may be transient (port temporarily busy), so retry on next reconnect. + for (const saved of target.portForwards) { + // Why: a reconnect mid-loop swaps the connection object; bail on identity change so we don't add forwards to a stale conn (leaking listeners). + if (connectionManager!.getConnection(targetId) !== conn) { + return + } + try { + await portForwardManager!.addForward( + targetId, + conn, + saved.localPort, + saved.remoteHost, + saved.remotePort, + saved.label + ) + } catch (err) { + console.warn( + `[ssh] Failed to restore forward :${saved.localPort} → ${saved.remoteHost}:${saved.remotePort}: ${err instanceof Error ? err.message : String(err)}` + ) + } + } + + persistPortForwardsWithUnrestored(targetId) + broadcastPortForwards(getMainWindow, targetId) +} diff --git a/src/main/ipc/ssh-relay-lost-backoff.ts b/src/main/ipc/ssh-relay-lost-backoff.ts new file mode 100644 index 00000000000..0b92e45605a --- /dev/null +++ b/src/main/ipc/ssh-relay-lost-backoff.ts @@ -0,0 +1,32 @@ +import type { SshConnectionStatus } from '../../shared/ssh-types' + +// Why: without backoff, a relay channel that keeps dying reconnects as fast as the network allows, hammering local + remote sshd; track attempts and back off to end the loop recoverably. +export type RelayLostBackoffState = { + attempts: number + reconnectTimer: ReturnType | null + stabilizedTimer: ReturnType | null +} +export const relayLostBackoff = new Map() +export const RELAY_LOST_MAX_ATTEMPTS = 6 +export const RELAY_LOST_BASE_DELAY_MS = 500 +export const RELAY_LOST_MAX_DELAY_MS = 15_000 +// Why: a reconnect whose mux dies within this window was a flap, not a recovery — don't reset the attempt counter. 5s covers provider re-registration + PTY reattach. +export const RELAY_LOST_STABILIZED_MS = 5_000 +// Why: transport states the SSH ladder never leaves on its own — waiting for a relay redeploy past one of these is an unbounded loop. +export const TRANSPORT_TERMINAL_STATUSES = new Set([ + 'disconnected', + 'auth-failed', + 'reconnection-failed', + 'error' +]) + +export function clearRelayLostBackoff(targetId: string): void { + const state = relayLostBackoff.get(targetId) + if (state?.reconnectTimer) { + clearTimeout(state.reconnectTimer) + } + if (state?.stabilizedTimer) { + clearTimeout(state.stabilizedTimer) + } + relayLostBackoff.delete(targetId) +} diff --git a/src/main/ipc/ssh-relay-session-callbacks.ts b/src/main/ipc/ssh-relay-session-callbacks.ts new file mode 100644 index 00000000000..976af992398 --- /dev/null +++ b/src/main/ipc/ssh-relay-session-callbacks.ts @@ -0,0 +1,197 @@ +import type { SshRelaySession } from '../ssh/ssh-relay-session' +import { rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { activeSessions } from './ssh-active-relay-sessions' +import { testingTargets } from './ssh-connect-attempt-registry' +import { relayGracePeriodForTarget } from './ssh-connection-state-callbacks' +import { + connectionManager, + currentRuntime, + getCurrentMainWindow, + persistedStore, + portForwardManager +} from './ssh-ipc-context' +import { restorePortForwards } from './ssh-port-forward-persistence' +import { + clearRelayLostBackoff, + RELAY_LOST_BASE_DELAY_MS, + RELAY_LOST_MAX_ATTEMPTS, + RELAY_LOST_MAX_DELAY_MS, + RELAY_LOST_STABILIZED_MS, + relayLostBackoff, + TRANSPORT_TERMINAL_STATUSES +} from './ssh-relay-lost-backoff' +import { + broadcastDetectedPortsFromCurrentWindow, + broadcastSshState, + clearRelayStateOverride, + connectionSupportsFolderDownload, + publishRelayOverride +} from './ssh-renderer-broadcast' + +export function configureRelaySessionCallbacks(session: SshRelaySession): void { + session.setOnTerminalRelayError((tid, err) => { + clearRelayLostBackoff(tid) + if (activeSessions.get(tid)?.getState() !== 'deploying') { + rotateSshProviderAuthority(tid) + } + console.warn( + `[ssh] Terminal relay error for ${tid}: ${err.message}; skipping reconnect backoff.` + ) + publishRelayOverride(getCurrentMainWindow, tid, 'error', err.message, 0) + }) + + session.setOnRelayLost((tid) => { + const s = activeSessions.get(tid) + if (!s) { + return + } + const c = connectionManager?.getConnection(tid) + if (!c) { + return + } + const t = getSshTargetRegistryStore()?.getTarget(tid) + + // Why: bounded exponential backoff — without it, a remote bug that closes every fresh --connect channel becomes an infinite relay-deploy loop. + const state = relayLostBackoff.get(tid) ?? { + attempts: 0, + reconnectTimer: null, + stabilizedTimer: null + } + if (state.stabilizedTimer) { + clearTimeout(state.stabilizedTimer) + state.stabilizedTimer = null + } + if (state.reconnectTimer) { + return + } + rotateSshProviderAuthority(tid) + + // Why: re-deploying the relay rides the SSH transport, so while the transport is itself down no attempt + // can succeed. Waiting at the max delay without consuming the budget keeps a flapping host off the + // manual-reconnect banner, which would tell the user to act on a link that is still auto-recovering. + const transportStatus = connectionManager?.getState(tid)?.status + const transportConnected = transportStatus === 'connected' + if (transportConnected && state.attempts >= RELAY_LOST_MAX_ATTEMPTS) { + console.warn( + `[ssh] Relay channel for ${tid} kept dying across ${state.attempts} attempts; giving up. User must reconnect manually.` + ) + relayLostBackoff.delete(tid) + // Why: surface the failure — a live SSH connection with a dead relay is otherwise invisible (typing in remote terminals just stops working). + publishRelayOverride( + getCurrentMainWindow, + tid, + 'error', + 'Relay channel kept dropping. Click Reconnect on the SSH target before retrying.', + 0 + ) + return + } + + const scheduleRelayRedeploy = (delay: number, attemptCharged: boolean): void => { + state.reconnectTimer = setTimeout(() => { + state.reconnectTimer = null + relayLostBackoff.set(tid, state) + const liveConn = connectionManager?.getConnection(tid) + if (!liveConn || !activeSessions.has(tid)) { + clearRelayLostBackoff(tid) + return + } + const status = connectionManager?.getState(tid)?.status + if (status === 'connected') { + if (!attemptCharged) { + // Why: waiting is free, but the deploy it defers is real — charge it here so a transport that + // flaps back to 'connected' can't redeploy forever on an uncharged budget. + state.attempts += 1 + } + void s.reconnect(liveConn, relayGracePeriodForTarget(t)) + return + } + if (status === undefined || TRANSPORT_TERMINAL_STATUSES.has(status)) { + // Why: the transport gave up for good; its own state is what the user acts on, so stop waiting for a redeploy that can never run. + clearRelayLostBackoff(tid) + return + } + // Why: still mid-transition — re-arm at the max delay without consuming an attempt. It ends once + // the transport settles: 'connected' redeploys, a terminal status or a dropped session clears above. + scheduleRelayRedeploy(RELAY_LOST_MAX_DELAY_MS, false) + }, delay) + relayLostBackoff.set(tid, state) + } + + if (!transportConnected) { + publishRelayOverride( + getCurrentMainWindow, + tid, + 'reconnecting', + 'Relay channel lost. Reconnecting...', + state.attempts + ) + scheduleRelayRedeploy(RELAY_LOST_MAX_DELAY_MS, false) + console.warn( + `[ssh] Relay channel for ${tid} lost while the SSH transport is ${transportStatus ?? 'unknown'}; waiting ${RELAY_LOST_MAX_DELAY_MS}ms without consuming an attempt` + ) + return + } + + const delay = Math.min(RELAY_LOST_BASE_DELAY_MS * 2 ** state.attempts, RELAY_LOST_MAX_DELAY_MS) + state.attempts += 1 + publishRelayOverride( + getCurrentMainWindow, + tid, + 'reconnecting', + 'Relay channel lost. Reconnecting...', + state.attempts + ) + scheduleRelayRedeploy(delay, true) + console.warn( + `[ssh] Relay channel for ${tid} lost; reconnect attempt ${state.attempts}/${RELAY_LOST_MAX_ATTEMPTS} in ${delay}ms` + ) + }) + + // Why: fires after both establish() and reconnect() reach 'ready'; re-create persisted port forwards so they survive restarts and blips. + session.setOnReady((tid) => { + const state = relayLostBackoff.get(tid) + if (state) { + if (state.stabilizedTimer) { + clearTimeout(state.stabilizedTimer) + } + // Why: stabilization counts post-ready uptime; slow deploy time before `ready` doesn't prove the new relay survived real work. + state.stabilizedTimer = setTimeout(() => { + const current = relayLostBackoff.get(tid) + if (current === state && !current.reconnectTimer) { + relayLostBackoff.delete(tid) + } + }, RELAY_LOST_STABILIZED_MS) + relayLostBackoff.set(tid, state) + } + clearRelayStateOverride(tid) + if (!testingTargets.has(tid)) { + broadcastSshState(getCurrentMainWindow, tid, { + targetId: tid, + status: 'connected', + error: null, + reconnectAttempt: 0, + supportsFolderDownload: connectionSupportsFolderDownload(tid) + }) + } + currentRuntime?.notifySshRelayReady?.(tid) + void restorePortForwards(tid, getCurrentMainWindow) + }) +} + +export function refreshActiveRelaySessions(): void { + if (!persistedStore || !portForwardManager) { + return + } + for (const session of activeSessions.values()) { + session.refreshEnvironment( + getCurrentMainWindow, + persistedStore, + portForwardManager, + currentRuntime, + broadcastDetectedPortsFromCurrentWindow + ) + configureRelaySessionCallbacks(session) + } +} diff --git a/src/main/ipc/ssh-renderer-broadcast.ts b/src/main/ipc/ssh-renderer-broadcast.ts new file mode 100644 index 00000000000..fa1f1f2c16e --- /dev/null +++ b/src/main/ipc/ssh-renderer-broadcast.ts @@ -0,0 +1,144 @@ +import type { BrowserWindow } from 'electron' +import type { SshPortForwardManager } from '../ssh/ssh-port-forward' +import type { + DetectedPort, + EnrichedDetectedPort, + SshConnectionStatus, + SshConnectionState +} from '../../shared/ssh-types' +import { isRuntimeOwnedSshTargetId } from '../../shared/execution-host' +import { + enrichSshDetectedPorts, + enrichSshForwardEntries, + getWorktreeIdsForConnection +} from '../ports/ssh-advertised-url-enrichment' +import { getSshProviderAuthority } from '../ssh/ssh-provider-authority' +import { activeSessions } from './ssh-active-relay-sessions' +import { + connectionManager, + currentRuntime, + getCurrentMainWindow, + persistedStore, + portForwardManager +} from './ssh-ipc-context' + +export const relayStateOverrides = new Map() + +export function broadcastSshState( + getMainWindow: () => BrowserWindow | null, + targetId: string, + state: SshConnectionState +): void { + // Why: runtime-owned (ephemeral-VM) targets are hidden from the renderer, so broadcasting their state only triggers wasted listTargets() lookups. + if (isRuntimeOwnedSshTargetId(targetId)) { + currentRuntime?.invalidateSshWorktreeScanCache?.(targetId) + return + } + const enrichedState = withSshRemotePlatform(targetId, state) + const win = getMainWindow() + if (win && !win.isDestroyed()) { + win.webContents.send('ssh:state-changed', { targetId, state: enrichedState }) + } + // Why: paired remote clients have no ssh:state-changed IPC; without this their terminals keep a stale reconnect overlay. + currentRuntime?.notifySshStateChanged?.(targetId, enrichedState) +} + +function withSshRemotePlatform(targetId: string, state: SshConnectionState): SshConnectionState { + const remotePlatform = activeSessions.get(targetId)?.getHostPlatform()?.os + const authority = getSshProviderAuthority(targetId) + return { + ...state, + targetId, + providerEpoch: authority.providerEpoch, + connectionGeneration: authority.connectionGeneration, + ...(remotePlatform ? { remotePlatform } : {}) + } +} + +export function publishRelayOverride( + getMainWindow: () => BrowserWindow | null, + targetId: string, + status: SshConnectionStatus, + error: string | null, + reconnectAttempt: number +): void { + const state = withSshRemotePlatform(targetId, { targetId, status, error, reconnectAttempt }) + relayStateOverrides.set(targetId, state) + broadcastSshState(getMainWindow, targetId, state) +} + +export function clearRelayStateOverride(targetId: string): void { + relayStateOverrides.delete(targetId) +} + +export function connectionSupportsFolderDownload(targetId: string): boolean { + // Why: connections without an explicit transport are ssh2-shaped; only a confirmed system-SSH transport lacks the SFTP-only capability. + return connectionManager?.getConnection(targetId)?.usesSystemSshTransport?.() !== true +} + +export function getPublicSshState(targetId: string): SshConnectionState | undefined { + const state = relayStateOverrides.get(targetId) ?? connectionManager!.getState(targetId) + return state ? withSshRemotePlatform(targetId, state) : undefined +} + +export function broadcastPortForwards( + getMainWindow: () => BrowserWindow | null, + targetId: string +): void { + const win = getMainWindow() + if (!win || win.isDestroyed()) { + return + } + win.webContents.send('ssh:port-forwards-changed', { + targetId, + forwards: listForwardsEnriched(targetId) + }) +} + +export function broadcastDetectedPorts( + getMainWindow: () => BrowserWindow | null, + targetId: string, + ports: DetectedPort[], + options?: Parameters[3] +): void { + const win = getMainWindow() + if (!win || win.isDestroyed()) { + return + } + win.webContents.send('ssh:detected-ports-changed', { + targetId, + ports: enrichDetected(targetId, ports, options) + }) +} + +function listForwardsEnriched(targetId: string): ReturnType { + const raw = portForwardManager!.listForwards(targetId) + if (!persistedStore) { + return raw + } + return enrichSshForwardEntries(raw, getWorktreeIdsForConnection(persistedStore, targetId)) +} + +export function enrichDetected( + targetId: string, + ports: DetectedPort[], + options?: Parameters[3] +): EnrichedDetectedPort[] { + if (!persistedStore) { + return ports + } + return enrichSshDetectedPorts( + ports, + getWorktreeIdsForConnection(persistedStore, targetId), + undefined, + options + ) +} + +export function broadcastDetectedPortsFromCurrentWindow( + targetId: string, + ports: DetectedPort[], + _platform: string +): void { + broadcastDetectedPorts(getCurrentMainWindow, targetId, ports) +} diff --git a/src/main/ipc/ssh-session-teardown.ts b/src/main/ipc/ssh-session-teardown.ts new file mode 100644 index 00000000000..b8636590747 --- /dev/null +++ b/src/main/ipc/ssh-session-teardown.ts @@ -0,0 +1,147 @@ +import type { SshConnection } from '../ssh/ssh-connection' +import type { SshRelaySession } from '../ssh/ssh-relay-session' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { activeSessions } from './ssh-active-relay-sessions' +import { invalidateConnectAttempt } from './ssh-connect-attempt-registry' +import { connectionManager, persistedStore, portForwardManager } from './ssh-ipc-context' +import { clearRelayLostBackoff } from './ssh-relay-lost-backoff' +import { clearRelayStateOverride } from './ssh-renderer-broadcast' +import { runTargetLifecycle } from './ssh-target-lifecycle-queue' + +export async function disconnectRegisteredSshTarget(targetId: string): Promise { + invalidateConnectAttempt(targetId) + await runTargetLifecycle(targetId, () => + teardownSshTargetTransport(targetId, (session) => session.detachAndPersist()) + ) +} + +export async function removeRegisteredSshTarget(targetId: string): Promise { + const store = getSshTargetRegistryStore() + if (!store) { + return + } + invalidateConnectAttempt(targetId) + await runTargetLifecycle(targetId, async () => { + try { + // Why: removal is destructive; dispose so remote PTYs cannot reattach to a deleted target. + await teardownSshTargetTransport(targetId, (session) => session.disposeAndPersist()) + } catch (err) { + // Why: a failed disconnect must not block metadata removal, else the target lingers in the store with uncleaned leases. + console.warn( + `[ssh] Failed to disconnect removed target ${targetId}: ${err instanceof Error ? err.message : String(err)}` + ) + } + persistedStore?.removeSshRemotePtyLeases(targetId) + store.removeTarget(targetId) + }) +} + +export async function teardownSshTargetTransport( + targetId: string, + teardown: (session: SshRelaySession) => void | Promise +): Promise { + let transportDisconnect: Promise<{ ok: true } | { ok: false; error: unknown }> + try { + transportDisconnect = Promise.resolve(connectionManager?.disconnect(targetId)).then( + () => ({ ok: true }) as const, + (error: unknown) => ({ ok: false, error }) as const + ) + } catch (error) { + transportDisconnect = Promise.resolve({ ok: false, error }) + } + const sessionTeardown = teardownActiveSshSession(targetId, teardown).then( + () => ({ ok: true }) as const, + (error: unknown) => ({ ok: false, error }) as const + ) + const [disconnectResult, teardownResult] = await Promise.all([ + transportDisconnect, + sessionTeardown + ]) + if (!teardownResult.ok) { + throw teardownResult.error + } + if (!disconnectResult.ok) { + throw disconnectResult.error + } +} + +export async function teardownActiveSshSession( + targetId: string, + teardown: (session: SshRelaySession) => void | Promise +): Promise { + const session = activeSessions.get(targetId) + if (!session) { + return + } + let teardownError: { error: unknown } | null = null + try { + // Why: await port teardown so local listeners are released before disconnect/remove completes, else an immediate reconnect hits EADDRINUSE. + await portForwardManager?.removeAllForwards(targetId) + } catch (error) { + teardownError = { error } + } + try { + await teardown(session) + } catch (error) { + teardownError ??= { error } + } + if (activeSessions.get(targetId) === session) { + activeSessions.delete(targetId) + clearRelayLostBackoff(targetId) + clearRelayStateOverride(targetId) + } + if (teardownError) { + throw teardownError.error + } +} + +// Why: a dropped session must detach, not just leave activeSessions — detach releases the SSH PTY +// consumer identity so the next connect reclaims its owner lease instead of minting a new one. +// Why awaited, and why the map entry outlives the await: a retry can start the moment this returns, +// so it must either find the session (and await this same latched teardown at the existing-session +// path) or find nothing because the 'detached' lease write is already durable. Deleting first lets a +// fast reconnect mark leases 'attached' and then have this session's late 'detached' write clobber it. +export async function abandonFailedSshSession( + targetId: string, + session: SshRelaySession +): Promise { + // Why: detachAndPersist transitions recovery ownership synchronously; only durability is awaited. + try { + await session.detachAndPersist() + } catch (error) { + // Why: a teardown throw must not mask the connect error the caller is about to rethrow. + console.warn( + `[ssh] Failed to detach abandoned session for ${targetId}: ${error instanceof Error ? error.message : String(error)}` + ) + } + if (activeSessions.get(targetId) === session) { + activeSessions.delete(targetId) + } +} + +// Why: a connect cancelled after its transport already opened still owns that transport and its +// unpublished session — nothing else will reach them, so it has to close them itself. Why only the +// connection it minted, and by identity: disconnecting by target id (or closing a transport it merely +// reused) would tear down the replacement connect's live transport. +export async function abandonCancelledConnectAttempt( + targetId: string, + session: SshRelaySession, + mintedConnection: SshConnection | null +): Promise { + // Why the identity guard: every path that removes a session from activeSessions detaches it first, + // so re-detaching a superseded session would only clobber the replacement's lease state. + if (activeSessions.get(targetId) === session) { + await abandonFailedSshSession(targetId, session) + } + if (!mintedConnection) { + return + } + try { + await connectionManager!.disconnectConnection(targetId, mintedConnection) + } catch (error) { + // Why: the caller is about to throw the cancellation; a teardown throw must not replace it. + console.warn( + `[ssh] Failed to disconnect cancelled connect transport for ${targetId}: ${error instanceof Error ? error.message : String(error)}` + ) + } +} diff --git a/src/main/ipc/ssh-shutdown-drain.ts b/src/main/ipc/ssh-shutdown-drain.ts new file mode 100644 index 00000000000..f66db066c01 --- /dev/null +++ b/src/main/ipc/ssh-shutdown-drain.ts @@ -0,0 +1,158 @@ +import { activeSessions } from './ssh-active-relay-sessions' +import { + connectInFlight, + invalidateConnectAttempt, + resetRelayInFlight, + testConnectionProbes +} from './ssh-connect-attempt-registry' +import { connectionManager } from './ssh-ipc-context' +import { teardownActiveSshSession } from './ssh-session-teardown' + +// Why one budget for the whole sequence rather than one per phase: an invalidated connect only +// observes its cancellation at the next checkpoint, and one blocked in the transport handshake can +// sit there for the whole SSH timeout. A per-phase timeout lets any single phase consume the global +// quit deadline; a shared absolute deadline cannot. +export const SSH_SHUTDOWN_BUDGET_MS = 6_000 + +export type SshShutdownPhase = 'drain' | 'in-flight-join' | 'final-drain' +export type SshShutdownUnfinished = { targetId: string; phase: SshShutdownPhase } +export type SshShutdownResult = { + unfinished: readonly SshShutdownUnfinished[] + errors: readonly unknown[] +} + +type SshShutdownTask = { targetId: string; promise: Promise } + +let sshShutdownDrain: Promise | null = null + +export function resetSshShutdownDrain(): void { + sshShutdownDrain = null +} + +async function settleTasksWithinMs( + tasks: readonly SshShutdownTask[], + timeoutMs: number +): Promise<{ timedOut: SshShutdownTask[]; errors: unknown[] }> { + const pending = new Set(tasks) + const errors: unknown[] = [] + if (tasks.length === 0) { + return { timedOut: [], errors } + } + const tracked = tasks.map(async (task) => { + try { + await task.promise + } catch (error) { + errors.push(error) + } + pending.delete(task) + }) + let timer: ReturnType | undefined + try { + await Promise.race([ + Promise.all(tracked), + new Promise((resolve) => { + timer = setTimeout(resolve, timeoutMs) + timer.unref?.() + }) + ]) + } finally { + clearTimeout(timer) + } + return { timedOut: [...pending], errors } +} + +function sshShutdownTasks(targetIds: readonly string[]): SshShutdownTask[] { + return [ + ...targetIds + .filter((targetId) => activeSessions.has(targetId)) + .map((targetId) => ({ + targetId, + promise: teardownActiveSshSession(targetId, (session) => session.detachAndPersist()) + })), + { targetId: '*transports', promise: connectionManager?.disconnectAll() ?? Promise.resolve() } + ] +} + +async function drainSshShutdown( + targetIds: readonly string[], + inFlight: readonly SshShutdownTask[], + detachErrors: readonly unknown[] = [] +): Promise { + const deadline = Date.now() + SSH_SHUTDOWN_BUDGET_MS + const unfinished: SshShutdownUnfinished[] = [] + const errors: unknown[] = [...detachErrors] + const runPhase = async ( + phase: SshShutdownPhase, + tasks: readonly SshShutdownTask[] + ): Promise => { + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) { + unfinished.push(...tasks.map((task) => ({ targetId: task.targetId, phase }))) + return false + } + const settled = await settleTasksWithinMs(tasks, remainingMs) + errors.push(...settled.errors) + unfinished.push(...settled.timedOut.map((task) => ({ targetId: task.targetId, phase }))) + return settled.timedOut.length === 0 + } + + await runPhase('drain', sshShutdownTasks(targetIds)) + // Why a second drain after the join: a connect paused in old-session teardown still publishes its + // replacement session and opens a transport before it reaches the cancellation checkpoint, so the + // first drain can miss both. + if (await runPhase('in-flight-join', inFlight)) { + await runPhase('final-drain', sshShutdownTasks([...activeSessions.keys()])) + } + + if (errors.length > 0 || unfinished.length > 0) { + // Why one aggregate line: per-target logging on the quit path competes with the final flush for + // the little time that remains. + console.warn( + `[ssh] Shutdown drain finished with ${errors.length} error(s); unfinished: ${ + unfinished.map((entry) => `${entry.targetId}/${entry.phase}`).join(', ') || 'none' + }` + ) + } + return { unfinished, errors } +} + +// Why one entry point that returns rather than awaits: every in-memory transition the final store +// flush must snapshot happens synchronously here, before this returns, so the caller can start that +// flush with no await in between. Idempotent — a later call joins the same drain and repeats no +// state transition. +// +// Why no fence latch here: the committed quit path sets it before calling this, so it is already on +// for the snapshot below. Called without that gate (tests), this degrades to a plain drain. +export function beginSshShutdown(): Promise { + if (sshShutdownDrain) { + return sshShutdownDrain + } + const inFlight: SshShutdownTask[] = [ + ...[...connectInFlight.entries()].map(([targetId, attempt]) => ({ + targetId, + promise: attempt.promise + })), + ...[...resetRelayInFlight.entries()].map(([targetId, promise]) => ({ targetId, promise })), + ...[...testConnectionProbes].map((promise) => ({ targetId: '*probe', promise })) + ] + for (const targetId of Array.from(connectInFlight.keys())) { + invalidateConnectAttempt(targetId) + } + const targetIds = [...activeSessions.keys()] + // Why before any await: this is the whole point of the split. Each session marks its recovery lease + // detached in memory now, and the final flush persists it — the remote PTYs keep running. + const detachErrors: unknown[] = [] + for (const session of activeSessions.values()) { + // Why per-session: this runs synchronously inside a non-async will-quit listener, so one throw + // (teardownProviders -> webContents.send on a destroyed renderer, routine on quit) would escape + // it and skip every later session, the drain assignment, and the store flush that persists all + // of this. Collect and keep going; the drain reports them. + try { + session.beginShutdownDetach() + } catch (error) { + detachErrors.push(error) + } + } + sshShutdownDrain = drainSshShutdown(targetIds, inFlight, detachErrors) + return sshShutdownDrain +} diff --git a/src/main/ipc/ssh-target-crud-handlers.ts b/src/main/ipc/ssh-target-crud-handlers.ts new file mode 100644 index 00000000000..03c4e457394 --- /dev/null +++ b/src/main/ipc/ssh-target-crud-handlers.ts @@ -0,0 +1,79 @@ +import { ipcMain } from 'electron' +import type { SshConfigHostListArgs, SshRepoReadoption, SshTarget } from '../../shared/ssh-types' +import { + listUserSshConfigHostSummaries, + resolveUserSshConfigHost +} from '../ssh/ssh-config-host-picker' +import { rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { getCurrentMainWindow } from './ssh-ipc-context' +import { removeRegisteredSshTarget } from './ssh-session-teardown' + +// Why: add/import can re-adopt workspaces orphaned on a removed target id (see ssh-target-readoption); the renderer must refresh its repo list to surface them. +function takeRepoReadoptions(): SshRepoReadoption[] { + const store = getSshTargetRegistryStore() + if (!store || store.lastRepoReadoptions.length === 0) { + return [] + } + const repoReadoptions = store.lastRepoReadoptions + store.lastRepoReadoptions = [] + for (const targetId of new Set( + repoReadoptions.flatMap(({ oldTargetId, newTargetId }) => [oldTargetId, newTargetId]) + )) { + rotateSshProviderAuthority(targetId) + } + const win = getCurrentMainWindow() + if (win && !win.isDestroyed()) { + win.webContents.send('repos:changed') + } + return repoReadoptions +} + +export function registerSshTargetCrudHandlers(): void { + ipcMain.handle('ssh:listTargets', () => { + return getSshTargetRegistryStore()!.listTargets() + }) + + ipcMain.handle('ssh:listRemovedTargetLabels', () => { + return getSshTargetRegistryStore()!.listRemovedTargetLabels() + }) + + ipcMain.handle('ssh:addTarget', (_event, args: { target: Omit }) => { + const target = getSshTargetRegistryStore()!.addTarget(args.target) + // Why: re-adding a removed host can re-adopt orphaned workspaces; refresh the renderer's repo list so they move back onto the live host. + const repoReadoptions = takeRepoReadoptions() + return { target, repoReadoptions } + }) + + ipcMain.handle( + 'ssh:updateTarget', + (_event, args: { id: string; updates: Partial> }) => { + return getSshTargetRegistryStore()!.updateTarget(args.id, args.updates) + } + ) + + ipcMain.handle('ssh:removeTarget', async (_event, args: { id: string }) => { + await removeRegisteredSshTarget(args.id) + }) + + ipcMain.handle('ssh:importConfig', (_event, args?: { reAdopt?: boolean }) => { + const targets = getSshTargetRegistryStore()!.importFromSshConfig(args) + const repoReadoptions = takeRepoReadoptions() + return { targets, repoReadoptions } + }) + + // Why: add-host dialog picks one config entry to prefill the form; does not + // mutate the target store (bulk sync stays on Settings → Import). + ipcMain.handle('ssh:listConfigHosts', (_event, args?: SshConfigHostListArgs) => { + return listUserSshConfigHostSummaries( + getSshTargetRegistryStore()!.listTargets(), + args?.query, + getSshTargetRegistryStore()!.listSuppressedSshConfigAliases(), + { refresh: args?.refresh === true } + ) + }) + + ipcMain.handle('ssh:resolveConfigHost', (_event, args: { alias: string }) => { + return resolveUserSshConfigHost(args.alias) + }) +} diff --git a/src/main/ipc/ssh-target-lifecycle-queue.ts b/src/main/ipc/ssh-target-lifecycle-queue.ts new file mode 100644 index 00000000000..014a5037175 --- /dev/null +++ b/src/main/ipc/ssh-target-lifecycle-queue.ts @@ -0,0 +1,33 @@ +// Serializes disconnect/remove/terminate/reset for a single SSH target so they cannot interleave. +export const targetLifecycleInFlight = new Map>() + +export function runTargetLifecycle( + targetId: string, + operation: () => Promise +): Promise { + const prior = targetLifecycleInFlight.get(targetId) + const operationPromise = (async () => { + if (prior) { + await prior.catch(() => undefined) + } + await operation() + })() + let trackedPromise!: Promise + trackedPromise = operationPromise.finally(() => { + if (targetLifecycleInFlight.get(targetId) === trackedPromise) { + targetLifecycleInFlight.delete(targetId) + } + }) + targetLifecycleInFlight.set(targetId, trackedPromise) + return trackedPromise +} + +export async function awaitTargetLifecycle(targetId: string): Promise { + while (true) { + const lifecycle = targetLifecycleInFlight.get(targetId) + if (!lifecycle) { + return + } + await lifecycle.catch(() => undefined) + } +} diff --git a/src/main/ipc/ssh.ts b/src/main/ipc/ssh.ts index e31b852a751..4dce1b236b4 100644 --- a/src/main/ipc/ssh.ts +++ b/src/main/ipc/ssh.ts @@ -1,47 +1,21 @@ -/* oxlint-disable max-lines -- Why: co-locates SSH IPC handlers, port-forward broadcasting, and session lifecycle to keep the data flow obvious. */ -import { ipcMain, powerMonitor, type BrowserWindow } from 'electron' -import { appendFileSync } from 'node:fs' +import { ipcMain, type BrowserWindow } from 'electron' import type { Store } from '../persistence' import { SshConnectionStore } from '../ssh/ssh-connection-store' -import { - listUserSshConfigHostSummaries, - resolveUserSshConfigHost -} from '../ssh/ssh-config-host-picker' -import type { SshConnection, SshConnectionCallbacks } from '../ssh/ssh-connection' import { SshConnectionManager } from '../ssh/ssh-connection-manager' -import { SshRelaySession, type SshRelayAiVaultHostInfo } from '../ssh/ssh-relay-session' +import type { SshRelayAiVaultHostInfo } from '../ssh/ssh-relay-session' import type { SshAiVaultRelayListParams, SshAiVaultRelayTitleParams } from '../../shared/ssh-ai-vault-relay' import { SshPortForwardManager } from '../ssh/ssh-port-forward' -import type { - DetectedPort, - EnrichedDetectedPort, - SavedPortForward, - SshConfigHostListArgs, - SshRepoReadoption, - SshTarget, - SshConnectionStatus, - SshConnectionState, - DirectSshAuthority -} from '../../shared/ssh-types' -import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' -import { quitTeardownStartGate } from '../quit-teardown-start-gate' import { isRuntimeOwnedSshTargetId } from '../../shared/execution-host' +import { quitTeardownStartGate } from '../quit-teardown-start-gate' import { getSshTargetRegistryStore, - setSshActiveMultiplexerResolver, setSshTargetRegistryHandlers, setSshTargetRegistryStore } from '../ssh/ssh-target-registry' -// Why at module scope: this resolver is pure state lookup with no handler lifecycle, so -// installing it on import keeps it correct even before registerSshHandlers runs. -setSshActiveMultiplexerResolver( - (connectionId) => activeSessions.get(connectionId)?.getMux() ?? undefined -) - // Why re-exported: the registry moved to ../ssh/ssh-target-registry so the runtime can // read it without pulling ipcMain in, but many existing importers reference these from // here. Re-exporting keeps them working without a repo-wide rename. @@ -52,45 +26,51 @@ export { listRegisteredRemovedSshTargetLabels, listRegisteredSshTargets } from '../ssh/ssh-target-registry' -import { isAuthError } from '../ssh/ssh-connection-utils' -import { createCancelledConnectAttemptError } from '../ssh/ssh-connect-attempt-cancellation' -import { forceStopRelayForTarget } from '../ssh/ssh-relay-reset' -import { isSshPtyNotFoundError } from '../providers/ssh-pty-errors' -import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' import { registerSshBrowseHandler } from './ssh-browse' -import { - getConnectionIdsForWorktree, - enrichSshDetectedPorts, - enrichSshForwardEntries, - getWorktreeIdsForConnection -} from '../ports/ssh-advertised-url-enrichment' -import { advertisedUrlWatcher } from '../ports/advertised-url-watcher' -import { requestCredential, registerCredentialHandler } from './ssh-passphrase' -import { - clearProviderPtyState, - deletePtyOwnership, - getPtyIdsForConnection, - getSshPtyProvider -} from './pty' +import { registerCredentialHandler } from './ssh-passphrase' import type { OrcaRuntimeService } from '../runtime/orca-runtime' import { initializeSshConnectionGenerationSession, resetSshConnectionGenerations } from '../ssh/ssh-connection-generation' +import { resetSshProviderAuthorities } from '../ssh/ssh-provider-authority' +import { activeSessions } from './ssh-active-relay-sessions' import { - getSshProviderAuthority, - isCurrentSshProviderAuthority, - resetSshProviderAuthorities, - rotateSshProviderAuthority -} from '../ssh/ssh-provider-authority' - -let connectionManager: SshConnectionManager | null = null -let portForwardManager: SshPortForwardManager | null = null -let persistedStore: Store | null = null -let advertisedUrlWatcherUnsubscribe: (() => void) | null = null -let powerMonitorUnsubscribe: (() => void) | null = null -let currentGetMainWindow: () => BrowserWindow | null = () => null -let currentRuntime: OrcaRuntimeService | undefined + registerAdvertisedUrlRefresh, + unregisterAdvertisedUrlRefresh +} from './ssh-advertised-url-refresh' +import { + connectInFlight, + credentialRequestedForTarget, + pendingTransportReconnects, + resetRelayInFlight, + testConnectionProbes, + testingTargets +} from './ssh-connect-attempt-registry' +import { createSshConnectionCallbacks } from './ssh-connection-state-callbacks' +import { registerSshConnectionHandlers } from './ssh-connection-handlers' +import { + registerPowerMonitorReconnect, + unregisterPowerMonitorReconnect +} from './ssh-host-sleep-reconnect' +import { + connectionManager, + getCurrentMainWindow, + portForwardManager, + setConnectionManager, + setCurrentGetMainWindow, + setCurrentRuntime, + setPersistedStore, + setPortForwardManager +} from './ssh-ipc-context' +import { registerSshPortForwardHandlers } from './ssh-port-forward-handlers' +import { persistPortForwardsWithUnrestored } from './ssh-port-forward-persistence' +import { clearRelayLostBackoff, relayLostBackoff } from './ssh-relay-lost-backoff' +import { refreshActiveRelaySessions } from './ssh-relay-session-callbacks' +import { broadcastPortForwards, relayStateOverrides } from './ssh-renderer-broadcast' +import { resetSshShutdownDrain } from './ssh-shutdown-drain' +import { registerSshTargetCrudHandlers } from './ssh-target-crud-handlers' +import { targetLifecycleInFlight } from './ssh-target-lifecycle-queue' const SSH_IPC_CHANNELS = [ 'ssh:listTargets', @@ -115,45 +95,6 @@ const SSH_IPC_CHANNELS = [ 'ssh:listDetectedPorts' ] as const -// Why: keep this outside registerSshHandlers so a BrowserWindow recreation mid-connect doesn't split credential tracking. -const credentialRequestedForTarget = new Set() - -function getCurrentMainWindow(): BrowserWindow | null { - return currentGetMainWindow() -} - -export async function disconnectRegisteredSshTarget(targetId: string): Promise { - invalidateConnectAttempt(targetId) - await runTargetLifecycle(targetId, () => - teardownSshTargetTransport(targetId, (session) => session.detachAndPersist()) - ) -} - -export async function removeRegisteredSshTarget(targetId: string): Promise { - const store = getSshTargetRegistryStore() - if (!store) { - return - } - invalidateConnectAttempt(targetId) - await runTargetLifecycle(targetId, async () => { - try { - // Why: removal is destructive; dispose so remote PTYs cannot reattach to a deleted target. - await teardownSshTargetTransport(targetId, (session) => session.disposeAndPersist()) - } catch (err) { - // Why: a failed disconnect must not block metadata removal, else the target lingers in the store with uncleaned leases. - console.warn( - `[ssh] Failed to disconnect removed target ${targetId}: ${err instanceof Error ? err.message : String(err)}` - ) - } - persistedStore?.removeSshRemotePtyLeases(targetId) - store.removeTarget(targetId) - }) -} - -// One session per SSH target owns the whole relay lifecycle (mux, providers, abort controller, state machine). -const activeSessions = new Map() -const targetLifecycleInFlight = new Map>() - export function getActiveSshAiVaultHostInfo(targetId: string): SshRelayAiVaultHostInfo | null { if (isRuntimeOwnedSshTargetId(targetId)) { return null @@ -201,757 +142,6 @@ export async function requestActiveSshAiVaultSessionTitles( return session.requestAiVaultSessionTitles(params, options) } -function runTargetLifecycle(targetId: string, operation: () => Promise): Promise { - const prior = targetLifecycleInFlight.get(targetId) - const operationPromise = (async () => { - if (prior) { - await prior.catch(() => undefined) - } - await operation() - })() - let trackedPromise!: Promise - trackedPromise = operationPromise.finally(() => { - if (targetLifecycleInFlight.get(targetId) === trackedPromise) { - targetLifecycleInFlight.delete(targetId) - } - }) - targetLifecycleInFlight.set(targetId, trackedPromise) - return trackedPromise -} - -async function awaitTargetLifecycle(targetId: string): Promise { - while (true) { - const lifecycle = targetLifecycleInFlight.get(targetId) - if (!lifecycle) { - return - } - await lifecycle.catch(() => undefined) - } -} - -async function teardownSshTargetTransport( - targetId: string, - teardown: (session: SshRelaySession) => void | Promise -): Promise { - let transportDisconnect: Promise<{ ok: true } | { ok: false; error: unknown }> - try { - transportDisconnect = Promise.resolve(connectionManager?.disconnect(targetId)).then( - () => ({ ok: true }) as const, - (error: unknown) => ({ ok: false, error }) as const - ) - } catch (error) { - transportDisconnect = Promise.resolve({ ok: false, error }) - } - const sessionTeardown = teardownActiveSshSession(targetId, teardown).then( - () => ({ ok: true }) as const, - (error: unknown) => ({ ok: false, error }) as const - ) - const [disconnectResult, teardownResult] = await Promise.all([ - transportDisconnect, - sessionTeardown - ]) - if (!teardownResult.ok) { - throw teardownResult.error - } - if (!disconnectResult.ok) { - throw disconnectResult.error - } -} - -async function teardownActiveSshSession( - targetId: string, - teardown: (session: SshRelaySession) => void | Promise -): Promise { - const session = activeSessions.get(targetId) - if (!session) { - return - } - let teardownError: { error: unknown } | null = null - try { - // Why: await port teardown so local listeners are released before disconnect/remove completes, else an immediate reconnect hits EADDRINUSE. - await portForwardManager?.removeAllForwards(targetId) - } catch (error) { - teardownError = { error } - } - try { - await teardown(session) - } catch (error) { - teardownError ??= { error } - } - if (activeSessions.get(targetId) === session) { - activeSessions.delete(targetId) - clearRelayLostBackoff(targetId) - clearRelayStateOverride(targetId) - } - if (teardownError) { - throw teardownError.error - } -} - -// Why: a dropped session must detach, not just leave activeSessions — detach releases the SSH PTY -// consumer identity so the next connect reclaims its owner lease instead of minting a new one. -// Why awaited, and why the map entry outlives the await: a retry can start the moment this returns, -// so it must either find the session (and await this same latched teardown at the existing-session -// path) or find nothing because the 'detached' lease write is already durable. Deleting first lets a -// fast reconnect mark leases 'attached' and then have this session's late 'detached' write clobber it. -async function abandonFailedSshSession(targetId: string, session: SshRelaySession): Promise { - // Why: detachAndPersist transitions recovery ownership synchronously; only durability is awaited. - try { - await session.detachAndPersist() - } catch (error) { - // Why: a teardown throw must not mask the connect error the caller is about to rethrow. - console.warn( - `[ssh] Failed to detach abandoned session for ${targetId}: ${error instanceof Error ? error.message : String(error)}` - ) - } - if (activeSessions.get(targetId) === session) { - activeSessions.delete(targetId) - } -} - -// Why: a connect cancelled after its transport already opened still owns that transport and its -// unpublished session — nothing else will reach them, so it has to close them itself. Why only the -// connection it minted, and by identity: disconnecting by target id (or closing a transport it merely -// reused) would tear down the replacement connect's live transport. -async function abandonCancelledConnectAttempt( - targetId: string, - session: SshRelaySession, - mintedConnection: SshConnection | null -): Promise { - // Why the identity guard: every path that removes a session from activeSessions detaches it first, - // so re-detaching a superseded session would only clobber the replacement's lease state. - if (activeSessions.get(targetId) === session) { - await abandonFailedSshSession(targetId, session) - } - if (!mintedConnection) { - return - } - try { - await connectionManager!.disconnectConnection(targetId, mintedConnection) - } catch (error) { - // Why: the caller is about to throw the cancellation; a teardown throw must not replace it. - console.warn( - `[ssh] Failed to disconnect cancelled connect transport for ${targetId}: ${error instanceof Error ? error.message : String(error)}` - ) - } -} - -function relayGracePeriodForTarget(target: SshTarget | null | undefined): number | undefined { - return target?.relayGracePeriodSeconds -} - -// Why: tabs must share one connect, while a disconnect must invalidate that -// attempt so its late continuation cannot clobber a replacement. -type ConnectAttempt = { - authority: DirectSshAuthority - promise: Promise -} - -const connectInFlight = new Map() -const pendingTransportReconnects = new Set() - -// Why the quit gate rather than a local latch: "the committed quit has begun" already has an owner, -// and a private copy could be set by something that is not actually quitting — leaving SSH connects -// refused for the rest of the process lifetime. -function assertSshConnectsNotFenced(): void { - if (quitTeardownStartGate.hasStarted()) { - throw new Error('SSH connects are closed for app shutdown') - } -} - -function invalidateConnectAttempt(targetId: string): void { - rotateSshProviderAuthority(targetId) - pendingTransportReconnects.delete(targetId) - connectInFlight.delete(targetId) - credentialRequestedForTarget.delete(targetId) -} - -function isCurrentConnectAttempt(targetId: string, authority: DirectSshAuthority): boolean { - return authority.targetId === targetId && isCurrentSshProviderAuthority(authority) -} - -// Why: publish reset's teardown/force-stop/disconnect lifecycle so new connects and duplicate resets can't race it. -const resetRelayInFlight = new Map>() - -// Why: ssh:testConnection connects then disconnects; suppressing broadcasts during the test avoids worktree cards flashing connected → disconnected. -const testingTargets = new Set() -const testConnectionProbes = new Set>() - -// Why: without backoff, a relay channel that keeps dying reconnects as fast as the network allows, hammering local + remote sshd; track attempts and back off to end the loop recoverably. -type RelayLostBackoffState = { - attempts: number - reconnectTimer: ReturnType | null - stabilizedTimer: ReturnType | null -} -const relayLostBackoff = new Map() -const relayStateOverrides = new Map() -const RELAY_LOST_MAX_ATTEMPTS = 6 -const RELAY_LOST_BASE_DELAY_MS = 500 -const RELAY_LOST_MAX_DELAY_MS = 15_000 -// Why: a reconnect whose mux dies within this window was a flap, not a recovery — don't reset the attempt counter. 5s covers provider re-registration + PTY reattach. -const RELAY_LOST_STABILIZED_MS = 5_000 -// Why: transport states the SSH ladder never leaves on its own — waiting for a relay redeploy past one of these is an unbounded loop. -const TRANSPORT_TERMINAL_STATUSES = new Set([ - 'disconnected', - 'auth-failed', - 'reconnection-failed', - 'error' -]) - -function clearRelayLostBackoff(targetId: string): void { - const state = relayLostBackoff.get(targetId) - if (state?.reconnectTimer) { - clearTimeout(state.reconnectTimer) - } - if (state?.stabilizedTimer) { - clearTimeout(state.stabilizedTimer) - } - relayLostBackoff.delete(targetId) -} - -function broadcastSshState( - getMainWindow: () => BrowserWindow | null, - targetId: string, - state: SshConnectionState -): void { - // Why: runtime-owned (ephemeral-VM) targets are hidden from the renderer, so broadcasting their state only triggers wasted listTargets() lookups. - if (isRuntimeOwnedSshTargetId(targetId)) { - currentRuntime?.invalidateSshWorktreeScanCache?.(targetId) - return - } - const enrichedState = withSshRemotePlatform(targetId, state) - const win = getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('ssh:state-changed', { targetId, state: enrichedState }) - } - // Why: paired remote clients have no ssh:state-changed IPC; without this their terminals keep a stale reconnect overlay. - currentRuntime?.notifySshStateChanged?.(targetId, enrichedState) -} - -function withSshRemotePlatform(targetId: string, state: SshConnectionState): SshConnectionState { - const remotePlatform = activeSessions.get(targetId)?.getHostPlatform()?.os - const authority = getSshProviderAuthority(targetId) - return { - ...state, - targetId, - providerEpoch: authority.providerEpoch, - connectionGeneration: authority.connectionGeneration, - ...(remotePlatform ? { remotePlatform } : {}) - } -} - -function publishRelayOverride( - getMainWindow: () => BrowserWindow | null, - targetId: string, - status: SshConnectionStatus, - error: string | null, - reconnectAttempt: number -): void { - const state = withSshRemotePlatform(targetId, { targetId, status, error, reconnectAttempt }) - relayStateOverrides.set(targetId, state) - broadcastSshState(getMainWindow, targetId, state) -} - -function clearRelayStateOverride(targetId: string): void { - relayStateOverrides.delete(targetId) -} - -function connectionSupportsFolderDownload(targetId: string): boolean { - // Why: connections without an explicit transport are ssh2-shaped; only a confirmed system-SSH transport lacks the SFTP-only capability. - return connectionManager?.getConnection(targetId)?.usesSystemSshTransport?.() !== true -} - -function getPublicSshState(targetId: string): SshConnectionState | undefined { - const state = relayStateOverrides.get(targetId) ?? connectionManager!.getState(targetId) - return state ? withSshRemotePlatform(targetId, state) : undefined -} - -function broadcastPortForwards(getMainWindow: () => BrowserWindow | null, targetId: string): void { - const win = getMainWindow() - if (!win || win.isDestroyed()) { - return - } - win.webContents.send('ssh:port-forwards-changed', { - targetId, - forwards: listForwardsEnriched(targetId) - }) -} - -function broadcastDetectedPorts( - getMainWindow: () => BrowserWindow | null, - targetId: string, - ports: DetectedPort[], - options?: Parameters[3] -): void { - const win = getMainWindow() - if (!win || win.isDestroyed()) { - return - } - win.webContents.send('ssh:detected-ports-changed', { - targetId, - ports: enrichDetected(targetId, ports, options) - }) -} - -function listForwardsEnriched(targetId: string): ReturnType { - const raw = portForwardManager!.listForwards(targetId) - if (!persistedStore) { - return raw - } - return enrichSshForwardEntries(raw, getWorktreeIdsForConnection(persistedStore, targetId)) -} - -function enrichDetected( - targetId: string, - ports: DetectedPort[], - options?: Parameters[3] -): EnrichedDetectedPort[] { - if (!persistedStore) { - return ports - } - return enrichSshDetectedPorts( - ports, - getWorktreeIdsForConnection(persistedStore, targetId), - undefined, - options - ) -} - -// Why: after user add/remove/update the runtime manager is the source of truth — persist exactly its entries (unrestored ones handled by a separate helper). -function persistPortForwards(targetId: string): void { - const active = portForwardManager!.listForwards(targetId) - const saved: SavedPortForward[] = active.map((f) => ({ - localPort: f.localPort, - remoteHost: f.remoteHost, - remotePort: f.remotePort, - label: f.label - })) - getSshTargetRegistryStore()!.updateTarget(targetId, { - portForwards: saved.length > 0 ? saved : undefined - }) -} - -// Why: keep forwards that failed to restore in the persisted list so they retry on next reconnect instead of being silently dropped. -function persistPortForwardsWithUnrestored(targetId: string): void { - const active = portForwardManager!.listForwards(targetId) - const activeKeys = new Set(active.map((f) => `${f.localPort}:${f.remoteHost}:${f.remotePort}`)) - - const existing = getSshTargetRegistryStore()!.getTarget(targetId)?.portForwards ?? [] - const unrestored = existing.filter( - (pf) => !activeKeys.has(`${pf.localPort}:${pf.remoteHost}:${pf.remotePort}`) - ) - - const saved: SavedPortForward[] = [ - ...active.map((f) => ({ - localPort: f.localPort, - remoteHost: f.remoteHost, - remotePort: f.remotePort, - label: f.label - })), - ...unrestored - ] - getSshTargetRegistryStore()!.updateTarget(targetId, { - portForwards: saved.length > 0 ? saved : undefined - }) -} - -async function restorePortForwards( - targetId: string, - getMainWindow: () => BrowserWindow | null -): Promise { - const target = getSshTargetRegistryStore()!.getTarget(targetId) - if (!target?.portForwards?.length) { - return - } - const conn = connectionManager!.getConnection(targetId) - if (!conn) { - return - } - - // Why: keep failed restores in persisted state — a failure may be transient (port temporarily busy), so retry on next reconnect. - for (const saved of target.portForwards) { - // Why: a reconnect mid-loop swaps the connection object; bail on identity change so we don't add forwards to a stale conn (leaking listeners). - if (connectionManager!.getConnection(targetId) !== conn) { - return - } - try { - await portForwardManager!.addForward( - targetId, - conn, - saved.localPort, - saved.remoteHost, - saved.remotePort, - saved.label - ) - } catch (err) { - console.warn( - `[ssh] Failed to restore forward :${saved.localPort} → ${saved.remoteHost}:${saved.remotePort}: ${err instanceof Error ? err.message : String(err)}` - ) - } - } - - persistPortForwardsWithUnrestored(targetId) - broadcastPortForwards(getMainWindow, targetId) -} - -function registerAdvertisedUrlRefresh(getMainWindow: () => BrowserWindow | null): void { - advertisedUrlWatcherUnsubscribe?.() - // Why: SSH port scans only emit on raw host/port/PID changes, but a terminal can print the advertised URL later, so the watcher must also refresh the renderer. - advertisedUrlWatcherUnsubscribe = advertisedUrlWatcher.onDidChange(({ worktreeId }) => { - if (!persistedStore) { - return - } - for (const targetId of getConnectionIdsForWorktree(persistedStore, worktreeId)) { - const session = activeSessions.get(targetId) - if (!session) { - continue - } - const scanner = session.getPortScanner() - if (scanner) { - // Why: watcher changes can arrive before the next SSH scan refreshes listener PIDs, so don't validate PIDs against cached scanner rows. - broadcastDetectedPorts(getMainWindow, targetId, scanner.getDetectedPorts(targetId), { - validatePid: false - }) - } - broadcastPortForwards(getMainWindow, targetId) - } - }) -} - -// Why: macOS can resume before the network is back, so a failed first probe gets one retry before the link is declared dead (#7773). -const RESUME_PROBE_TIMEOUT_MS = 5_000 -const RESUME_PROBE_ATTEMPTS = 2 - -async function isRelayLinkAliveAfterResume(session: SshRelaySession): Promise { - const mux = session.getMux() - if (!mux || mux.isDisposed()) { - return false - } - for (let attempt = 0; attempt < RESUME_PROBE_ATTEMPTS; attempt++) { - if (await mux.probeLiveness(RESUME_PROBE_TIMEOUT_MS)) { - return true - } - } - return false -} - -function registerPowerMonitorReconnect(): void { - powerMonitorUnsubscribe?.() - const onSuspend = (): void => { - for (const session of activeSessions.values()) { - session.prepareForHostSleep() - } - } - const onResume = (): void => { - for (const [targetId, session] of activeSessions) { - const manager = connectionManager - const conn = manager?.getConnection(targetId) - if (!conn) { - continue - } - void (async () => { - // Why: unconditional reconnect on wake tore down live sessions and flashed the overlay (#7773); only reconnect if the relay link actually died during sleep. - if (await isRelayLinkAliveAfterResume(session)) { - return - } - // Why: the probe can take ~10s; bail if the session/connection was replaced or torn down meanwhile, else we'd resurrect it. - if (activeSessions.get(targetId) !== session || manager?.getConnection(targetId) !== conn) { - return - } - try { - await manager?.reconnect(targetId) - } catch (err) { - console.warn( - `[ssh] Failed to reconnect ${targetId} after system resume: ${ - err instanceof Error ? err.message : String(err) - }` - ) - } - })() - } - } - powerMonitor.on('suspend', onSuspend) - powerMonitor.on('resume', onResume) - powerMonitorUnsubscribe = () => { - powerMonitor.off('suspend', onSuspend) - powerMonitor.off('resume', onResume) - } -} - -function createSshConnectionCallbacks(): SshConnectionCallbacks { - return { - onCredentialRequest: (targetId, kind, detail) => { - credentialRequestedForTarget.add(targetId) - return requestCredential(getCurrentMainWindow, targetId, kind, detail) - }, - onStateChange: (targetId: string, state: SshConnectionState) => { - if (testingTargets.has(targetId)) { - return - } - - // Why: an SSH reconnect must re-deploy the relay and rebuild providers; the guard below fires only for real reconnects, not an explicit connect's 'deploying'. - const session = activeSessions.get(targetId) - const sessionState = session?.getState() - const transportReconnectStarted = - state.status === 'reconnecting' && - (sessionState === 'ready' || sessionState === 'reconnecting') && - !pendingTransportReconnects.has(targetId) - if (transportReconnectStarted) { - rotateSshProviderAuthority(targetId) - pendingTransportReconnects.add(targetId) - } else if ( - state.status === 'disconnected' || - state.status === 'auth-failed' || - state.status === 'reconnection-failed' || - state.status === 'error' - ) { - pendingTransportReconnects.delete(targetId) - } - const completedTransportReconnect = - state.status === 'connected' && pendingTransportReconnects.delete(targetId) - const shouldReconnectRelay = - session !== undefined && - completedTransportReconnect && - state.reconnectAttempt === 0 && - (sessionState === 'ready' || sessionState === 'reconnecting') - const relayReconnectAlreadyInFlight = - !completedTransportReconnect && - state.status === 'connected' && - sessionState === 'reconnecting' && - relayStateOverrides.has(targetId) - - if (shouldReconnectRelay) { - // Why: this branch redeploys the relay itself over a fresh transport, so any pending relay-lost retry is stale — dropping it also gives the new transport generation a full attempt budget. - clearRelayLostBackoff(targetId) - // Why: SSH connects before the relay providers rebuild; keep renderer actions gated until SshRelaySession reaches ready again. - publishRelayOverride( - getCurrentMainWindow, - targetId, - 'reconnecting', - 'Relay channel reconnecting...', - state.reconnectAttempt - ) - } else if (relayReconnectAlreadyInFlight) { - // Why: duplicate connected notifications belong to the same socket generation and must not expose providers before relay recovery finishes. - return - } else if ( - state.status === 'connected' && - session !== undefined && - sessionState !== 'ready' && - !completedTransportReconnect && - connectInFlight.has(targetId) - ) { - // Why: the raw SSH transport reaches 'connected' before the relay session establishes during an - // explicit connect. Forwarding it makes the renderer treat the host as fully up — it remounts - // SSH panes (-> window.api.ssh.connect) and fires connected-gated data reads before any provider - // exists. On a permanent relay-deploy failure that premature 'connected' drives an unbounded - // reconnect loop. Hold it at 'deploying-relay'; the in-flight doConnect broadcasts the - // authoritative 'connected' directly (bypassing this callback) after establish() succeeds, or a - // terminal state on failure. The connectInFlight gate keeps this scoped to a live connect, so a - // stray raw 'connected' with no follow-up (e.g. a transport blip on a session left 'idle' by a - // relay version mismatch) is never wedged at 'deploying-relay'. - clearRelayStateOverride(targetId) - broadcastSshState(getCurrentMainWindow, targetId, { - targetId, - status: 'deploying-relay', - error: state.error, - reconnectAttempt: state.reconnectAttempt - }) - } else { - clearRelayStateOverride(targetId) - broadcastSshState(getCurrentMainWindow, targetId, state) - } - - if (!session) { - return - } - // Why: allow reconnect from both 'ready' and 'reconnecting'; without the latter, a failed relay deploy would permanently brick the session. - if (shouldReconnectRelay) { - const target = getSshTargetRegistryStore()?.getTarget(targetId) - const conn = connectionManager?.getConnection(targetId) - if (conn) { - void session.reconnect(conn, relayGracePeriodForTarget(target)) - } - } - } - } -} - -function broadcastDetectedPortsFromCurrentWindow( - targetId: string, - ports: DetectedPort[], - _platform: string -): void { - broadcastDetectedPorts(getCurrentMainWindow, targetId, ports) -} - -function configureRelaySessionCallbacks(session: SshRelaySession): void { - session.setOnTerminalRelayError((tid, err) => { - clearRelayLostBackoff(tid) - if (activeSessions.get(tid)?.getState() !== 'deploying') { - rotateSshProviderAuthority(tid) - } - console.warn( - `[ssh] Terminal relay error for ${tid}: ${err.message}; skipping reconnect backoff.` - ) - publishRelayOverride(getCurrentMainWindow, tid, 'error', err.message, 0) - }) - - session.setOnRelayLost((tid) => { - const s = activeSessions.get(tid) - if (!s) { - return - } - const c = connectionManager?.getConnection(tid) - if (!c) { - return - } - const t = getSshTargetRegistryStore()?.getTarget(tid) - - // Why: bounded exponential backoff — without it, a remote bug that closes every fresh --connect channel becomes an infinite relay-deploy loop. - const state = relayLostBackoff.get(tid) ?? { - attempts: 0, - reconnectTimer: null, - stabilizedTimer: null - } - if (state.stabilizedTimer) { - clearTimeout(state.stabilizedTimer) - state.stabilizedTimer = null - } - if (state.reconnectTimer) { - return - } - rotateSshProviderAuthority(tid) - - // Why: re-deploying the relay rides the SSH transport, so while the transport is itself down no attempt - // can succeed. Waiting at the max delay without consuming the budget keeps a flapping host off the - // manual-reconnect banner, which would tell the user to act on a link that is still auto-recovering. - const transportStatus = connectionManager?.getState(tid)?.status - const transportConnected = transportStatus === 'connected' - if (transportConnected && state.attempts >= RELAY_LOST_MAX_ATTEMPTS) { - console.warn( - `[ssh] Relay channel for ${tid} kept dying across ${state.attempts} attempts; giving up. User must reconnect manually.` - ) - relayLostBackoff.delete(tid) - // Why: surface the failure — a live SSH connection with a dead relay is otherwise invisible (typing in remote terminals just stops working). - publishRelayOverride( - getCurrentMainWindow, - tid, - 'error', - 'Relay channel kept dropping. Click Reconnect on the SSH target before retrying.', - 0 - ) - return - } - - const scheduleRelayRedeploy = (delay: number, attemptCharged: boolean): void => { - state.reconnectTimer = setTimeout(() => { - state.reconnectTimer = null - relayLostBackoff.set(tid, state) - const liveConn = connectionManager?.getConnection(tid) - if (!liveConn || !activeSessions.has(tid)) { - clearRelayLostBackoff(tid) - return - } - const status = connectionManager?.getState(tid)?.status - if (status === 'connected') { - if (!attemptCharged) { - // Why: waiting is free, but the deploy it defers is real — charge it here so a transport that - // flaps back to 'connected' can't redeploy forever on an uncharged budget. - state.attempts += 1 - } - void s.reconnect(liveConn, relayGracePeriodForTarget(t)) - return - } - if (status === undefined || TRANSPORT_TERMINAL_STATUSES.has(status)) { - // Why: the transport gave up for good; its own state is what the user acts on, so stop waiting for a redeploy that can never run. - clearRelayLostBackoff(tid) - return - } - // Why: still mid-transition — re-arm at the max delay without consuming an attempt. It ends once - // the transport settles: 'connected' redeploys, a terminal status or a dropped session clears above. - scheduleRelayRedeploy(RELAY_LOST_MAX_DELAY_MS, false) - }, delay) - relayLostBackoff.set(tid, state) - } - - if (!transportConnected) { - publishRelayOverride( - getCurrentMainWindow, - tid, - 'reconnecting', - 'Relay channel lost. Reconnecting...', - state.attempts - ) - scheduleRelayRedeploy(RELAY_LOST_MAX_DELAY_MS, false) - console.warn( - `[ssh] Relay channel for ${tid} lost while the SSH transport is ${transportStatus ?? 'unknown'}; waiting ${RELAY_LOST_MAX_DELAY_MS}ms without consuming an attempt` - ) - return - } - - const delay = Math.min(RELAY_LOST_BASE_DELAY_MS * 2 ** state.attempts, RELAY_LOST_MAX_DELAY_MS) - state.attempts += 1 - publishRelayOverride( - getCurrentMainWindow, - tid, - 'reconnecting', - 'Relay channel lost. Reconnecting...', - state.attempts - ) - scheduleRelayRedeploy(delay, true) - console.warn( - `[ssh] Relay channel for ${tid} lost; reconnect attempt ${state.attempts}/${RELAY_LOST_MAX_ATTEMPTS} in ${delay}ms` - ) - }) - - // Why: fires after both establish() and reconnect() reach 'ready'; re-create persisted port forwards so they survive restarts and blips. - session.setOnReady((tid) => { - const state = relayLostBackoff.get(tid) - if (state) { - if (state.stabilizedTimer) { - clearTimeout(state.stabilizedTimer) - } - // Why: stabilization counts post-ready uptime; slow deploy time before `ready` doesn't prove the new relay survived real work. - state.stabilizedTimer = setTimeout(() => { - const current = relayLostBackoff.get(tid) - if (current === state && !current.reconnectTimer) { - relayLostBackoff.delete(tid) - } - }, RELAY_LOST_STABILIZED_MS) - relayLostBackoff.set(tid, state) - } - clearRelayStateOverride(tid) - if (!testingTargets.has(tid)) { - broadcastSshState(getCurrentMainWindow, tid, { - targetId: tid, - status: 'connected', - error: null, - reconnectAttempt: 0, - supportsFolderDownload: connectionSupportsFolderDownload(tid) - }) - } - currentRuntime?.notifySshRelayReady?.(tid) - void restorePortForwards(tid, getCurrentMainWindow) - }) -} - -function refreshActiveRelaySessions(): void { - if (!persistedStore || !portForwardManager) { - return - } - for (const session of activeSessions.values()) { - session.refreshEnvironment( - getCurrentMainWindow, - persistedStore, - portForwardManager, - currentRuntime, - broadcastDetectedPortsFromCurrentWindow - ) - configureRelaySessionCallbacks(session) - } -} - export function registerSshHandlers( store: Store, getMainWindow: () => BrowserWindow | null, @@ -963,10 +153,10 @@ export function registerSshHandlers( ipcMain.removeHandler(ch) } - currentGetMainWindow = getMainWindow - currentRuntime = runtime + setCurrentGetMainWindow(getMainWindow) + setCurrentRuntime(runtime) setSshTargetRegistryStore(new SshConnectionStore(store)) - persistedStore = store + setPersistedStore(store) registerAdvertisedUrlRefresh(getCurrentMainWindow) registerCredentialHandler(getCurrentMainWindow) @@ -975,10 +165,10 @@ export function registerSshHandlers( if (connectionManager) { connectionManager.setCallbacks(callbacks) } else { - connectionManager = new SshConnectionManager(callbacks) + setConnectionManager(new SshConnectionManager(callbacks)) } - portForwardManager ??= new SshPortForwardManager() - portForwardManager.setCallbacks({ + setPortForwardManager(portForwardManager ?? new SshPortForwardManager()) + portForwardManager!.setCallbacks({ onForwardClosed: (entry, reason) => { if (reason.kind === 'unexpected-exit') { console.warn( @@ -995,776 +185,23 @@ export function registerSshHandlers( registerPowerMonitorReconnect() registerSshBrowseHandler(() => connectionManager) - // ── Target CRUD ──────────────────────────────────────────────────── + registerSshTargetCrudHandlers() + registerSshConnectionHandlers() + registerSshPortForwardHandlers() - // Why: add/import can re-adopt workspaces orphaned on a removed target id (see ssh-target-readoption); the renderer must refresh its repo list to surface them. - function takeRepoReadoptions(): SshRepoReadoption[] { - const store = getSshTargetRegistryStore() - if (!store || store.lastRepoReadoptions.length === 0) { - return [] - } - const repoReadoptions = store.lastRepoReadoptions - store.lastRepoReadoptions = [] - for (const targetId of new Set( - repoReadoptions.flatMap(({ oldTargetId, newTargetId }) => [oldTargetId, newTargetId]) - )) { - rotateSshProviderAuthority(targetId) - } - const win = getCurrentMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('repos:changed') - } - return repoReadoptions + return { + connectionManager: connectionManager!, + sshStore: getSshTargetRegistryStore() as SshConnectionStore } - - ipcMain.handle('ssh:listTargets', () => { - return getSshTargetRegistryStore()!.listTargets() - }) - - ipcMain.handle('ssh:listRemovedTargetLabels', () => { - return getSshTargetRegistryStore()!.listRemovedTargetLabels() - }) - - ipcMain.handle('ssh:addTarget', (_event, args: { target: Omit }) => { - const target = getSshTargetRegistryStore()!.addTarget(args.target) - // Why: re-adding a removed host can re-adopt orphaned workspaces; refresh the renderer's repo list so they move back onto the live host. - const repoReadoptions = takeRepoReadoptions() - return { target, repoReadoptions } - }) - - ipcMain.handle( - 'ssh:updateTarget', - (_event, args: { id: string; updates: Partial> }) => { - return getSshTargetRegistryStore()!.updateTarget(args.id, args.updates) - } - ) - - ipcMain.handle('ssh:removeTarget', async (_event, args: { id: string }) => { - await removeRegisteredSshTarget(args.id) - }) - - ipcMain.handle('ssh:importConfig', (_event, args?: { reAdopt?: boolean }) => { - const targets = getSshTargetRegistryStore()!.importFromSshConfig(args) - const repoReadoptions = takeRepoReadoptions() - return { targets, repoReadoptions } - }) - - // Why: add-host dialog picks one config entry to prefill the form; does not - // mutate the target store (bulk sync stays on Settings → Import). - ipcMain.handle('ssh:listConfigHosts', (_event, args?: SshConfigHostListArgs) => { - return listUserSshConfigHostSummaries( - getSshTargetRegistryStore()!.listTargets(), - args?.query, - getSshTargetRegistryStore()!.listSuppressedSshConfigAliases(), - { refresh: args?.refresh === true } - ) - }) - - ipcMain.handle('ssh:resolveConfigHost', (_event, args: { alias: string }) => { - return resolveUserSshConfigHost(args.alias) - }) - - // ── Connection lifecycle ─────────────────────────────────────────── - - async function connectTarget(targetId: string): Promise { - const e2eProbePath = process.env.ORCA_E2E_FORBID_LOCAL_SSH_CONNECT_PROBE - if (e2eProbePath) { - appendFileSync(e2eProbePath, `${JSON.stringify(targetId)}\n`) - throw new Error('e2e_forbidden_local_ssh_connect') - } - // Why: fence callers that entered before a same-turn disconnect/reset but resume after its cleanup. - const admissionAuthority = getSshProviderAuthority(targetId) - await awaitTargetLifecycle(targetId) - const reset = resetRelayInFlight.get(targetId) - if (reset) { - await reset - } - - // Why: serialize concurrent ssh:connect for the same target; interleaved connects otherwise leak the first session. - const existing = connectInFlight.get(targetId) - let replacePendingTransport = false - if (existing) { - if (isCurrentConnectAttempt(targetId, existing.authority)) { - return existing.promise - } - } - if (!isCurrentConnectAttempt(targetId, admissionAuthority)) { - throw createCancelledConnectAttemptError() - } - const observedAuthority = admissionAuthority - if (existing) { - if (connectInFlight.get(targetId) === existing) { - connectInFlight.delete(targetId) - replacePendingTransport = true - } - } - if (!isCurrentSshProviderAuthority(observedAuthority)) { - throw createCancelledConnectAttemptError() - } - // Why: the shutdown drain fences and snapshots synchronously, so a connect either registers in - // connectInFlight below (and gets joined) or fails here — it can never slip between the two. - assertSshConnectsNotFenced() - - pendingTransportReconnects.delete(targetId) - const promise = doConnect(targetId, replacePendingTransport) - const attempt = { authority: getSshProviderAuthority(targetId), promise } - connectInFlight.set(targetId, attempt) - try { - return await promise - } finally { - if (connectInFlight.get(targetId) === attempt) { - connectInFlight.delete(targetId) - } - } - } - - setSshTargetRegistryHandlers({ - connect: connectTarget, - getState: (targetId: string) => getPublicSshState(targetId) - }) - - ipcMain.handle('ssh:connect', async (_event, args: { targetId: string }) => { - return connectTarget(args.targetId) - }) - - async function doConnect( - targetId: string, - replacePendingTransport = false - ): Promise { - const target = getSshTargetRegistryStore()!.getTarget(targetId) - if (!target) { - throw new Error(`SSH target "${targetId}" not found`) - } - - const existingSession = activeSessions.get(targetId) - const existingState = connectionManager!.getState(targetId) - const existingMux = existingSession?.getMux() - if ( - existingSession?.getState() === 'ready' && - existingState?.status === 'connected' && - connectionManager!.getConnection(targetId) && - existingMux && - !existingMux.isDisposed() && - !relayStateOverrides.has(targetId) && - !relayLostBackoff.has(targetId) - ) { - // Why: BrowserWindow reactivation re-fires ssh:connect for already-live targets; treat as a refresh instead of tearing down the relay and its forwards. - broadcastSshState(getCurrentMainWindow, targetId, existingState) - return getPublicSshState(targetId)! - } - - const authority = rotateSshProviderAuthority(targetId) - clearRelayStateOverride(targetId) - const pendingTransportDisconnect = replacePendingTransport - ? connectionManager!.disconnect(targetId).then( - () => ({ ok: true }) as const, - (error: unknown) => ({ ok: false, error }) as const - ) - : null - let conn - // Why: tear down any existing session first to avoid leaking its multiplexer, providers, and timers (double-connect / reconnect-after-error). - if (existingSession) { - // Why: await port teardown before disposing, else the new session's restorePortForwards can hit EADDRINUSE on not-yet-released ports. - await portForwardManager!.removeAllForwards(targetId) - if (!isCurrentConnectAttempt(targetId, authority)) { - throw createCancelledConnectAttemptError() - } - try { - await existingSession.detachAndPersist() - } finally { - // Why finally: detachAndPersist runs its in-memory half synchronously, so the session is - // dead even when the lease write rejects — keeping it in activeSessions would strand every - // later connect on the same dead session. Why still after the await, not before it: the - // write has settled by now, so it can no longer clobber the replacement's 'attached' write. - if (activeSessions.get(targetId) === existingSession) { - activeSessions.delete(targetId) - clearRelayLostBackoff(targetId) - clearRelayStateOverride(targetId) - } - } - } - - if (pendingTransportDisconnect) { - const disconnectResult = await pendingTransportDisconnect - if (!disconnectResult.ok) { - throw disconnectResult.error - } - if (!isCurrentConnectAttempt(targetId, authority)) { - throw createCancelledConnectAttemptError() - } - } - - // Why here and not only at entry: this is the publication point, and it is the last statement - // before the transport opens. Checking it in the same synchronous block as activeSessions.set - // means a connect either registers before the shutdown drain snapshots, or registers never and - // owns nothing to clean up. - assertSshConnectsNotFenced() - // Why: create the session early so onStateChange sees it in 'deploying' and skips reconnect logic. - const session = new SshRelaySession( - targetId, - getCurrentMainWindow, - persistedStore!, - portForwardManager!, - currentRuntime, - broadcastDetectedPortsFromCurrentWindow - ) - configureRelaySessionCallbacks(session) - activeSessions.set(targetId, session) - const ownsSession = (): boolean => - isCurrentConnectAttempt(targetId, authority) && activeSessions.get(targetId) === session - - // Why captured here and not with existingState: connect() reuses an already-connected transport, - // and only a transport this attempt opened is this attempt's to close when it loses the race. - const priorConnection = connectionManager!.getConnection(targetId) - const mintedConnection = (): SshConnection | null => - conn && conn !== priorConnection ? conn : null - - try { - conn = await connectionManager!.connect(target) - if (!ownsSession()) { - throw createCancelledConnectAttemptError() - } - } catch (err) { - // Why: connect()'s internal state may not have reached the renderer; broadcast explicitly so the UI leaves 'connecting'. - const errObj = err instanceof Error ? err : new Error(String(err)) - const status: SshConnectionStatus = isAuthError(errObj) ? 'auth-failed' : 'error' - if (!ownsSession()) { - await abandonCancelledConnectAttempt(targetId, session, mintedConnection()) - throw createCancelledConnectAttemptError() - } - // Why: clear this failed connect's flag so a later non-prompting connect isn't deferred. - credentialRequestedForTarget.delete(targetId) - await abandonFailedSshSession(targetId, session) - clearRelayLostBackoff(targetId) - clearRelayStateOverride(targetId) - broadcastSshState(getCurrentMainWindow, targetId, { - targetId, - status, - error: errObj.message, - reconnectAttempt: 0 - }) - throw err - } - - try { - callbacks.onStateChange(targetId, { - targetId, - status: 'deploying-relay', - error: null, - reconnectAttempt: 0 - }) - - await session.establish(conn, relayGracePeriodForTarget(target)) - if (!ownsSession()) { - throw createCancelledConnectAttemptError() - } - - // Why: we manually pushed `deploying-relay`, so send `connected` straight to the renderer — routing through onStateChange would trigger reconnect logic. - clearRelayStateOverride(targetId) - broadcastSshState(getCurrentMainWindow, targetId, { - targetId, - status: 'connected', - error: null, - reconnectAttempt: 0, - supportsFolderDownload: conn.usesSystemSshTransport?.() !== true - }) - } catch (err) { - if (!ownsSession()) { - await abandonCancelledConnectAttempt(targetId, session, mintedConnection()) - throw createCancelledConnectAttemptError() - } - await abandonFailedSshSession(targetId, session) - clearRelayLostBackoff(targetId) - try { - await connectionManager!.disconnect(targetId) - } catch (disconnectError) { - // Why: the establish failure is the actionable error; a teardown throw must not replace it. - console.warn( - `[ssh] Failed to disconnect transport after failed establish for ${targetId}: ${disconnectError instanceof Error ? disconnectError.message : String(disconnectError)}` - ) - } - throw err - } - - // Why: persist whether this connect needed a credential so startup can partition targets into eager vs deferred without re-probing keys. - const requiredPassphrase = credentialRequestedForTarget.has(targetId) - credentialRequestedForTarget.delete(targetId) - getSshTargetRegistryStore()!.updateTarget(targetId, { - lastRequiredPassphrase: requiredPassphrase - }) - - return getPublicSshState(targetId)! - } - - ipcMain.handle('ssh:disconnect', async (_event, args: { targetId: string }) => { - await disconnectRegisteredSshTarget(args.targetId) - }) - - ipcMain.handle('ssh:terminateSessions', async (_event, args: { targetId: string }) => { - invalidateConnectAttempt(args.targetId) - await runTargetLifecycle(args.targetId, async () => { - const provider = getSshPtyProvider(args.targetId) - const leases = persistedStore!.getSshRemotePtyLeases(args.targetId) - const ptyIdsByRelayId = new Map() - // Why: only leases the app still believes it owns may force a reconnect; 'expired' ones are - // swept opportunistically because they can name a host that is gone for good (issue #2626). - const ownedRelayIds = new Set() - const trackPtyId = (ptyId: string, owned: boolean): void => { - const relayPtyId = toRelaySshPtyId(args.targetId, ptyId) - if (!ptyIdsByRelayId.has(relayPtyId)) { - ptyIdsByRelayId.set(relayPtyId, toAppSshPtyId(args.targetId, ptyId)) - } - if (owned) { - ownedRelayIds.add(relayPtyId) - } - } - for (const ptyId of getPtyIdsForConnection(args.targetId)) { - trackPtyId(ptyId, true) - } - for (const lease of leases) { - if (lease.state === 'terminated') { - continue - } - // Why: 'expired' records that reattach gave up, never that the remote shell died — those are - // precisely the orphans, so the user's terminate action has to be able to reach them. - trackPtyId(lease.ptyId, lease.state !== 'expired') - } - const ptyIds = Array.from(ptyIdsByRelayId, ([relayPtyId, appPtyId]) => ({ - relayPtyId, - appPtyId - })) - - if (ownedRelayIds.size > 0 && !provider) { - throw new Error( - `${SSH_TERMINATE_RECONNECT_REQUIRED}: SSH relay is not connected; reconnect before terminating remote sessions.` - ) - } - const shutdownResults = provider - ? await Promise.allSettled( - ptyIds.map(({ appPtyId }) => - provider.shutdown(appPtyId, { immediate: true, keepHistory: false }) - ) - ) - : [] - const shutdownFailures: string[] = [] - for (const [index, result] of shutdownResults.entries()) { - const { appPtyId, relayPtyId } = ptyIds[index] - if (result.status !== 'fulfilled' && !isSshPtyNotFoundError(result.reason)) { - shutdownFailures.push( - `${relayPtyId}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}` - ) - continue - } - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - persistedStore!.markSshRemotePtyLease(args.targetId, relayPtyId, 'terminated') - } - if (shutdownFailures.length > 0) { - // Why: a failed relay shutdown can leave the remote process alive in the grace window; keep the lease/session so the user can retry. - throw new Error(`Failed to terminate SSH host sessions: ${shutdownFailures.join('; ')}`) - } - await teardownSshTargetTransport(args.targetId, (session) => session.disposeAndPersist()) - }) - }) - - async function doResetRelay(targetId: string, target: SshTarget): Promise { - const inFlightConnect = connectInFlight.get(targetId) - if (inFlightConnect) { - try { - // Why: resetting activeSessions mid-deploy would dispose the session doConnect will use. - await inFlightConnect.promise - } catch { - // The reset can still recover a stale remote relay after a failed connect. - } - } - - rotateSshProviderAuthority(targetId) - const session = activeSessions.get(targetId) - if (session) { - // Why: detach() not dispose() — reset has its own stale-lease semantics below that dispose()'s clean-termination recording would hide. - await teardownActiveSshSession(targetId, (capturedSession) => - capturedSession.detachAndPersist() - ) - } - - const existingConn = connectionManager!.getConnection(targetId) - let conn = existingConn - if (!conn) { - // Why re-check: admission fenced this reset before it parked on the in-flight connect, so shutdown - // may have started (and drained) while we waited — opening a transport now would outlive the drain. - assertSshConnectsNotFenced() - conn = await connectionManager!.connect(target) - } - try { - await forceStopRelayForTarget(conn, targetId) - } finally { - const ptyIds = new Set(getPtyIdsForConnection(targetId)) - for (const lease of persistedStore!.getSshRemotePtyLeases(targetId)) { - if (lease.state !== 'terminated' && lease.state !== 'expired') { - ptyIds.add(lease.ptyId) - persistedStore!.markSshRemotePtyLease(targetId, lease.ptyId, 'expired') - } - } - // Why: reset force-kills the remote relay, so every local PTY handle it owned is stale even if the reset command failed after SIGTERM. - for (const ptyId of ptyIds) { - const appPtyId = toAppSshPtyId(targetId, ptyId) - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - } - // Why: reset's connect() may trip onCredentialRequest; clear so a later non-prompting doConnect doesn't persist lastRequiredPassphrase=true. - credentialRequestedForTarget.delete(targetId) - await connectionManager!.disconnect(targetId) - } - } - - ipcMain.handle('ssh:resetRelay', (_event, args: { targetId: string }) => { - const existingReset = resetRelayInFlight.get(args.targetId) - if (existingReset) { - return existingReset - } - - const target = getSshTargetRegistryStore()!.getTarget(args.targetId) - if (!target) { - throw new Error(`SSH target "${args.targetId}" not found`) - } - // Why: reset opens its own transport, so it must be fenced by shutdown the same way connect is. - assertSshConnectsNotFenced() - - let resetPromise: Promise - resetPromise = runTargetLifecycle(args.targetId, () => - doResetRelay(args.targetId, target) - ).finally(() => { - if (resetRelayInFlight.get(args.targetId) === resetPromise) { - resetRelayInFlight.delete(args.targetId) - } - }) - resetRelayInFlight.set(args.targetId, resetPromise) - return resetPromise - }) - - ipcMain.handle('ssh:getState', (_event, args: { targetId: string }) => { - return getPublicSshState(args.targetId) - }) - - // Why: auto-connect callers need to know whether connecting will prompt; true when the last connect required a credential and no live conn has it cached. - ipcMain.handle('ssh:needsPassphrasePrompt', (_event, args: { targetId: string }) => { - const target = getSshTargetRegistryStore()!.getTarget(args.targetId) - if (!target?.lastRequiredPassphrase) { - return false - } - const conn = connectionManager!.getConnection(args.targetId) - return !conn?.hasCachedCredential() - }) - - ipcMain.handle('ssh:testConnection', async (_event, args: { targetId: string }) => { - const target = getSshTargetRegistryStore()!.getTarget(args.targetId) - if (!target) { - throw new Error(`SSH target "${args.targetId}" not found`) - } - - // Why: with a live/reconnecting session, testConnection's disconnect() would tear down the relay stack (PTYs, watchers), so skip. - const existingSession = activeSessions.get(args.targetId) - const sessionState = existingSession?.getState() - if ( - sessionState === 'ready' || - sessionState === 'deploying' || - sessionState === 'reconnecting' - ) { - return { success: true, state: connectionManager!.getState(args.targetId) } - } - - // Why: testConnection's disconnect() would tear down an in-flight connect's relay deployment; await it instead. - const inFlight = connectInFlight.get(args.targetId) - if (inFlight) { - try { - const state = await inFlight.promise - return { success: true, state } - } catch (err) { - return { - success: false, - error: err instanceof Error ? err.message : String(err) - } - } - } - - testingTargets.add(args.targetId) - // Why a tracked promise and not just the id: a probe holds a real transport that no session owns, - // so shutdown has to be able to join it before the final drain disconnects what is left. - const probe = (async () => { - // Why: a probe transport opened after the shutdown drain would outlive orderly teardown. - assertSshConnectsNotFenced() - const conn = await connectionManager!.connect(target) - const state = conn.getState() - await connectionManager!.disconnect(args.targetId) - return state - })() - testConnectionProbes.add(probe) - try { - return { success: true, state: await probe } - } catch (err) { - return { - success: false, - error: err instanceof Error ? err.message : String(err) - } - } finally { - testConnectionProbes.delete(probe) - testingTargets.delete(args.targetId) - // Why: clear so a test's credential prompt doesn't leave lastRequiredPassphrase=true and defer this target at startup. - credentialRequestedForTarget.delete(args.targetId) - } - }) - - // ── Port forwarding ───────────────────────────────────────────────── - - ipcMain.handle( - 'ssh:addPortForward', - async ( - _event, - args: { - targetId: string - localPort: number - remoteHost: string - remotePort: number - label?: string - } - ) => { - const conn = connectionManager!.getConnection(args.targetId) - if (!conn) { - throw new Error(`SSH connection "${args.targetId}" not found`) - } - const entry = await portForwardManager!.addForward( - args.targetId, - conn, - args.localPort, - args.remoteHost, - args.remotePort, - args.label - ) - persistPortForwards(args.targetId) - broadcastPortForwards(getCurrentMainWindow, args.targetId) - return entry - } - ) - - ipcMain.handle( - 'ssh:updatePortForward', - async ( - _event, - args: { - id: string - targetId: string - localPort: number - remoteHost: string - remotePort: number - label?: string - } - ) => { - const conn = connectionManager!.getConnection(args.targetId) - if (!conn) { - throw new Error(`SSH connection "${args.targetId}" not found`) - } - try { - const entry = await portForwardManager!.updateForward( - args.id, - conn, - args.localPort, - args.remoteHost, - args.remotePort, - args.label - ) - persistPortForwards(entry.connectionId) - broadcastPortForwards(getCurrentMainWindow, entry.connectionId) - return entry - } catch (err) { - // Why: edit/rollback may have failed, so resync renderer to actual runtime state. - persistPortForwards(args.targetId) - broadcastPortForwards(getCurrentMainWindow, args.targetId) - throw err - } - } - ) - - ipcMain.handle('ssh:removePortForward', async (_event, args: { id: string }) => { - const removed = await portForwardManager!.removeForwardAndWait(args.id) - if (removed) { - persistPortForwards(removed.connectionId) - broadcastPortForwards(getCurrentMainWindow, removed.connectionId) - } - return removed - }) - - ipcMain.handle('ssh:listPortForwards', (_event, args?: { targetId?: string }) => { - const all = portForwardManager!.listForwards(args?.targetId) - if (!persistedStore || !args?.targetId) { - // Why: cross-target entries can't be mapped to worktrees in one call, so serve the raw list. - return all - } - return enrichSshForwardEntries(all, getWorktreeIdsForConnection(persistedStore, args.targetId)) - }) - - ipcMain.handle('ssh:listDetectedPorts', (_event, args: { targetId: string }) => { - const session = activeSessions.get(args.targetId) - const ports = session?.getPortScanner()?.getDetectedPorts(args.targetId) ?? [] - return enrichDetected(args.targetId, ports) - }) - - return { connectionManager, sshStore: getSshTargetRegistryStore() as SshConnectionStore } } export function getSshConnectionManager(): SshConnectionManager | null { return connectionManager } -// Why one budget for the whole sequence rather than one per phase: an invalidated connect only -// observes its cancellation at the next checkpoint, and one blocked in the transport handshake can -// sit there for the whole SSH timeout. A per-phase timeout lets any single phase consume the global -// quit deadline; a shared absolute deadline cannot. -export const SSH_SHUTDOWN_BUDGET_MS = 6_000 - -export type SshShutdownPhase = 'drain' | 'in-flight-join' | 'final-drain' -export type SshShutdownUnfinished = { targetId: string; phase: SshShutdownPhase } -export type SshShutdownResult = { - unfinished: readonly SshShutdownUnfinished[] - errors: readonly unknown[] -} - -type SshShutdownTask = { targetId: string; promise: Promise } - -let sshShutdownDrain: Promise | null = null - -async function settleTasksWithinMs( - tasks: readonly SshShutdownTask[], - timeoutMs: number -): Promise<{ timedOut: SshShutdownTask[]; errors: unknown[] }> { - const pending = new Set(tasks) - const errors: unknown[] = [] - if (tasks.length === 0) { - return { timedOut: [], errors } - } - const tracked = tasks.map(async (task) => { - try { - await task.promise - } catch (error) { - errors.push(error) - } - pending.delete(task) - }) - let timer: ReturnType | undefined - try { - await Promise.race([ - Promise.all(tracked), - new Promise((resolve) => { - timer = setTimeout(resolve, timeoutMs) - timer.unref?.() - }) - ]) - } finally { - clearTimeout(timer) - } - return { timedOut: [...pending], errors } -} - -function sshShutdownTasks(targetIds: readonly string[]): SshShutdownTask[] { - return [ - ...targetIds - .filter((targetId) => activeSessions.has(targetId)) - .map((targetId) => ({ - targetId, - promise: teardownActiveSshSession(targetId, (session) => session.detachAndPersist()) - })), - { targetId: '*transports', promise: connectionManager?.disconnectAll() ?? Promise.resolve() } - ] -} - -async function drainSshShutdown( - targetIds: readonly string[], - inFlight: readonly SshShutdownTask[], - detachErrors: readonly unknown[] = [] -): Promise { - const deadline = Date.now() + SSH_SHUTDOWN_BUDGET_MS - const unfinished: SshShutdownUnfinished[] = [] - const errors: unknown[] = [...detachErrors] - const runPhase = async ( - phase: SshShutdownPhase, - tasks: readonly SshShutdownTask[] - ): Promise => { - const remainingMs = deadline - Date.now() - if (remainingMs <= 0) { - unfinished.push(...tasks.map((task) => ({ targetId: task.targetId, phase }))) - return false - } - const settled = await settleTasksWithinMs(tasks, remainingMs) - errors.push(...settled.errors) - unfinished.push(...settled.timedOut.map((task) => ({ targetId: task.targetId, phase }))) - return settled.timedOut.length === 0 - } - - await runPhase('drain', sshShutdownTasks(targetIds)) - // Why a second drain after the join: a connect paused in old-session teardown still publishes its - // replacement session and opens a transport before it reaches the cancellation checkpoint, so the - // first drain can miss both. - if (await runPhase('in-flight-join', inFlight)) { - await runPhase('final-drain', sshShutdownTasks([...activeSessions.keys()])) - } - - if (errors.length > 0 || unfinished.length > 0) { - // Why one aggregate line: per-target logging on the quit path competes with the final flush for - // the little time that remains. - console.warn( - `[ssh] Shutdown drain finished with ${errors.length} error(s); unfinished: ${ - unfinished.map((entry) => `${entry.targetId}/${entry.phase}`).join(', ') || 'none' - }` - ) - } - return { unfinished, errors } -} - -// Why one entry point that returns rather than awaits: every in-memory transition the final store -// flush must snapshot happens synchronously here, before this returns, so the caller can start that -// flush with no await in between. Idempotent — a later call joins the same drain and repeats no -// state transition. -// -// Why no fence latch here: the committed quit path sets it before calling this, so it is already on -// for the snapshot below. Called without that gate (tests), this degrades to a plain drain. -export function beginSshShutdown(): Promise { - if (sshShutdownDrain) { - return sshShutdownDrain - } - const inFlight: SshShutdownTask[] = [ - ...[...connectInFlight.entries()].map(([targetId, attempt]) => ({ - targetId, - promise: attempt.promise - })), - ...[...resetRelayInFlight.entries()].map(([targetId, promise]) => ({ targetId, promise })), - ...[...testConnectionProbes].map((promise) => ({ targetId: '*probe', promise })) - ] - for (const targetId of Array.from(connectInFlight.keys())) { - invalidateConnectAttempt(targetId) - } - const targetIds = [...activeSessions.keys()] - // Why before any await: this is the whole point of the split. Each session marks its recovery lease - // detached in memory now, and the final flush persists it — the remote PTYs keep running. - const detachErrors: unknown[] = [] - for (const session of activeSessions.values()) { - // Why per-session: this runs synchronously inside a non-async will-quit listener, so one throw - // (teardownProviders -> webContents.send on a destroyed renderer, routine on quit) would escape - // it and skip every later session, the drain assignment, and the store flush that persists all - // of this. Collect and keep going; the drain reports them. - try { - session.beginShutdownDetach() - } catch (error) { - detachErrors.push(error) - } - } - sshShutdownDrain = drainSshShutdown(targetIds, inFlight, detachErrors) - return sshShutdownDrain -} - export async function resetSshHandlerStateForTests(): Promise { - advertisedUrlWatcherUnsubscribe?.() - advertisedUrlWatcherUnsubscribe = null - powerMonitorUnsubscribe?.() - powerMonitorUnsubscribe = null + unregisterAdvertisedUrlRefresh() + unregisterPowerMonitorReconnect() for (const ch of SSH_IPC_CHANNELS) { ipcMain.removeHandler(ch) } @@ -1789,17 +226,17 @@ export async function resetSshHandlerStateForTests(): Promise { testConnectionProbes.clear() credentialRequestedForTarget.clear() quitTeardownStartGate.resetForTests() - sshShutdownDrain = null + resetSshShutdownDrain() await connectionManager?.disconnectAll() portForwardManager?.dispose() - connectionManager = null - portForwardManager = null + setConnectionManager(null) + setPortForwardManager(null) setSshTargetRegistryStore(null) - persistedStore = null + setPersistedStore(null) setSshTargetRegistryHandlers({ connect: null, getState: null }) - currentGetMainWindow = () => null - currentRuntime = undefined + setCurrentGetMainWindow(() => null) + setCurrentRuntime(undefined) } export function getSshConnectionStore(): SshConnectionStore | null {