Files
orca/src/relay/pty-handler.ts
T
Jinjing aa0993f546 Improve remote connection terminal error msg and session restore recovery (#7661)
* Robustify SSH terminal reconnect and session restore recovery

- Release the replay guard after a fallback timeout to prevent permanent
  keyboard input lockouts when an unmounted terminal never parses.
- Verify backing process liveness on PTY attach and reap stale entries
  so dead shells cleanly trigger a fresh pane spawn.
- Normalize connection IDs during restore to prevent spurious mismatch
  errors, and treat true mismatches as expired sessions instead of crashing.

* Route disconnected SSH terminal panes through deferred connection gate

Avoid spawning SSH terminal processes against disconnected targets,
which otherwise throws "No PTY provider" and leaves panes stranded.

- Intercept spawning via a new connect gate that triggers the deferred
  connection flow when the SSH target is disconnected.
- Fall back to composite worktree IDs during cold-start hydration to
  ensure deferred SSH session IDs are properly stashed.
- Retry spawning and remounting terminal panes upon SSH reconnect if
  they are stranded or failed to spawn.
2026-07-06 21:41:23 -07:00

1099 lines
43 KiB
TypeScript

/* oxlint-disable max-lines */
import type { IPty } from 'node-pty'
import type * as NodePty from 'node-pty'
import { resolveWindowsGitBashShellPath } from '../main/git-bash'
import { WINDOWS_GIT_BASH_SHELL } from '../shared/windows-terminal-shell'
import type { RelayDispatcher, RequestContext } from './dispatcher'
import {
resolveDefaultShell,
resolveDefaultCwd,
resolveProcessCwd,
processHasChildren,
getForegroundProcessName,
isProcessAlive,
listShellProfiles
} from './pty-shell-utils'
import { getRelayShellLaunchConfig } from './pty-shell-launch'
import { DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../shared/ssh-types'
import { shouldUseShellReadyStartupDelivery } from '../shared/codex-startup-delivery'
import { buildStartupCommandSubmission } from '../shared/startup-command-submission'
import { resolveSetupAgentSequenceLaunchCommand } from '../shared/setup-agent-sequencing'
import {
createShellReadyScanState,
drainShellReadyHeldBytes,
scanForShellReady,
type ShellReadyScanState
} from '../main/shell-ready-marker-scanner'
// Why: node-pty is a native addon that may not be installed on the remote.
// Dynamic import keeps the require() lazy so loadPty() returns null gracefully
// when the native module is unavailable. The static type import lets vitest
// intercept it in tests.
let ptyModule: typeof NodePty | null = null
async function loadPty(): Promise<typeof NodePty | null> {
if (ptyModule) {
return ptyModule
}
try {
ptyModule = await import('node-pty')
return ptyModule
} catch {
return null
}
}
type ManagedPty = {
id: string
pty: IPty
initialCwd: string
buffered: string
/** Timer for SIGKILL fallback after a graceful SIGTERM shutdown. */
killTimer?: ReturnType<typeof setTimeout>
/** True once disposeManagedPty has run. Prevents double-dispose (onExit + an
* explicit shutdown can both fire for the same PTY) and converts post-dispose
* entry-point calls into a clean "not found" error instead of a silent no-op
* (POSIX proc.kill is neutralized inside disposeManagedPty). */
disposed?: boolean
/** True once external cleanup observers have been notified. Forced cleanup
* paths can run before node-pty emits onExit; this prevents duplicate
* overlay/cache cleanup if onExit arrives later. */
exitListenerNotified?: boolean
/** Renderer-supplied paneKey from spawn env (ORCA_PANE_KEY). Captured so
* external observers (the relay-hook-server cache) can evict per-pane
* state when this PTY exits. Symmetric with Orca's local pty.ts. */
paneKey?: string
tabId?: string
/** Attach-only identity metadata supplied over RPC. Kept separate from
* paneKey/tabId because those fields also control shell env/revive hooks. */
attachIdentity?: PtyIdentity
worktreeId?: string
terminalHandle?: string
startupCommand?: ManagedStartupCommand
}
type PendingPtyOutput = {
data: string
}
type ManagedStartupCommand = {
command: string
delivered: boolean
waitForShellReady: boolean
scanState: ShellReadyScanState | null
timer: ReturnType<typeof setTimeout> | null
}
function disposeManagedPty(managed: ManagedPty): void {
if (managed.disposed) {
return
}
managed.disposed = true
// Why: clear any pending 5s SIGKILL fallback timer. If graceful-shutdown
// armed a killTimer and the child then exited cleanly (firing onExit →
// disposeManagedPty), the timer would otherwise fire later and attempt
// pty.kill('SIGKILL') on an already-disposed instance. The ptys.has(id)
// guard inside the timer short-circuits today, but symmetry is clearer.
if (managed.killTimer) {
clearTimeout(managed.killTimer)
managed.killTimer = undefined
}
// Why: UnixTerminal.destroy() registers `_socket.once('close', () => this.kill('SIGHUP'))`.
// The close event fires asynchronously; by then the child may have exited and
// its pid been recycled. On the Linux remote hosts the relay typically runs on,
// pid recycling is fast — SIGHUP to a stranger is a real hazard. Neutralize
// managed.pty.kill before destroy() runs. Windows exempt: WindowsTerminal.destroy
// IS a kill() call via _deferNoArgs — neutralizing it leaks the ConPTY agent.
if (process.platform !== 'win32') {
;(managed.pty as unknown as { kill: (sig?: string) => void }).kill = () => {}
}
try {
;(managed.pty as unknown as { destroy?: () => void }).destroy?.()
} catch {
/* swallow */
}
}
const DEFAULT_GRACE_TIME_MS = DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS * 1000
export const REPLAY_BUFFER_MAX = 100 * 1024
const PTY_OUTPUT_BATCH_INTERVAL_MS = 8
const PTY_OUTPUT_DRAIN_CONTINUE_MS = 1
const PTY_OUTPUT_FLUSH_CHUNK_CHARS = 16 * 1024
const PTY_OUTPUT_FLUSH_MAX_WRITES = 2
const INTERACTIVE_OUTPUT_WINDOW_MS = 100
const INTERACTIVE_OUTPUT_MAX_CHARS = 1024
const INTERACTIVE_REDRAW_MAX_CHARS = PTY_OUTPUT_FLUSH_CHUNK_CHARS
const INTERACTIVE_OUTPUT_BUDGET_CHARS = 32 * 1024
const STARTUP_COMMAND_WRITE_DELAY_MS = 50
const STARTUP_COMMAND_SHELL_READY_FALLBACK_MS = 1500
const ALLOWED_SIGNALS = new Set([
'SIGINT',
'SIGTERM',
'SIGHUP',
'SIGKILL',
'SIGTSTP',
'SIGCONT',
'SIGWINCH',
'SIGUSR1',
'SIGUSR2'
])
const ALLOWED_WINDOWS_SHELL_OVERRIDES = new Set([
'powershell.exe',
'powershell',
'pwsh.exe',
'pwsh',
'cmd.exe',
'cmd',
'wsl.exe',
'wsl',
WINDOWS_GIT_BASH_SHELL
])
function resolvePtyShellOverride(shellOverride: string): string {
if (!shellOverride) {
return ''
}
if (process.platform !== 'win32') {
return ''
}
const normalized = shellOverride.toLowerCase()
if (!ALLOWED_WINDOWS_SHELL_OVERRIDES.has(normalized)) {
throw new Error(`Unsupported Windows shell override: ${shellOverride}`)
}
return resolveWindowsGitBashShellPath(shellOverride) ?? shellOverride
}
type PtyProcessSummary = {
id: string
cwd: string
title: string
terminalHandle?: string
}
type SerializedPtyEntry = {
id: string
pid: number
cols: number
rows: number
cwd: string
paneKey?: string
tabId?: string
attachIdentity?: PtyIdentity
worktreeId?: string
terminalHandle?: string
}
export type PtyExitListener = (event: { id: string; paneKey?: string }) => void
type PtyIdentity = { paneKey?: string; tabId?: string }
/**
* True when a reattach's expected pane identity contradicts the target PTY's
* own. Used to reject cross-relay-generation id collisions: a reset relay mints
* `pty-N` from 1 again, so an old lease's `pty-N` can name a different pane's
* fresh PTY. Only compares fields present on *both* sides — absent identity on
* either side is permissive so legacy PTYs and identity-less callers still attach.
*/
export function attachIdentityMismatches(expected: PtyIdentity, managed: PtyIdentity): boolean {
return Boolean(
(expected.paneKey && managed.paneKey && expected.paneKey !== managed.paneKey) ||
(expected.tabId && managed.tabId && expected.tabId !== managed.tabId)
)
}
/** Returns env to merge into the PTY's spawn env. Receives spawn context so
* augmenters that need a per-PTY identity (e.g. OPENCODE_CONFIG_DIR overlay
* paths derived from the renderer's paneKey) can compute it without pulling
* the renderer's env in twice. `command` is the renderer-chosen agent launch
* command (`pi`, `omp`, …) — supplied by ssh-pty-provider.ts so the Pi
* overlay can resolve the per-agent source dir without disk-presence
* guessing. NEVER undefined for client-driven spawns that target a
* Pi-compatible agent; may be undefined for CLI-launched bare shells. */
export type PtyEnvAugmenter = (ctx: {
id: string
paneKey?: string
shell: string
env: Record<string, string>
command?: string
}) => Record<string, string>
export class PtyHandler {
private ptys = new Map<string, ManagedPty>()
private nextId = 1
private dispatcher: RelayDispatcher
private graceTimeMs: number
private graceTimer: ReturnType<typeof setTimeout> | null = null
private outputFlushTimer: ReturnType<typeof setTimeout> | null = null
private pendingOutputByPty = new Map<string, PendingPtyOutput>()
private lastInputAtByPty = new Map<string, number>()
private interactiveOutputCharsByPty = new Map<string, number>()
// Why: external observers need to drop per-pane state when a PTY exits.
// Today the relay composes multiple consumers (hook-server cache eviction
// and plugin-overlay dir cleanup) into a single callback at the call site
// (see relay.ts setExitListener). A single optional slot is intentional —
// callers compose externally rather than us maintaining a listener list.
// A throw inside the listener is swallowed so it can never block
// disposeManagedPty / map cleanup.
private exitListener: PtyExitListener | null = null
// Why: env augmenters injected at relay boot (currently the relay-hook
// server's ORCA_AGENT_HOOK_* coords). Run on every spawn so every PTY
// sees the live hook coordinates without the dispatcher needing to know
// about agent hooks.
private envAugmenters: PtyEnvAugmenter[] = []
constructor(dispatcher: RelayDispatcher, graceTimeMs = DEFAULT_GRACE_TIME_MS) {
this.dispatcher = dispatcher
this.graceTimeMs = graceTimeMs
this.registerHandlers()
}
setGraceTimeMs(graceTimeMs: number): void {
this.graceTimeMs = Math.max(0, Math.floor(graceTimeMs))
}
get configuredGraceTimeMs(): number {
return this.graceTimeMs
}
/** Subscribe to PTY-exit events. Used by the relay-hook server to evict
* per-paneKey cached payloads when the backing PTY ends. */
setExitListener(listener: PtyExitListener | null): void {
this.exitListener = listener
}
/** Register an env augmenter whose return value is merged into every spawn
* env *after* `process.env` and the renderer-supplied env. Used by the
* relay-hook server to inject ORCA_AGENT_HOOK_PORT/TOKEN/ENV/VERSION/
* ENDPOINT — values the agent CLI inside the PTY needs to find the local
* hook receiver. See docs/design/agent-status-over-ssh.md §3. */
addEnvAugmenter(augmenter: PtyEnvAugmenter): () => void {
this.envAugmenters.push(augmenter)
return () => {
const idx = this.envAugmenters.indexOf(augmenter)
if (idx !== -1) {
this.envAugmenters.splice(idx, 1)
}
}
}
/** Build the augmented spawn env. Augmenter values override `process.env`
* and any renderer-supplied env (the augmenter contract — see
* addEnvAugmenter doc-comment). Used by both spawn() and revive() so the
* relationship between process.env, renderer env, and augmenters cannot
* drift between the two paths — revived shells after a relay restart must
* see the fresh ORCA_AGENT_HOOK_* coords just like freshly-spawned ones,
* otherwise agent-status over SSH silently breaks on every revive. */
private buildSpawnEnv(
rendererEnv: Record<string, string> | undefined,
ctx: { id: string; paneKey?: string; shell: string; command?: string }
): Record<string, string> {
const baseEnv = { ...process.env, ...rendererEnv } as Record<string, string>
const augmented: Record<string, string> = {}
for (const augmenter of this.envAugmenters) {
try {
Object.assign(augmented, augmenter({ ...ctx, env: baseEnv }))
} catch (err) {
process.stderr.write(
`[pty-handler] env augmenter threw: ${err instanceof Error ? err.message : String(err)}\n`
)
}
}
return { ...baseEnv, ...augmented }
}
private clearStartupCommandTimer(managed: ManagedPty): void {
if (managed.startupCommand?.timer) {
clearTimeout(managed.startupCommand.timer)
managed.startupCommand.timer = null
}
}
private appendReplayBuffer(managed: ManagedPty, data: string): void {
managed.buffered += data
if (managed.buffered.length > REPLAY_BUFFER_MAX) {
managed.buffered = managed.buffered.slice(-REPLAY_BUFFER_MAX)
}
}
private releaseStartupCommand(managed: ManagedPty): void {
this.clearStartupCommandTimer(managed)
managed.startupCommand = undefined
}
private scheduleStartupCommandDelivery(managed: ManagedPty, delayMs: number): void {
const startup = managed.startupCommand
if (!startup || startup.delivered || managed.disposed) {
return
}
this.clearStartupCommandTimer(managed)
startup.timer = setTimeout(() => {
startup.timer = null
this.deliverStartupCommand(managed)
}, delayMs)
}
private deliverStartupCommand(managed: ManagedPty): void {
const startup = managed.startupCommand
if (!startup || startup.delivered || managed.disposed) {
return
}
startup.delivered = true
this.clearStartupCommandTimer(managed)
if (startup.scanState) {
const heldBytes = drainShellReadyHeldBytes(startup.scanState)
if (heldBytes) {
this.appendReplayBuffer(managed, heldBytes)
this.enqueuePtyOutput(managed.id, heldBytes)
}
}
const submit = process.platform === 'win32' ? '\r' : '\n'
// Why: a multiline startup prompt is pasted literally via bracketed paste
// only when the Orca shell-ready wrapper is active (waitForShellReady) —
// that is the bash/zsh overlay that arms bracketed-paste mode. Other remote
// shells keep the raw submit path so the ESC[200~ markers are not echoed.
const payload = buildStartupCommandSubmission(startup.command, {
submit,
bracketedPasteSafe: startup.waitForShellReady
})
managed.startupCommand = undefined
managed.pty.write(payload)
}
/** Wire onData/onExit listeners for a managed PTY and store it. */
private wireAndStore(managed: ManagedPty): void {
this.ptys.set(managed.id, managed)
managed.pty.onData((data: string) => {
const startup = managed.startupCommand
if (startup?.waitForShellReady && startup.scanState && !startup.delivered) {
const scanned = scanForShellReady(startup.scanState, data)
data = scanned.output
if (scanned.matched) {
this.scheduleStartupCommandDelivery(managed, STARTUP_COMMAND_WRITE_DELAY_MS)
}
}
this.appendReplayBuffer(managed, data)
this.enqueuePtyOutput(managed.id, data)
})
managed.pty.onExit(({ exitCode }: { exitCode: number }) => {
if (managed.disposed) {
return
}
// Why: neutralize managed.pty.kill synchronously BEFORE anything else
// in this callback. node-pty's UnixTerminal has
// `_socket.once('close', () => this.kill('SIGHUP'))` wired at destroy
// time, and the master socket can emit 'close' concurrently with this
// onExit on natural exit. If 'close' wins, SIGHUP targets the reaped
// pid — recycled to an unrelated process on Linux (the typical relay
// host). Synchronous neutralization closes that window. Windows is
// exempt (WindowsTerminal.destroy uses kill() to close ConPTY).
if (process.platform !== 'win32') {
;(managed.pty as unknown as { kill: (sig?: string) => void }).kill = () => {}
}
// Why: If the PTY exits normally (or via SIGTERM), we must clear the
// SIGKILL fallback timer to avoid firing SIGKILL later.
if (managed.killTimer) {
clearTimeout(managed.killTimer)
managed.killTimer = undefined
}
this.clearStartupCommandTimer(managed)
this.flushPtyOutput(managed.id)
this.dispatcher.notify('pty.exit', { id: managed.id, code: exitCode })
this.notifyExitListener(managed)
this.ptys.delete(managed.id)
this.clearPtyFlowState(managed.id)
// Why: release the ptmx fd on the natural-exit path. Without this the
// node-pty wrapper's _socket stays alive until GC and the master fd
// leaks (see docs/fix-pty-fd-leak.md).
disposeManagedPty(managed)
})
}
private notifyExitListener(managed: ManagedPty): void {
if (managed.exitListenerNotified) {
return
}
managed.exitListenerNotified = true
// Why: external observers own relay-hook cache eviction and plugin-overlay
// cleanup. Natural exits, immediate shutdown, SIGKILL fallback, and relay
// process disposal all need the same cleanup even when node-pty never
// delivers onExit.
if (this.exitListener) {
try {
this.exitListener({ id: managed.id, paneKey: managed.paneKey })
} catch (err) {
process.stderr.write(
`[pty-handler] exit listener threw: ${err instanceof Error ? err.message : String(err)}\n`
)
}
}
}
private registerHandlers(): void {
this.dispatcher.onRequest('pty.spawn', (p, context) => this.spawn(p, context))
this.dispatcher.onRequest('pty.attach', (p) => this.attach(p))
this.dispatcher.onRequest('pty.shutdown', (p) => this.shutdown(p))
this.dispatcher.onRequest('pty.sendSignal', (p) => this.sendSignal(p))
this.dispatcher.onRequest('pty.getCwd', (p) => this.getCwd(p))
this.dispatcher.onRequest('pty.getInitialCwd', (p) => this.getInitialCwd(p))
this.dispatcher.onRequest('pty.clearBuffer', (p) => this.clearBuffer(p))
this.dispatcher.onRequest('pty.hasChildProcesses', (p) => this.hasChildProcesses(p))
this.dispatcher.onRequest('pty.getForegroundProcess', (p) => this.getForegroundProcess(p))
this.dispatcher.onRequest('pty.listProcesses', () => this.listProcesses())
this.dispatcher.onRequest('pty.getDefaultShell', async () => resolveDefaultShell())
this.dispatcher.onRequest('pty.serialize', (p) => this.serialize(p))
this.dispatcher.onRequest('pty.revive', (p) => this.revive(p))
this.dispatcher.onRequest('pty.getProfiles', async () => listShellProfiles())
this.dispatcher.onNotification('pty.data', (p) => this.writeData(p))
this.dispatcher.onNotification('pty.resize', (p) => this.resize(p))
this.dispatcher.onNotification('pty.ackData', (_p) => {
/* flow control ack -- not yet enforced */
})
}
private isLikelyInteractiveRedraw(data: string): boolean {
if (data.length <= INTERACTIVE_OUTPUT_MAX_CHARS) {
return true
}
return data.length <= INTERACTIVE_REDRAW_MAX_CHARS && data.includes('\x1b[')
}
private shouldSendInteractiveOutputNow(id: string, data: string): boolean {
const lastInputAt = this.lastInputAtByPty.get(id)
const now = performance.now()
if (lastInputAt === undefined || now - lastInputAt > INTERACTIVE_OUTPUT_WINDOW_MS) {
this.interactiveOutputCharsByPty.delete(id)
return false
}
if (!this.isLikelyInteractiveRedraw(data)) {
this.interactiveOutputCharsByPty.set(id, INTERACTIVE_OUTPUT_BUDGET_CHARS)
return false
}
const usedChars = this.interactiveOutputCharsByPty.get(id) ?? 0
if (usedChars + data.length > INTERACTIVE_OUTPUT_BUDGET_CHARS) {
this.interactiveOutputCharsByPty.set(id, INTERACTIVE_OUTPUT_BUDGET_CHARS)
return false
}
this.interactiveOutputCharsByPty.set(id, usedChars + data.length)
return true
}
private enqueuePtyOutput(id: string, data: string): void {
const existing = this.pendingOutputByPty.get(id)
const pending = { data: (existing?.data ?? '') + data }
if (this.shouldSendInteractiveOutputNow(id, pending.data)) {
this.pendingOutputByPty.delete(id)
this.clearOutputFlushTimerIfIdle()
// Why: remote agent TUIs redraw around each keystroke. Background relay
// batching should reduce SSH chatter, not add visible input echo delay.
this.dispatcher.notify('pty.data', { id, data: pending.data })
return
}
this.pendingOutputByPty.set(id, pending)
this.scheduleOutputFlush(PTY_OUTPUT_BATCH_INTERVAL_MS)
}
private scheduleOutputFlush(delayMs: number): void {
if (this.outputFlushTimer !== null) {
return
}
this.outputFlushTimer = setTimeout(() => this.flushPendingOutput(), delayMs)
}
private flushPendingOutput(): void {
this.outputFlushTimer = null
let writes = 0
for (const [id, pending] of Array.from(this.pendingOutputByPty.entries())) {
if (writes >= PTY_OUTPUT_FLUSH_MAX_WRITES) {
break
}
this.pendingOutputByPty.delete(id)
const chunk = pending.data.slice(0, PTY_OUTPUT_FLUSH_CHUNK_CHARS)
const remaining = pending.data.slice(PTY_OUTPUT_FLUSH_CHUNK_CHARS)
if (remaining) {
this.pendingOutputByPty.set(id, { data: remaining })
}
this.dispatcher.notify('pty.data', { id, data: chunk })
writes++
}
if (this.pendingOutputByPty.size > 0 && writes > 0) {
// Why: relay-side output can arrive as a large single PTY chunk. Yield
// between slices so client input and control frames can interleave.
this.scheduleOutputFlush(PTY_OUTPUT_DRAIN_CONTINUE_MS)
}
}
private flushPtyOutput(id: string): void {
const pending = this.pendingOutputByPty.get(id)
if (!pending) {
return
}
this.pendingOutputByPty.delete(id)
this.dispatcher.notify('pty.data', { id, data: pending.data })
this.clearOutputFlushTimerIfIdle()
}
private clearOutputFlushTimerIfIdle(): void {
if (this.pendingOutputByPty.size > 0 || this.outputFlushTimer === null) {
return
}
clearTimeout(this.outputFlushTimer)
this.outputFlushTimer = null
}
private clearPtyFlowState(id: string): void {
this.pendingOutputByPty.delete(id)
this.lastInputAtByPty.delete(id)
this.interactiveOutputCharsByPty.delete(id)
this.clearOutputFlushTimerIfIdle()
}
private async spawn(
params: Record<string, unknown>,
context?: RequestContext
): Promise<{ id: string }> {
if (this.ptys.size >= 50) {
throw new Error('Maximum number of PTY sessions reached (50)')
}
const pty = await loadPty()
if (!pty) {
throw new Error('node-pty is not available on this remote host')
}
const cols = (params.cols as number) || 80
const rows = (params.rows as number) || 24
const cwd = (params.cwd as string) || resolveDefaultCwd()
const env = params.env as Record<string, string> | undefined
const shellOverride =
typeof params.shellOverride === 'string' ? params.shellOverride.trim() : ''
const resolvedShellOverride = resolvePtyShellOverride(shellOverride)
const shell = resolvedShellOverride || resolveDefaultShell()
const id = `pty-${this.nextId++}`
// Why: server-side augmenter values (ORCA_AGENT_HOOK_* and plugin overlay
// dirs) override renderer-supplied env so live remote paths and hook coords
// win over local userData paths. The context lets overlay augmenters derive
// per-PTY OpenCode/Pi directories from the stable paneKey when present.
// `command` is usually forwarded by ssh-pty-provider.ts only as a hint
// for overlay resolution; runtime-owned PTYs opt into relay delivery
// because no renderer TerminalPane exists to type the command.
const paneKey = typeof env?.ORCA_PANE_KEY === 'string' ? env.ORCA_PANE_KEY : undefined
// Why: kept so a restarted runtime can re-adopt this live PTY under its
// originally-exported handle (reported via listProcesses, survives revive).
const terminalHandle =
typeof env?.ORCA_TERMINAL_HANDLE === 'string' ? env.ORCA_TERMINAL_HANDLE : undefined
const command = typeof params.command === 'string' ? params.command : undefined
const terminalWindowsWslDistro =
typeof params.terminalWindowsWslDistro === 'string' ? params.terminalWindowsWslDistro : null
const commandDelivery = params.commandDelivery === 'provider' ? 'provider' : 'renderer'
const shouldProviderDeliverCommand = commandDelivery === 'provider' && command !== undefined
const spawnEnv = this.buildSpawnEnv(env, { id, paneKey, shell, command })
const launchCommandHint = resolveSetupAgentSequenceLaunchCommand(spawnEnv, command)
const shouldEmitShellReadyMarker =
launchCommandHint !== undefined &&
shouldUseShellReadyStartupDelivery({
command: launchCommandHint,
startupCommandDelivery:
params.startupCommandDelivery === 'shell-ready' ? 'shell-ready' : undefined
})
// Why: renderer- and provider-delivered startup commands both use this
// marker; the side responsible for delivery also strips it from output.
const shellLaunch = getRelayShellLaunchConfig(shell, spawnEnv, process.platform, {
terminalWindowsWslDistro,
emitReadyMarker: shouldEmitShellReadyMarker
})
// Why: SSH exec channels give the relay a minimal environment without
// .zprofile/.bash_profile sourced. Spawning a login shell ensures PATH
// includes Homebrew, nvm, and user-installed CLIs (claude, codex, gh).
// When overlays are injected, the launch wrapper keeps those paths after
// user startup files re-export their defaults.
const term = pty.spawn(shell, shellLaunch.args, {
name: 'xterm-256color',
cols,
rows,
cwd,
// Why: relay shells inherit process.env; never let an ambient Orca marker
// enable shell-ready behavior unless this spawn explicitly requested it.
env: { ...spawnEnv, ORCA_SHELL_READY_MARKER: '0', ...shellLaunch.env }
})
// Why: capture the renderer-supplied paneKey on the managed entry so the
// exit listener can evict per-pane caches without the relay needing a
// separate ptyId→paneKey map. ORCA_PANE_KEY is shaped `${tabId}:${paneId}`
// and is bounded by the renderer; the relay treats it as opaque.
const tabId = typeof env?.ORCA_TAB_ID === 'string' ? env.ORCA_TAB_ID : undefined
const attachIdentity = {
paneKey: typeof params.paneKey === 'string' ? params.paneKey : paneKey,
tabId: typeof params.tabId === 'string' ? params.tabId : tabId
}
const worktreeId = typeof env?.ORCA_WORKTREE_ID === 'string' ? env.ORCA_WORKTREE_ID : undefined
const managed: ManagedPty = {
id,
pty: term,
initialCwd: cwd,
buffered: '',
paneKey,
tabId,
...(attachIdentity.paneKey || attachIdentity.tabId ? { attachIdentity } : {}),
worktreeId,
...(terminalHandle ? { terminalHandle } : {}),
...(shouldProviderDeliverCommand
? {
startupCommand: {
command,
delivered: false,
waitForShellReady: shellLaunch.env.ORCA_SHELL_READY_MARKER === '1',
scanState:
shellLaunch.env.ORCA_SHELL_READY_MARKER === '1'
? createShellReadyScanState()
: null,
timer: null
}
}
: {})
}
this.wireAndStore(managed)
if (context?.isStale()) {
// Why: if the client reconnected while pty.spawn was in flight, the
// response is discarded and no renderer can own this PTY. Shut it down
// immediately so it does not linger as an unreachable remote shell.
this.releaseStartupCommand(managed)
term.kill('SIGTERM')
managed.killTimer = setTimeout(() => {
const still = this.ptys.get(id)
if (still && !still.disposed) {
still.pty.kill('SIGKILL')
// Why: stale-spawn cleanup has no client who will ever attach. If
// SIGKILL's onExit is missed (kernel edge case, uninterruptible
// sleep), the managed entry + ptmx fd would leak forever. Dispose
// synchronously so the entry is gone regardless of onExit timing.
this.notifyExitListener(still)
disposeManagedPty(still)
this.ptys.delete(id)
}
}, 5000)
} else if (managed.startupCommand) {
this.scheduleStartupCommandDelivery(
managed,
managed.startupCommand.waitForShellReady
? STARTUP_COMMAND_SHELL_READY_FALLBACK_MS
: STARTUP_COMMAND_WRITE_DELAY_MS
)
}
return { id }
}
private async attach(params: Record<string, unknown>): Promise<{ replay?: string }> {
const id = params.id as string
const managed = this.ptys.get(id)
// Why: treat a disposed managed entry the same as "not found" — after
// disposeManagedPty has run, managed.pty is torn down and any write/kill
// would hit a neutralized no-op on POSIX. The explicit check converts a
// silent failure into the existing error callers already handle.
if (!managed || managed.disposed) {
throw new Error(`PTY "${id}" not found`)
}
// Why: a reattach can arrive for a relay PTY whose backing shell already
// died without node-pty delivering onExit (e.g. the child was reaped out of
// band while the SSH channel was down). The map entry lingers, so attach
// would otherwise "succeed" with an empty replay and strand the reattached
// pane on a black, unresponsive shell. Prove liveness here; if the pid is
// provably gone, reap the stale entry and report not-found so the caller
// drops the dead lease and spawns fresh — the same recovery path an expired
// grace window already takes.
if (managed.pty.pid && !isProcessAlive(managed.pty.pid)) {
this.notifyExitListener(managed)
disposeManagedPty(managed)
this.ptys.delete(id)
this.clearPtyFlowState(id)
throw new Error(`PTY "${id}" not found`)
}
// Why: PTY ids are a per-relay-process counter (pty-1, pty-2, …). When the
// relay changes generation — an app update deploys a new content-hashed
// relay dir, or a grace-expired relay restarts — the counter resets, so an
// old lease's `pty-N` can name a freshly spawned `pty-N` that belongs to a
// *different* pane. Attaching by id alone then wires a tab to the wrong
// shell. Reject when the caller's expected identity disagrees with this
// PTY's own so the client falls back to a fresh spawn. Absent identity on
// either side stays permissive for backward compatibility.
const mismatch = attachIdentityMismatches(
{
paneKey: typeof params.expectedPaneKey === 'string' ? params.expectedPaneKey : undefined,
tabId: typeof params.expectedTabId === 'string' ? params.expectedTabId : undefined
},
managed.attachIdentity ?? { paneKey: managed.paneKey, tabId: managed.tabId }
)
if (mismatch) {
throw new Error(`PTY "${id}" not found (identity mismatch)`)
}
// Replay buffered output. During pty.spawn({ sessionId }) the renderer has
// not registered replay handlers yet, so return the bytes to the caller
// instead of notifying them too early.
// Why: the buffer is NOT cleared after replay. It always holds the last
// 100 KB of raw output (capped in onData). The client clears xterm before
// writing the replay, so returning the full buffer on every attach does
// not cause duplication. Keeping the buffer intact means a second app
// restart still replays the full terminal history instead of only output
// generated since the previous attach.
if (managed.buffered) {
// Why: relay batching may still hold bytes that are already included in
// the full replay buffer. Drop that pending notification before attach
// so reconnect/suppressed replay cannot render the same bytes twice.
this.pendingOutputByPty.delete(id)
this.clearOutputFlushTimerIfIdle()
if (params.suppressReplayNotification) {
return { replay: managed.buffered }
}
this.dispatcher.notify('pty.replay', { id, data: managed.buffered })
}
return {}
}
private writeData(params: Record<string, unknown>): void {
const id = params.id as string
const data = params.data as string
if (typeof data !== 'string') {
return
}
const managed = this.ptys.get(id)
if (managed && !managed.disposed) {
this.lastInputAtByPty.set(id, performance.now())
this.interactiveOutputCharsByPty.set(id, 0)
managed.pty.write(data)
}
}
private resize(params: Record<string, unknown>): void {
const id = params.id as string
const cols = Math.max(1, Math.min(500, Math.floor(Number(params.cols) || 80)))
const rows = Math.max(1, Math.min(500, Math.floor(Number(params.rows) || 24)))
const managed = this.ptys.get(id)
if (managed && !managed.disposed) {
managed.pty.resize(cols, rows)
}
}
private async shutdown(params: Record<string, unknown>): Promise<void> {
const id = params.id as string
const immediate = params.immediate as boolean
const managed = this.ptys.get(id)
if (!managed) {
return
}
if (immediate) {
this.releaseStartupCommand(managed)
this.flushPtyOutput(id)
managed.pty.kill('SIGKILL')
// Why: SIGKILL has already reaped the child; release the ptmx fd on the
// same tick. Deferring to onExit leaves a window where the fd is live
// with a dead child. Idempotent via the disposed guard — if onExit fires
// later and also calls disposeManagedPty, the second call is a no-op.
disposeManagedPty(managed)
// Why: mirror the graceful-shutdown killTimer cleanup. If SIGKILL's
// onExit never fires (kernel edge case: uninterruptible sleep,
// D-state child on a bad NFS mount), the disposed managed entry would
// linger in this.ptys forever. Each stranded entry consumes a slot in
// the 50-PTY cap and is returned by listProcesses/serialize. Deleting
// here makes the map hygiene a hard guarantee, not "hopefully onExit
// runs". If onExit DOES fire later, its own `this.ptys.delete(id)` is
// a no-op.
this.notifyExitListener(managed)
this.ptys.delete(id)
this.clearPtyFlowState(id)
} else {
this.releaseStartupCommand(managed)
managed.pty.kill('SIGTERM')
// Why: Some processes ignore SIGTERM (e.g. a hung child, a custom signal
// handler). Without a SIGKILL fallback the PTY process would leak and the
// managed entry would never be cleaned up. The 5-second window gives
// well-behaved processes time to flush and exit gracefully. The timer is
// cleared in the onExit handler if the process terminates on its own.
// Do NOT call disposeManagedPty here: destroy()-right-after-SIGTERM
// collapses the graceful-shutdown window and risks interrupting shell
// EXIT traps. Fd release happens via onExit (natural exit) or via the
// killTimer → SIGKILL → disposeManagedPty chain below.
managed.killTimer = setTimeout(() => {
const still = this.ptys.get(id)
if (still && !still.disposed) {
still.pty.kill('SIGKILL')
this.flushPtyOutput(id)
// Why: emit pty.exit BEFORE disposeManagedPty sets disposed=true.
// The natural onExit short-circuits on `managed.disposed`, so
// without this notify the renderer never learns the pane is dead
// when the SIGKILL fallback fires for a SIGTERM-ignoring child.
this.dispatcher.notify('pty.exit', { id, code: -1 })
// Why: if SIGKILL's onExit never fires (kernel edge case,
// uninterruptible sleep, child wedged on a bad NFS mount), the
// fd and map entry would leak forever. Dispose synchronously so
// graceful-shutdown's SIGKILL fallback is a hard guarantee, not
// "hopefully onExit will run". The disposed guard inside
// disposeManagedPty makes a later onExit's dispose a no-op.
this.notifyExitListener(still)
disposeManagedPty(still)
this.ptys.delete(id)
this.clearPtyFlowState(id)
}
}, 5000)
}
}
private async sendSignal(params: Record<string, unknown>): Promise<void> {
const id = params.id as string
const signal = params.signal as string
if (!ALLOWED_SIGNALS.has(signal)) {
throw new Error(`Signal not allowed: ${signal}`)
}
const managed = this.ptys.get(id)
// Why: POSIX disposeManagedPty neutralizes managed.pty.kill. Without the
// disposed check, a post-dispose sendSignal would silently succeed (no
// error, no action). Convert to the existing "not found" error.
if (!managed || managed.disposed) {
throw new Error(`PTY "${id}" not found`)
}
managed.pty.kill(signal)
}
private async getCwd(params: Record<string, unknown>): Promise<string> {
const id = params.id as string
const managed = this.ptys.get(id)
if (!managed || managed.disposed) {
throw new Error(`PTY "${id}" not found`)
}
return resolveProcessCwd(managed.pty.pid, managed.initialCwd)
}
private async getInitialCwd(params: Record<string, unknown>): Promise<string> {
const id = params.id as string
const managed = this.ptys.get(id)
if (!managed || managed.disposed) {
throw new Error(`PTY "${id}" not found`)
}
return managed.initialCwd
}
private async clearBuffer(params: Record<string, unknown>): Promise<void> {
const id = params.id as string
const managed = this.ptys.get(id)
if (managed && !managed.disposed) {
managed.pty.clear()
}
}
private async hasChildProcesses(params: Record<string, unknown>): Promise<boolean> {
const id = params.id as string
const managed = this.ptys.get(id)
if (!managed || managed.disposed) {
return false
}
return await processHasChildren(managed.pty.pid)
}
private async getForegroundProcess(params: Record<string, unknown>): Promise<string | null> {
const id = params.id as string
const managed = this.ptys.get(id)
if (!managed || managed.disposed) {
return null
}
return await getForegroundProcessName(managed.pty.pid, managed.pty.process || null)
}
private async listProcesses(): Promise<PtyProcessSummary[]> {
const results: PtyProcessSummary[] = []
for (const [id, managed] of this.ptys) {
const title =
(await getForegroundProcessName(managed.pty.pid, managed.pty.process || null)) || 'shell'
results.push({
id,
cwd: managed.initialCwd,
title,
...(managed.terminalHandle ? { terminalHandle: managed.terminalHandle } : {})
})
}
return results
}
private async serialize(params: Record<string, unknown>): Promise<string> {
const ids = params.ids as string[]
const entries: SerializedPtyEntry[] = []
for (const id of ids) {
const managed = this.ptys.get(id)
if (!managed) {
continue
}
const { pid, cols, rows } = managed.pty
entries.push({
id,
pid,
cols,
rows,
cwd: managed.initialCwd,
paneKey: managed.paneKey,
tabId: managed.tabId,
attachIdentity: managed.attachIdentity,
worktreeId: managed.worktreeId,
...(managed.terminalHandle ? { terminalHandle: managed.terminalHandle } : {})
})
}
return JSON.stringify(entries)
}
private async revive(params: Record<string, unknown>): Promise<void> {
const state = params.state as string
const entries = JSON.parse(state) as SerializedPtyEntry[]
for (const entry of entries) {
if (this.ptys.has(entry.id)) {
continue
}
// Only re-attach if the original process is still alive
try {
process.kill(entry.pid, 0)
} catch {
continue
}
const ptyMod = await loadPty()
if (!ptyMod) {
continue
}
// Why: revive must apply the same hook env as spawn(). The hook-server
// coords come from augmenters, while pane identity comes from the
// serialized PTY entry because managed hook scripts exit without
// ORCA_PANE_KEY.
const revivedEnv: Record<string, string> = {}
if (entry.paneKey) {
revivedEnv.ORCA_PANE_KEY = entry.paneKey
}
if (entry.tabId) {
revivedEnv.ORCA_TAB_ID = entry.tabId
}
if (entry.worktreeId) {
revivedEnv.ORCA_WORKTREE_ID = entry.worktreeId
}
if (entry.terminalHandle) {
revivedEnv.ORCA_TERMINAL_HANDLE = entry.terminalHandle
}
const shell = resolveDefaultShell()
// Why: `command` is intentionally absent from this revive path because
// SerializedPtyEntry (see line 99) does not persist it — ManagedPty
// never stored the renderer-chosen launch command. The Pi/OMP extension
// installer in src/relay/relay.ts therefore sees `ctx.command ===
// undefined` for revived PTYs and prepares the Pi default plus OMP's
// typed-command wrapper. Plumbing `command` through serialization is a
// separate, larger change.
const spawnEnv = this.buildSpawnEnv(revivedEnv, {
id: entry.id,
paneKey: entry.paneKey,
shell
})
const shellLaunch = getRelayShellLaunchConfig(shell, spawnEnv)
const term = ptyMod.spawn(shell, shellLaunch.args, {
name: 'xterm-256color',
cols: entry.cols,
rows: entry.rows,
cwd: entry.cwd,
// Why: revived shells should not inherit an ambient shell-ready marker
// because no provider-delivered startup command is waiting on it.
env: { ...spawnEnv, ORCA_SHELL_READY_MARKER: '0', ...shellLaunch.env }
})
this.wireAndStore({
id: entry.id,
pty: term,
initialCwd: entry.cwd,
buffered: '',
paneKey: entry.paneKey,
tabId: entry.tabId,
attachIdentity: entry.attachIdentity,
worktreeId: entry.worktreeId,
...(entry.terminalHandle ? { terminalHandle: entry.terminalHandle } : {})
})
// Why: nextId starts at 1 and is only incremented by spawn(). Revived
// PTYs carry their original IDs (e.g. "pty-3"), so without this bump the
// next spawn() would generate an ID that collides with an already-active
// revived PTY.
const match = entry.id.match(/^pty-(\d+)$/)
if (match) {
const revivedNum = Number.parseInt(match[1], 10)
if (revivedNum >= this.nextId) {
this.nextId = revivedNum + 1
}
}
}
}
startGraceTimer(onExpire: () => void, timeoutMs = this.graceTimeMs): void {
this.cancelGraceTimer()
if (timeoutMs === 0) {
return
}
// Why: callers may shorten the first empty-detached startup window, but
// connected relays still use the configured grace so live PTYs can survive
// app restarts and reconnects.
this.graceTimer = setTimeout(() => {
onExpire()
}, timeoutMs)
}
cancelGraceTimer(): void {
if (this.graceTimer) {
clearTimeout(this.graceTimer)
this.graceTimer = null
}
}
dispose(): void {
this.cancelGraceTimer()
if (this.outputFlushTimer !== null) {
clearTimeout(this.outputFlushTimer)
this.outputFlushTimer = null
}
this.pendingOutputByPty.clear()
this.lastInputAtByPty.clear()
this.interactiveOutputCharsByPty.clear()
for (const [, managed] of this.ptys) {
if (managed.killTimer) {
clearTimeout(managed.killTimer)
managed.killTimer = undefined
}
this.clearStartupCommandTimer(managed)
// Why: SIGKILL (not SIGTERM) before destroy. The relay process is
// exiting; any SIGTERM-ignoring remote shell (editor with unsaved
// buffers, a hung child with a bad handler, a process in
// uninterruptible sleep) would survive SIGTERM + immediate destroy()
// as an orphan on the remote host. SIGKILL is not ignorable and the
// ptmx fd release via disposeManagedPty is synchronous, so there is
// no graceful-shutdown window to preserve at this point.
try {
managed.pty.kill('SIGKILL')
} catch {
/* child may already be dead */
}
this.notifyExitListener(managed)
disposeManagedPty(managed)
}
this.ptys.clear()
}
get activePtyCount(): number {
return this.ptys.size
}
get retainedStartupCommandCount(): number {
let count = 0
for (const managed of this.ptys.values()) {
if (managed.startupCommand) {
count += 1
}
}
return count
}
get graceTimerActive(): boolean {
return this.graceTimer !== null
}
}