mirror of
https://github.com/stablyai/orca.git
synced 2026-10-19 08:03:13 +00:00
* fix(daemon): retire macOS daemons whose login session died (#7936) A daemon that survives a full macOS logout is unsalvageable: its PAM context can no longer host login(1) spawns (every new PTY becomes a 'Login incorrect' prompt zombie) and its Mach bootstrap namespace has lost the system DNS resolver, so terminals it hosts have no egress. Today it also keeps the #9301 preflight's cached 'accepted' verdict, so it keeps wrapping spawns in login(1) forever; only a manual daemon restart recovers. GUI-spawned daemons now watch for login-session death from the inside: a fresh cache-bypassing PAM probe (triggered by PTY-exit bursts, fresh client hellos, and a slow periodic timer) must conclusively reject three consecutive times AND the in-process system resolver must be degraded; then the daemon exits crash-style so session meta stays unclean and the replacement daemon cold-restores scrollback. A conclusive rejection also flips the spawn-wrapper cache off immediately. Headless serve/SSH daemons never get the watch (they must survive their spawning session ending), and a session that never conclusively accepted login(1) never arms it — a PAM anomaly alone can't kill a healthy daemon (fast user switching keeps accepting, so switched-away sessions are preserved). * test(daemon): e2e seam to drive login-session death oracles from a verdict file A dead macOS login session cannot be fabricated without root (PAM owns audit-session teardown), so live lifecycle QA drives the death watch's probe and resolver oracles from ORCA_E2E_LOGIN_SESSION_PROBE_FILE: 'alive' → accepted/healthy, 'dead' → rejected/unhealthy, anything else inconclusive — with compressed watch timing. Mirrors the existing ORCA_E2E_DAEMON_INIT_DELAY_MS seam; inert unless the env var is set. * fix(daemon): close the hang-shaped gap in login-session death detection The conclusive-PAM-verdict trigger had one blind failure shape: login(1) hanging at the prompt past the probe bound (killed → inconclusive forever → the watch never fires). Three changes close it: - The death-watch probe gets its own 4s bound (the 500ms preflight bound exists for spawn-path latency, which doesn't apply off-path), so a slow-but-answering PAM stack isn't misread as a hang. - An inconclusive pipe probe escalates to a PTY-hosted probe via script(1) — a dead session's PAM stack may only misbehave under a real tty (the pipe-vs-PTY fidelity limit the preflight documents). - A streak of timeout-killed probes (which a live session never produces) is a second retirement trigger, at a higher threshold (5) and still gated on the degraded resolver, logged with a distinct cause so field logs discriminate the two paths. Every dead-session behavior — fast reject, prompt-then-EOF, or hang — now fires retirement; all inconclusive states still fail toward preserving the daemon. * fix(daemon): keep login-session retirement conclusive * fix(daemon): stop login watch before clean shutdown * fix(daemon): make login-session PTY probe reliable * fix(daemon): close login-session watch races * fix(daemon): ignore health probes for login watch activity
1040 lines
38 KiB
TypeScript
1040 lines
38 KiB
TypeScript
/* eslint-disable max-lines -- Why: one class owns the daemon socket protocol, routing, stream fanout, and session lifecycle. */
|
|
import { createServer, type Server, type Socket } from 'node:net'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { performance } from 'node:perf_hooks'
|
|
import { writeFileSync, chmodSync } from 'node:fs'
|
|
import { StringDecoder } from 'node:string_decoder'
|
|
import { encodeNdjson, createNdjsonParser } from './ndjson'
|
|
import { TerminalHost } from './terminal-host'
|
|
import { DaemonStreamDataBatcher } from './daemon-stream-data-batcher'
|
|
import {
|
|
BackgroundTransientFactRelay,
|
|
BACKGROUND_STREAM_DROP_ENABLED
|
|
} from './daemon-background-transient-facts'
|
|
import { extractHiddenStartupRendererQueryData } from '../../shared/terminal-reply-query-extraction'
|
|
import {
|
|
recordDaemonStreamBacklogEvent,
|
|
startDaemonStreamBacklogProbe
|
|
} from './daemon-stream-backlog-probe'
|
|
import { readCurrentProcessMacSystemResolverHealth } from '../network/macos-system-resolver-health'
|
|
import type { SubprocessHandle } from './session'
|
|
import { checkPtySpawnHealth } from './pty-subprocess'
|
|
import { createNoopDaemonFileLog, type DaemonFileLog } from './daemon-file-log'
|
|
import { isTuiAgent } from '../../shared/tui-agent-config'
|
|
import { parsePtyStartupIngressIntent } from '../../shared/pty-startup-ingress'
|
|
import { unlinkOwnedDaemonPidFile, unlinkOwnedDaemonTokenFile } from './daemon-spawner'
|
|
import {
|
|
CLEAN_DISCONNECT_PROTOCOL_VERSION,
|
|
PROTOCOL_VERSION,
|
|
NOTIFY_PREFIX,
|
|
SessionNotFoundError,
|
|
TerminalAttachCanceledError,
|
|
type HelloMessage,
|
|
type DaemonRequest
|
|
} from './types'
|
|
import {
|
|
isAgentSessionExecutionClaim,
|
|
isAgentSessionSurfaceBinding
|
|
} from '../../shared/agent-session-host-authority'
|
|
|
|
export type DaemonServerOptions = {
|
|
socketPath: string
|
|
tokenPath: string
|
|
pidPath?: string
|
|
launchNonce?: string
|
|
startedAtMs?: number
|
|
/** Direct-construction seam for protocol fixture tests; production never overrides it. */
|
|
protocolVersion?: number
|
|
onIdleShutdown?: () => void
|
|
/** Direct-construction-only controls; production uses the compiled initial-adoption timeout. */
|
|
initialAdoptionTestConfig?: {
|
|
timeoutMs: number
|
|
clock: {
|
|
setTimeout(callback: () => void, delayMs: number): unknown
|
|
clearTimeout(handle: unknown): void
|
|
now(): number
|
|
}
|
|
}
|
|
ptySpawnHealthCheck?: () => Promise<void>
|
|
preparePtySpawn?: () => Promise<void>
|
|
// Why: login-session death detection (#7936) probes on PTY-exit bursts and fresh app connections.
|
|
onPtySessionExit?: (sessionId: string) => void
|
|
onAuthenticatedClientPair?: () => void
|
|
log?: DaemonFileLog
|
|
spawnSubprocess: (opts: {
|
|
sessionId: string
|
|
cols: number
|
|
rows: number
|
|
cwd?: string
|
|
env?: Record<string, string>
|
|
command?: string
|
|
shellOverride?: string
|
|
}) => SubprocessHandle
|
|
}
|
|
|
|
type ConnectedClient = {
|
|
clientId: string
|
|
controlSocket: Socket
|
|
streamSocket: Socket | null
|
|
authenticatedPairEstablished: boolean
|
|
}
|
|
|
|
type PendingPtySpawnPreparation = {
|
|
canceled: boolean
|
|
// Why: preparations are keyed by sessionId, but a control-socket close must
|
|
// cancel only the disconnecting client's preps, not another client's (F4).
|
|
clientId: string
|
|
}
|
|
|
|
type PendingShutdownReply = {
|
|
start: () => void
|
|
}
|
|
|
|
export class DaemonServer {
|
|
// Why: survive long enough to adopt a first client pair, but don't orphan forever if the parent crashes first.
|
|
private static readonly INITIAL_ADOPTION_TIMEOUT_MS = 2 * 60 * 1000
|
|
private static readonly SHUTDOWN_REPLY_FLUSH_TIMEOUT_MS = 1_000
|
|
private server: Server | null = null
|
|
private token: string
|
|
private host: TerminalHost
|
|
private socketPath: string
|
|
private tokenPath: string
|
|
private pidPath: string | null
|
|
private launchNonce: string | null
|
|
private startedAtMs: number | null
|
|
private protocolVersion: number
|
|
private onIdleShutdown: () => void
|
|
private onAuthenticatedClientPair: () => void
|
|
private ptySpawnHealthCheck: () => Promise<void>
|
|
private preparePtySpawn: () => Promise<void>
|
|
private log: DaemonFileLog
|
|
private transportSockets = new Set<Socket>()
|
|
private createOrAttachInFlight = 0
|
|
private idleShutdownState: 'running' | 'idle-shutdown-pending' | 'shutting-down' = 'running'
|
|
private initialAdoptionTimer: unknown | null = null
|
|
private initialAdoptionDeadlineMs: number | null = null
|
|
private retirementRequested = false
|
|
private shutdownPromise: Promise<void> | null = null
|
|
private ordinaryShutdownServerClose: Promise<void> | null = null
|
|
private pendingShutdownReplies = new Map<string, PendingShutdownReply>()
|
|
private initialAdoptionTimeoutMs: number
|
|
private lifecycleClock: NonNullable<DaemonServerOptions['initialAdoptionTestConfig']>['clock']
|
|
|
|
private clients = new Map<string, ConnectedClient>()
|
|
private streamDataBatcher = new DaemonStreamDataBatcher(
|
|
(clientId) => this.clients.get(clientId),
|
|
{
|
|
isSessionDroppable: (sessionId) =>
|
|
BACKGROUND_STREAM_DROP_ENABLED && this.transientFactRelay.isBackgrounded(sessionId),
|
|
salvageDroppedData: (dropped) => {
|
|
if (!dropped.includes('\x1b')) {
|
|
return ''
|
|
}
|
|
const extracted = extractHiddenStartupRendererQueryData(dropped, '')
|
|
return (
|
|
extracted.statelessQueryData + extracted.statefulQueryData + extracted.oscColorQueryData
|
|
)
|
|
}
|
|
}
|
|
)
|
|
// Facts ride the stream queue as control entries so they hold byte order (else a fact could arrive after the reveal snapshot).
|
|
private transientFactRelay = new BackgroundTransientFactRelay((sessionId, fact) => {
|
|
const clientId = this.streamClientIdBySessionId.get(sessionId)
|
|
if (clientId) {
|
|
this.streamDataBatcher.enqueueControlEvent(clientId, sessionId, {
|
|
type: 'event',
|
|
event: 'transientFact',
|
|
sessionId,
|
|
payload: fact
|
|
})
|
|
}
|
|
})
|
|
private streamClientIdBySessionId = new Map<string, string>()
|
|
private lastInputAtBySessionId = new Map<string, number>()
|
|
private pendingPtySpawnPreparations = new Map<string, Set<PendingPtySpawnPreparation>>()
|
|
private stopStreamBacklogProbe: () => void = () => {}
|
|
|
|
// Why: bypass batching within this window so keystroke echo/redraws skip the daemon's fixed batch delay.
|
|
private static readonly INTERACTIVE_OUTPUT_WINDOW_MS = 100
|
|
private static readonly INTERACTIVE_OUTPUT_MAX_CHARS = 1024
|
|
|
|
constructor(opts: DaemonServerOptions) {
|
|
this.socketPath = opts.socketPath
|
|
this.tokenPath = opts.tokenPath
|
|
this.pidPath = opts.pidPath ?? null
|
|
this.protocolVersion = opts.protocolVersion ?? PROTOCOL_VERSION
|
|
this.launchNonce =
|
|
opts.launchNonce ??
|
|
(this.protocolVersion >= CLEAN_DISCONNECT_PROTOCOL_VERSION ? randomUUID() : null)
|
|
this.startedAtMs =
|
|
opts.startedAtMs ??
|
|
(this.protocolVersion >= CLEAN_DISCONNECT_PROTOCOL_VERSION
|
|
? Date.now() - process.uptime() * 1000
|
|
: null)
|
|
this.onIdleShutdown = opts.onIdleShutdown ?? (() => {})
|
|
this.initialAdoptionTimeoutMs =
|
|
opts.initialAdoptionTestConfig?.timeoutMs ?? DaemonServer.INITIAL_ADOPTION_TIMEOUT_MS
|
|
this.lifecycleClock = opts.initialAdoptionTestConfig?.clock ?? {
|
|
setTimeout: (callback, delayMs) => {
|
|
const timer = setTimeout(callback, delayMs)
|
|
timer.unref()
|
|
return timer
|
|
},
|
|
clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
now: () => Date.now()
|
|
}
|
|
this.token = randomUUID()
|
|
this.onAuthenticatedClientPair = opts.onAuthenticatedClientPair ?? (() => {})
|
|
this.host = new TerminalHost({
|
|
spawnSubprocess: opts.spawnSubprocess,
|
|
...(opts.onPtySessionExit ? { onSessionReaped: opts.onPtySessionExit } : {})
|
|
})
|
|
this.ptySpawnHealthCheck = opts.ptySpawnHealthCheck ?? checkPtySpawnHealth
|
|
this.preparePtySpawn = opts.preparePtySpawn ?? (() => Promise.resolve())
|
|
this.stopStreamBacklogProbe = startDaemonStreamBacklogProbe(() => ({
|
|
clients: Array.from(this.clients.values(), (client) => ({
|
|
clientId: client.clientId,
|
|
socketBufferedBytes: client.streamSocket?.writableLength ?? 0,
|
|
batcherQueuedChars: this.streamDataBatcher.queuedCharsForClient(client.clientId)
|
|
})),
|
|
backgroundedSessionIdSuffixes: this.transientFactRelay.backgroundedSessionIdSuffixes()
|
|
}))
|
|
this.log = opts.log ?? createNoopDaemonFileLog()
|
|
}
|
|
|
|
async start(): Promise<void> {
|
|
return new Promise((resolve, reject) => {
|
|
this.server = createServer((socket) => this.handleConnection(socket))
|
|
const onListenError = (err: Error): void => {
|
|
reject(err)
|
|
}
|
|
|
|
this.server.once('error', onListenError)
|
|
|
|
this.server.listen(this.socketPath, () => {
|
|
// Why: drop the startup error listener after bind so it doesn't retain this closure.
|
|
this.server?.off('error', onListenError)
|
|
writeFileSync(this.tokenPath, this.token, { mode: 0o600 })
|
|
try {
|
|
chmodSync(this.socketPath, 0o600)
|
|
} catch {
|
|
// Best-effort on platforms that support it
|
|
}
|
|
if (this.protocolVersion >= CLEAN_DISCONNECT_PROTOCOL_VERSION) {
|
|
// Why: a parent crash before the first full client pair must not leave an empty daemon alive forever.
|
|
this.armInitialAdoptionTimeout()
|
|
}
|
|
resolve()
|
|
})
|
|
})
|
|
}
|
|
|
|
async shutdown(): Promise<void> {
|
|
if (this.shutdownPromise) {
|
|
return this.shutdownPromise
|
|
}
|
|
const serverClose = this.beginOrdinaryShutdownFence()
|
|
this.shutdownPromise = this.finishOrdinaryShutdown(serverClose)
|
|
return this.shutdownPromise
|
|
}
|
|
|
|
private beginOrdinaryShutdownFence(): Promise<void> {
|
|
this.idleShutdownState = 'shutting-down'
|
|
this.cancelInitialAdoptionTimer()
|
|
this.ordinaryShutdownServerClose ??= this.beginServerClose()
|
|
return this.ordinaryShutdownServerClose
|
|
}
|
|
|
|
private async finishOrdinaryShutdown(serverClose: Promise<void>): Promise<void> {
|
|
this.unlinkOwnedEndpointArtifacts()
|
|
await this.disposeDaemonResources()
|
|
await serverClose
|
|
}
|
|
|
|
private unlinkOwnedEndpointArtifacts(): void {
|
|
// Why: ownership checks prevent removing a late replacement's token or PID record.
|
|
unlinkOwnedDaemonTokenFile(this.tokenPath, this.token)
|
|
if (this.pidPath && this.launchNonce) {
|
|
unlinkOwnedDaemonPidFile(this.pidPath, process.pid, this.launchNonce)
|
|
}
|
|
}
|
|
|
|
private async disposeDaemonResources(): Promise<void> {
|
|
this.stopStreamBacklogProbe()
|
|
this.transientFactRelay.dispose()
|
|
this.cancelAllPendingPtySpawnPreparations()
|
|
try {
|
|
await this.host.dispose()
|
|
} catch (err) {
|
|
// Why: an unreapable child must not block daemon exit — post-exit it reparents to init anyway.
|
|
this.log.log('shutdown-dispose-failed', {
|
|
error: err instanceof Error ? err.message : String(err)
|
|
})
|
|
}
|
|
this.streamDataBatcher.clear()
|
|
this.pendingShutdownReplies.clear()
|
|
|
|
for (const [, client] of this.clients) {
|
|
client.controlSocket.destroy()
|
|
client.streamSocket?.destroy()
|
|
}
|
|
this.clients.clear()
|
|
for (const socket of this.transportSockets) {
|
|
socket.destroy()
|
|
}
|
|
this.transportSockets.clear()
|
|
}
|
|
|
|
private beginServerClose(): Promise<void> {
|
|
const server = this.server
|
|
this.server = null
|
|
if (!server) {
|
|
return Promise.resolve()
|
|
}
|
|
return new Promise<void>((resolve) => {
|
|
// Why: close synchronously before any awaited cleanup so no new transport enters after the empty proof.
|
|
server.close(() => {
|
|
// Node owns unlinking its Unix listener; an extra unlink here could delete a concurrent replacement.
|
|
resolve()
|
|
})
|
|
})
|
|
}
|
|
|
|
private isIdle(): boolean {
|
|
return (
|
|
this.transportSockets.size === 0 &&
|
|
this.clients.size === 0 &&
|
|
this.createOrAttachInFlight === 0 &&
|
|
this.host.listSessions().length === 0
|
|
)
|
|
}
|
|
|
|
private reevaluateIdleShutdown(): void {
|
|
if (this.idleShutdownState !== 'running') {
|
|
return
|
|
}
|
|
if (this.retirementRequested) {
|
|
this.cancelInitialAdoptionTimer()
|
|
if (this.isIdle()) {
|
|
this.beginIdleShutdown()
|
|
}
|
|
return
|
|
}
|
|
if (!this.isIdle() || this.initialAdoptionDeadlineMs === null) {
|
|
this.cancelInitialAdoptionTimer()
|
|
return
|
|
}
|
|
if (this.initialAdoptionTimer !== null) {
|
|
return
|
|
}
|
|
const remainingMs = Math.max(0, this.initialAdoptionDeadlineMs - this.lifecycleClock.now())
|
|
if (remainingMs === 0) {
|
|
this.initialAdoptionDeadlineMs = null
|
|
this.retirementRequested = true
|
|
this.beginIdleShutdown()
|
|
return
|
|
}
|
|
this.initialAdoptionTimer = this.lifecycleClock.setTimeout(() => {
|
|
this.initialAdoptionTimer = null
|
|
this.initialAdoptionDeadlineMs = null
|
|
this.retirementRequested = true
|
|
this.beginIdleShutdown()
|
|
}, remainingMs)
|
|
}
|
|
|
|
private armInitialAdoptionTimeout(): void {
|
|
this.initialAdoptionDeadlineMs = this.lifecycleClock.now() + this.initialAdoptionTimeoutMs
|
|
this.reevaluateIdleShutdown()
|
|
}
|
|
|
|
private cancelInitialAdoptionTimer(): void {
|
|
if (this.initialAdoptionTimer === null) {
|
|
return
|
|
}
|
|
this.lifecycleClock.clearTimeout(this.initialAdoptionTimer)
|
|
this.initialAdoptionTimer = null
|
|
}
|
|
|
|
private beginIdleShutdown(): void {
|
|
this.initialAdoptionTimer = null
|
|
if (this.idleShutdownState !== 'running') {
|
|
return
|
|
}
|
|
this.idleShutdownState = 'idle-shutdown-pending'
|
|
if (!this.isIdle()) {
|
|
// Why: work admitted before the fence wins; clear pending state to keep it usable.
|
|
this.idleShutdownState = 'running'
|
|
this.reevaluateIdleShutdown()
|
|
return
|
|
}
|
|
|
|
this.idleShutdownState = 'shutting-down'
|
|
// beginServerClose() runs synchronously up to server.close() before any yield to a racing connection.
|
|
const serverClose = this.beginServerClose()
|
|
this.shutdownPromise = this.finishIdleShutdown(serverClose)
|
|
}
|
|
|
|
private async finishIdleShutdown(serverClose: Promise<void>): Promise<void> {
|
|
this.unlinkOwnedEndpointArtifacts()
|
|
await this.disposeDaemonResources()
|
|
await serverClose
|
|
this.onIdleShutdown()
|
|
}
|
|
|
|
private handleConnection(socket: Socket): void {
|
|
this.cancelInitialAdoptionTimer()
|
|
this.transportSockets.add(socket)
|
|
const removeTransport = (): void => {
|
|
this.transportSockets.delete(socket)
|
|
this.reevaluateIdleShutdown()
|
|
}
|
|
socket.once('close', removeTransport)
|
|
socket.on('error', () => socket.destroy())
|
|
|
|
if (this.idleShutdownState !== 'running') {
|
|
// Why: a connection accepted just before close() gets an explicit retry signal instead of dying mid-auth.
|
|
socket.end(
|
|
encodeNdjson({
|
|
type: 'hello',
|
|
ok: false,
|
|
error: 'Daemon temporarily unavailable; reconnect',
|
|
retryable: true
|
|
})
|
|
)
|
|
return
|
|
}
|
|
// Why: keep UTF-8 sequences intact across socket chunks before NDJSON parsing.
|
|
const decoder = new StringDecoder('utf8')
|
|
const parser = createNdjsonParser(
|
|
(msg) => this.handleFirstMessage(socket, msg, parser),
|
|
() => {
|
|
socket.destroy()
|
|
}
|
|
)
|
|
|
|
socket.on('data', (chunk) => parser.feed(decoder.write(chunk)))
|
|
}
|
|
|
|
private handleFirstMessage(
|
|
socket: Socket,
|
|
msg: unknown,
|
|
_parser: ReturnType<typeof createNdjsonParser>
|
|
): void {
|
|
const hello = msg as HelloMessage
|
|
if (hello.type !== 'hello') {
|
|
this.log.log('client-hello-rejected', { reason: 'expected-hello' })
|
|
socket.write(encodeNdjson({ type: 'hello', ok: false, error: 'Expected hello' }))
|
|
socket.destroy()
|
|
return
|
|
}
|
|
|
|
if (hello.version !== this.protocolVersion) {
|
|
this.log.log('client-hello-rejected', {
|
|
reason: 'protocol-mismatch',
|
|
clientVersion: hello.version
|
|
})
|
|
socket.write(encodeNdjson({ type: 'hello', ok: false, error: 'Protocol version mismatch' }))
|
|
socket.destroy()
|
|
return
|
|
}
|
|
|
|
if (hello.token !== this.token) {
|
|
this.log.log('client-hello-rejected', { reason: 'invalid-token', role: hello.role })
|
|
socket.write(encodeNdjson({ type: 'hello', ok: false, error: 'Invalid token' }))
|
|
socket.destroy()
|
|
return
|
|
}
|
|
|
|
this.log.log('client-hello-accepted', { role: hello.role, clientId: hello.clientId })
|
|
socket.write(
|
|
encodeNdjson({
|
|
type: 'hello',
|
|
ok: true,
|
|
...(this.launchNonce && this.startedAtMs
|
|
? {
|
|
daemonIdentity: {
|
|
pid: process.pid,
|
|
startedAtMs: this.startedAtMs,
|
|
launchNonce: this.launchNonce
|
|
}
|
|
}
|
|
: {})
|
|
})
|
|
)
|
|
|
|
if (hello.role === 'control') {
|
|
const previous = this.clients.get(hello.clientId)
|
|
const client: ConnectedClient = {
|
|
clientId: hello.clientId,
|
|
controlSocket: socket,
|
|
streamSocket: null,
|
|
authenticatedPairEstablished: false
|
|
}
|
|
this.clients.set(hello.clientId, client)
|
|
this.setupControlSocket(socket, hello.clientId)
|
|
if (previous) {
|
|
// Why: reconnect reuses clientId before stale close fires; cancel the old owner's preflight at handoff.
|
|
this.cancelPendingPtySpawnPreparationsForClient(hello.clientId)
|
|
this.recordFullyAuthenticatedDisconnect(previous.authenticatedPairEstablished)
|
|
// Why: tear down the old sockets after installing the new owner so a stale close can't delete the replacement.
|
|
previous.streamSocket?.destroy()
|
|
previous.controlSocket.destroy()
|
|
}
|
|
} else if (hello.role === 'stream') {
|
|
const client = this.clients.get(hello.clientId)
|
|
if (!client) {
|
|
// Why: a stream socket is meaningless without its control socket; drop the orphan.
|
|
socket.destroy()
|
|
return
|
|
}
|
|
this.setupStreamSocket(socket, client)
|
|
client.authenticatedPairEstablished = true
|
|
// Why: one-shot health probes authenticate only a control socket; they are not fresh app activity.
|
|
this.onAuthenticatedClientPair()
|
|
// A complete app connection (unlike a probe) re-owns the endpoint and cancels pending retirement.
|
|
this.initialAdoptionDeadlineMs = null
|
|
this.retirementRequested = false
|
|
this.cancelInitialAdoptionTimer()
|
|
}
|
|
}
|
|
|
|
private setupControlSocket(socket: Socket, clientId: string): void {
|
|
// Why: decode as a UTF-8 stream so emoji/Unicode split across chunks isn't corrupted.
|
|
const decoder = new StringDecoder('utf8')
|
|
const parser = createNdjsonParser(
|
|
(msg) => this.handleRequest(socket, clientId, msg as DaemonRequest),
|
|
() => {} // Ignore parse errors
|
|
)
|
|
|
|
// Remove the initial data listener and replace with the RPC parser
|
|
socket.removeAllListeners('data')
|
|
socket.on('data', (chunk) => parser.feed(decoder.write(chunk)))
|
|
|
|
socket.on('close', () => {
|
|
const client = this.clients.get(clientId)
|
|
if (client?.controlSocket !== socket) {
|
|
return
|
|
}
|
|
// Why: a client that disconnects mid-preflight would otherwise still create
|
|
// its daemon PTY, orphaning a durable, unattached session — cancel its preps (F4).
|
|
this.cancelPendingPtySpawnPreparationsForClient(clientId)
|
|
const wasFullyAuthenticated = client.authenticatedPairEstablished
|
|
this.streamDataBatcher.clear(clientId)
|
|
client.streamSocket?.destroy()
|
|
this.clients.delete(clientId)
|
|
this.recordFullyAuthenticatedDisconnect(wasFullyAuthenticated)
|
|
this.reevaluateIdleShutdown()
|
|
})
|
|
}
|
|
|
|
private recordFullyAuthenticatedDisconnect(wasFullyAuthenticated: boolean): void {
|
|
if (
|
|
!wasFullyAuthenticated ||
|
|
[...this.clients.values()].some((remaining) => remaining.authenticatedPairEstablished) ||
|
|
this.idleShutdownState !== 'running'
|
|
) {
|
|
return
|
|
}
|
|
// Why: once the last full client is gone, incomplete transports may block retirement but never erase it.
|
|
this.retirementRequested = true
|
|
}
|
|
|
|
private setupStreamSocket(socket: Socket, client: ConnectedClient): void {
|
|
const previous = client.streamSocket
|
|
socket.removeAllListeners('data')
|
|
client.streamSocket = socket
|
|
// Why: 'drain' is the wake-up for the batcher's shallow-gate held bulk.
|
|
socket.on('drain', () => {
|
|
this.streamDataBatcher.flush(client.clientId)
|
|
})
|
|
|
|
const cleanup = (): void => {
|
|
socket.removeListener('close', cleanup)
|
|
socket.removeListener('error', cleanup)
|
|
if (this.clients.get(client.clientId) !== client || client.streamSocket !== socket) {
|
|
return
|
|
}
|
|
// Why: a preflight that outlives its output channel would create an unattached daemon PTY.
|
|
this.cancelPendingPtySpawnPreparationsForClient(client.clientId)
|
|
this.streamDataBatcher.clear(client.clientId)
|
|
client.streamSocket = null
|
|
}
|
|
|
|
socket.on('close', cleanup)
|
|
socket.on('error', cleanup)
|
|
|
|
if (previous && previous !== socket) {
|
|
// Why: replacing a stream socket must not leave the old channel alive and untracked.
|
|
previous.destroy()
|
|
}
|
|
}
|
|
|
|
private async handleRequest(
|
|
socket: Socket,
|
|
clientId: string,
|
|
request: DaemonRequest
|
|
): Promise<void> {
|
|
const isNotify = request.id.startsWith(NOTIFY_PREFIX)
|
|
|
|
try {
|
|
const result = await this.routeRequest(clientId, request)
|
|
if (!isNotify) {
|
|
const pendingShutdown = this.pendingShutdownReplies.get(
|
|
this.shutdownReplyKey(clientId, request.id)
|
|
)
|
|
socket.write(encodeNdjson({ id: request.id, ok: true, payload: result }), () => {
|
|
pendingShutdown?.start()
|
|
})
|
|
}
|
|
} catch (err) {
|
|
if (!isNotify) {
|
|
socket.write(
|
|
encodeNdjson({
|
|
id: request.id,
|
|
ok: false,
|
|
error: err instanceof Error ? err.message : String(err)
|
|
})
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
private shutdownReplyKey(clientId: string, requestId: string): string {
|
|
return `${clientId}\u0000${requestId}`
|
|
}
|
|
|
|
private deferShutdownUntilReply(
|
|
clientId: string,
|
|
requestId: string,
|
|
socket: Socket,
|
|
finish: () => Promise<void>
|
|
): void {
|
|
const key = this.shutdownReplyKey(clientId, requestId)
|
|
let started = false
|
|
let timer: ReturnType<typeof setTimeout>
|
|
const start = (): void => {
|
|
if (started) {
|
|
return
|
|
}
|
|
started = true
|
|
clearTimeout(timer)
|
|
socket.off('close', start)
|
|
socket.off('error', start)
|
|
this.pendingShutdownReplies.delete(key)
|
|
if (!this.shutdownPromise) {
|
|
this.shutdownPromise = finish()
|
|
}
|
|
}
|
|
// Why: a non-reading peer must not pin a fenced daemon by holding its ack behind permanent socket backpressure.
|
|
timer = setTimeout(start, DaemonServer.SHUTDOWN_REPLY_FLUSH_TIMEOUT_MS)
|
|
timer.unref()
|
|
socket.once('close', start)
|
|
socket.once('error', start)
|
|
this.pendingShutdownReplies.set(key, { start })
|
|
}
|
|
|
|
private async preparePtySpawnUnlessCanceled(sessionId: string, clientId: string): Promise<void> {
|
|
const preparation: PendingPtySpawnPreparation = { canceled: false, clientId }
|
|
const pending = this.pendingPtySpawnPreparations.get(sessionId) ?? new Set()
|
|
pending.add(preparation)
|
|
this.pendingPtySpawnPreparations.set(sessionId, pending)
|
|
try {
|
|
// Why: register before the async probe so a concurrent close can cancel this creation before a subprocess exists.
|
|
await this.preparePtySpawn()
|
|
if (preparation.canceled) {
|
|
throw new TerminalAttachCanceledError(sessionId)
|
|
}
|
|
} finally {
|
|
pending.delete(preparation)
|
|
if (pending.size === 0) {
|
|
this.pendingPtySpawnPreparations.delete(sessionId)
|
|
}
|
|
}
|
|
}
|
|
|
|
private cancelPendingPtySpawnPreparations(sessionId: string): boolean {
|
|
const pending = this.pendingPtySpawnPreparations.get(sessionId)
|
|
if (!pending) {
|
|
return false
|
|
}
|
|
for (const preparation of pending) {
|
|
preparation.canceled = true
|
|
}
|
|
return true
|
|
}
|
|
|
|
private cancelAllPendingPtySpawnPreparations(): void {
|
|
for (const sessionId of this.pendingPtySpawnPreparations.keys()) {
|
|
this.cancelPendingPtySpawnPreparations(sessionId)
|
|
}
|
|
}
|
|
|
|
private cancelPendingPtySpawnPreparationsForClient(clientId: string): void {
|
|
for (const pending of this.pendingPtySpawnPreparations.values()) {
|
|
for (const preparation of pending) {
|
|
if (preparation.clientId === clientId) {
|
|
preparation.canceled = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private async routeRequest(clientId: string, request: DaemonRequest): Promise<unknown> {
|
|
const client = this.clients.get(clientId)
|
|
|
|
switch (request.type) {
|
|
case 'createOrAttach': {
|
|
if (this.idleShutdownState !== 'running') {
|
|
throw new Error('Daemon temporarily unavailable; reconnect')
|
|
}
|
|
if (!client?.authenticatedPairEstablished || client.streamSocket === null) {
|
|
// Why: a control-only replacement can't own terminal admission or erase the prior client's retirement request.
|
|
throw new Error('Daemon client connection is incomplete; reconnect')
|
|
}
|
|
this.createOrAttachInFlight++
|
|
const p = request.payload
|
|
let routedSessionId = p.sessionId
|
|
let result: Awaited<ReturnType<TerminalHost['createOrAttach']>>
|
|
try {
|
|
if (
|
|
p.agentSessionEnsure !== undefined &&
|
|
(!isAgentSessionExecutionClaim(p.agentSessionEnsure.claim) ||
|
|
!isAgentSessionSurfaceBinding(p.agentSessionEnsure.surface))
|
|
) {
|
|
throw new Error('agent_session_identity_required')
|
|
}
|
|
await this.preparePtySpawnUnlessCanceled(p.sessionId, clientId)
|
|
result = await this.host.createOrAttach({
|
|
sessionId: p.sessionId,
|
|
cols: p.cols,
|
|
rows: p.rows,
|
|
cwd: p.cwd,
|
|
env: p.env,
|
|
envToDelete: p.envToDelete,
|
|
command: p.command,
|
|
startupCommandDelivery: p.startupCommandDelivery,
|
|
// Why: RPC payloads are untrusted JSON; persist only the allowlisted routing enum, never arbitrary identity.
|
|
...(isTuiAgent(p.launchAgent) ? { launchAgent: p.launchAgent } : {}),
|
|
shellOverride: p.shellOverride,
|
|
terminalWindowsWslDistro: p.terminalWindowsWslDistro,
|
|
terminalWindowsPowerShellImplementation: p.terminalWindowsPowerShellImplementation,
|
|
shellReadySupported: p.shellReadySupported,
|
|
historySeed: p.historySeed,
|
|
startupIngress: parsePtyStartupIngressIntent(p.startupIngress),
|
|
...(p.shellReadyTimeoutMs !== undefined
|
|
? { shellReadyTimeoutMs: p.shellReadyTimeoutMs }
|
|
: {}),
|
|
...(p.agentSessionEnsure ? { agentSessionEnsure: p.agentSessionEnsure } : {}),
|
|
onSessionResolved: (sessionId) => {
|
|
routedSessionId = sessionId
|
|
},
|
|
streamClient: {
|
|
onData: (data, rawLength = data.length, transformed = false, seq) => {
|
|
// Scan BEFORE enqueue: the batcher may drop this chunk, but its facts must be captured regardless.
|
|
this.transientFactRelay.onSessionData(routedSessionId, data)
|
|
const lastInputAt = this.lastInputAtBySessionId.get(routedSessionId)
|
|
const isInteractiveOutput =
|
|
data.length <= DaemonServer.INTERACTIVE_OUTPUT_MAX_CHARS &&
|
|
lastInputAt !== undefined &&
|
|
performance.now() - lastInputAt <= DaemonServer.INTERACTIVE_OUTPUT_WINDOW_MS
|
|
this.streamDataBatcher.enqueue(clientId, routedSessionId, data, {
|
|
flushImmediately: isInteractiveOutput,
|
|
flushMaxChars: DaemonServer.INTERACTIVE_OUTPUT_MAX_CHARS,
|
|
rawLength,
|
|
transformed,
|
|
seq
|
|
})
|
|
},
|
|
onExit: (code, incarnationId) => {
|
|
// Why: exit tears down renderer handlers, so it must ride the ordered queue behind final output.
|
|
this.log.log('session-exited', { sessionId: routedSessionId, code })
|
|
this.streamDataBatcher.enqueueControlEvent(clientId, routedSessionId, {
|
|
type: 'event',
|
|
event: 'exit',
|
|
sessionId: routedSessionId,
|
|
payload: { code, incarnationId }
|
|
})
|
|
this.streamDataBatcher.flush(clientId)
|
|
recordDaemonStreamBacklogEvent('sessionExit', {
|
|
sessionIdSuffix: routedSessionId.slice(-10)
|
|
})
|
|
this.transientFactRelay.onSessionExit(routedSessionId)
|
|
this.streamClientIdBySessionId.delete(routedSessionId)
|
|
this.lastInputAtBySessionId.delete(routedSessionId)
|
|
this.reevaluateIdleShutdown()
|
|
}
|
|
}
|
|
})
|
|
} finally {
|
|
this.createOrAttachInFlight--
|
|
this.reevaluateIdleShutdown()
|
|
}
|
|
routedSessionId = result.agentSessionEnsure?.owner.ptyId ?? p.sessionId
|
|
this.streamClientIdBySessionId.set(routedSessionId, clientId)
|
|
// Why an attach-time marker: background resync can precede this attach, so scan suppression must start at the new stream's head.
|
|
if (this.transientFactRelay.isBackgrounded(routedSessionId)) {
|
|
this.streamDataBatcher.enqueueControlEvent(clientId, routedSessionId, {
|
|
type: 'event',
|
|
event: 'sessionBackgroundMarker',
|
|
sessionId: routedSessionId,
|
|
payload: { background: true }
|
|
})
|
|
}
|
|
this.log.log(result.isNew ? 'session-created' : 'session-attached', {
|
|
sessionId: routedSessionId,
|
|
pid: result.pid
|
|
})
|
|
return {
|
|
isNew: result.isNew,
|
|
snapshot: result.snapshot,
|
|
pid: result.pid,
|
|
shellState: result.shellState,
|
|
incarnationId: result.incarnationId,
|
|
...(result.launchAgent ? { launchAgent: result.launchAgent } : {}),
|
|
wslDistro: result.wslDistro,
|
|
...(result.historySeeded !== undefined ? { historySeeded: result.historySeeded } : {}),
|
|
...(result.agentSessionEnsure ? { agentSessionEnsure: result.agentSessionEnsure } : {})
|
|
}
|
|
}
|
|
|
|
case 'cancelCreateOrAttach':
|
|
this.cancelPendingPtySpawnPreparations(request.payload.sessionId)
|
|
return {}
|
|
|
|
case 'closeStartupQueryAuthority':
|
|
return {
|
|
appliedSeq: this.host.closeStartupQueryAuthority(request.payload.sessionId)
|
|
}
|
|
|
|
case 'write':
|
|
try {
|
|
this.lastInputAtBySessionId.set(request.payload.sessionId, performance.now())
|
|
this.host.write(request.payload.sessionId, request.payload.data)
|
|
} catch (err) {
|
|
this.lastInputAtBySessionId.delete(request.payload.sessionId)
|
|
if (err instanceof SessionNotFoundError) {
|
|
this.sendExitEvent(client, request.payload.sessionId, -1)
|
|
}
|
|
throw err
|
|
}
|
|
return {}
|
|
|
|
case 'resize':
|
|
try {
|
|
this.host.resize(request.payload.sessionId, request.payload.cols, request.payload.rows)
|
|
} catch (err) {
|
|
if (err instanceof SessionNotFoundError) {
|
|
this.sendExitEvent(client, request.payload.sessionId, -1)
|
|
}
|
|
throw err
|
|
}
|
|
return {}
|
|
|
|
case 'pausePty':
|
|
this.host.pauseProducer(request.payload.sessionId)
|
|
return {}
|
|
|
|
case 'resumePty':
|
|
this.host.resumeProducer(request.payload.sessionId)
|
|
return {}
|
|
|
|
case 'setSessionBackground': {
|
|
const sessionId = request.payload.sessionId
|
|
const background = request.payload.background === true
|
|
recordDaemonStreamBacklogEvent('setSessionBackground', {
|
|
sessionIdSuffix: sessionId.slice(-10),
|
|
background
|
|
})
|
|
if (!this.transientFactRelay.setSessionBackground(sessionId, background)) {
|
|
return {}
|
|
}
|
|
if (background) {
|
|
// Seed the fresh relay tracker with the emulator's dangling escape so a handoff-split sequence still parses.
|
|
this.transientFactRelay.seedSessionScanState(
|
|
sessionId,
|
|
this.host.getPartialEscapeTailAnsi(sessionId)
|
|
)
|
|
}
|
|
const streamClientId = this.streamClientIdBySessionId.get(sessionId)
|
|
if (!streamClientId) {
|
|
// Not attached yet — the attach-time marker covers the handoff.
|
|
return {}
|
|
}
|
|
// Reveal intentionally keeps the queued tail: main needs those bytes, and the normal flush/drain delivers them in order ahead of the marker.
|
|
const scanSeedAnsi = background ? '' : this.host.getPartialEscapeTailAnsi(sessionId)
|
|
this.streamDataBatcher.enqueueControlEvent(streamClientId, sessionId, {
|
|
type: 'event',
|
|
event: 'sessionBackgroundMarker',
|
|
sessionId,
|
|
payload: {
|
|
background,
|
|
...(scanSeedAnsi.length > 0 ? { scanSeedAnsi } : {})
|
|
}
|
|
})
|
|
return {}
|
|
}
|
|
|
|
case 'kill': {
|
|
const canceledPendingSpawn = this.cancelPendingPtySpawnPreparations(
|
|
request.payload.sessionId
|
|
)
|
|
this.lastInputAtBySessionId.delete(request.payload.sessionId)
|
|
this.log.log('session-killed', {
|
|
sessionId: request.payload.sessionId,
|
|
immediate: request.payload.immediate === true
|
|
})
|
|
try {
|
|
await this.host.kill(request.payload.sessionId, { immediate: request.payload.immediate })
|
|
} catch (error) {
|
|
// Why: a kill that wins before session registration already canceled the pending spawn, so its intent is done.
|
|
if (!(canceledPendingSpawn && error instanceof SessionNotFoundError)) {
|
|
throw error
|
|
}
|
|
}
|
|
return {}
|
|
}
|
|
|
|
case 'signal':
|
|
this.host.signal(request.payload.sessionId, request.payload.signal)
|
|
return {}
|
|
|
|
case 'detach':
|
|
// Note: detach token handling simplified — full impl would track tokens per client
|
|
this.log.log('session-detached', { sessionId: request.payload.sessionId })
|
|
return {}
|
|
|
|
case 'getCwd':
|
|
return { cwd: await this.host.getCwd(request.payload.sessionId) }
|
|
|
|
case 'getForegroundProcess':
|
|
return { foregroundProcess: this.host.getForegroundProcess(request.payload.sessionId) }
|
|
|
|
case 'confirmForegroundProcess':
|
|
return {
|
|
foregroundProcess: await this.host.confirmForegroundProcess(request.payload.sessionId)
|
|
}
|
|
|
|
case 'clearScrollback':
|
|
this.host.clearScrollback(request.payload.sessionId)
|
|
return {}
|
|
|
|
case 'listSessions':
|
|
return { sessions: this.host.listSessions() }
|
|
|
|
case 'shutdownIfIdle': {
|
|
const authenticatedClient = this.clients.get(clientId)
|
|
const retiring =
|
|
authenticatedClient !== undefined &&
|
|
authenticatedClient.streamSocket !== null &&
|
|
this.clients.size === 1 &&
|
|
this.createOrAttachInFlight === 0 &&
|
|
this.host.listSessions().length === 0 &&
|
|
[...this.transportSockets].every(
|
|
(transport) =>
|
|
transport === authenticatedClient.controlSocket ||
|
|
transport === authenticatedClient.streamSocket
|
|
)
|
|
if (!retiring) {
|
|
return { retiring: false }
|
|
}
|
|
this.idleShutdownState = 'shutting-down'
|
|
this.initialAdoptionDeadlineMs = null
|
|
this.retirementRequested = false
|
|
this.cancelInitialAdoptionTimer()
|
|
// Why: close before acknowledging retirement so no new terminal races between the empty proof and disposal.
|
|
const serverClose = this.beginServerClose()
|
|
this.deferShutdownUntilReply(clientId, request.id, authenticatedClient.controlSocket, () =>
|
|
this.finishIdleShutdown(serverClose)
|
|
)
|
|
return { retiring: true }
|
|
}
|
|
|
|
case 'getSnapshot': {
|
|
const snapshotStart = performance.now()
|
|
const requestedScrollbackRows = request.payload.scrollbackRows
|
|
const scrollbackRows =
|
|
typeof requestedScrollbackRows === 'number' && Number.isFinite(requestedScrollbackRows)
|
|
? Math.max(0, Math.min(50_000, Math.floor(requestedScrollbackRows)))
|
|
: undefined
|
|
const snapshot = this.host.getSnapshot(request.payload.sessionId, { scrollbackRows })
|
|
const snapshotMs = performance.now() - snapshotStart
|
|
if (snapshotMs >= 25) {
|
|
// Serialize stalls block the daemon's single thread; surface them to attribute field typing stalls (issue #5096 family).
|
|
recordDaemonStreamBacklogEvent('slowGetSnapshot', {
|
|
sessionIdSuffix: request.payload.sessionId.slice(-10),
|
|
snapshotMs: Math.round(snapshotMs)
|
|
})
|
|
}
|
|
return { snapshot }
|
|
}
|
|
|
|
case 'getSize':
|
|
return { size: this.host.getAppliedSize(request.payload.sessionId) }
|
|
|
|
case 'takePendingOutput':
|
|
// Why no await: with includeSnapshot, drain+serialize must share one sync turn or cold restore replays doubled PTY bytes.
|
|
return this.host.takePendingOutput(
|
|
request.payload.sessionId,
|
|
request.payload.includeSnapshot === true,
|
|
{ teardownSnapshot: request.payload.teardownSnapshot === true }
|
|
)
|
|
|
|
case 'ping':
|
|
return { pong: true }
|
|
|
|
case 'systemResolverHealth':
|
|
return { health: await readCurrentProcessMacSystemResolverHealth() }
|
|
|
|
case 'ptySpawnHealth':
|
|
await this.ptySpawnHealthCheck()
|
|
return { healthy: true }
|
|
|
|
case 'shutdown': {
|
|
this.log.log('shutdown', {
|
|
reason: 'rpc',
|
|
killSessions: request.payload.killSessions === true
|
|
})
|
|
const serverClose = this.beginOrdinaryShutdownFence()
|
|
if (request.payload.killSessions) {
|
|
try {
|
|
await this.host.dispose()
|
|
} catch (err) {
|
|
// Why: shutdown must always self-terminate; failed owners stay retryable for the follow-up shutdown() below.
|
|
this.log.log('shutdown-dispose-failed', {
|
|
error: err instanceof Error ? err.message : String(err)
|
|
})
|
|
}
|
|
}
|
|
const controlSocket = this.clients.get(clientId)?.controlSocket
|
|
if (controlSocket) {
|
|
this.deferShutdownUntilReply(clientId, request.id, controlSocket, () =>
|
|
this.finishOrdinaryShutdown(serverClose)
|
|
)
|
|
} else if (!this.shutdownPromise) {
|
|
this.shutdownPromise = this.finishOrdinaryShutdown(serverClose)
|
|
}
|
|
return {}
|
|
}
|
|
}
|
|
throw new Error(`Unknown request type: ${(request as { type: string }).type}`)
|
|
}
|
|
|
|
private sendExitEvent(
|
|
client: ConnectedClient | undefined,
|
|
sessionId: string,
|
|
code: number
|
|
): void {
|
|
if (!client?.streamSocket) {
|
|
return
|
|
}
|
|
// Why: write/resize don't wait for replies, so this synthetic exit is the renderer's only signal to clear stale pane bindings.
|
|
this.streamDataBatcher.enqueueControlEvent(client.clientId, sessionId, {
|
|
type: 'event',
|
|
event: 'exit',
|
|
sessionId,
|
|
payload: { code }
|
|
})
|
|
this.streamDataBatcher.flush(client.clientId)
|
|
}
|
|
}
|