mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 08:03:12 +00:00
`pty.shutdown` was the only signal the relay ever got that a tab had closed, and it retired nothing: it requested a kill and returned. Every retirement path in the relay keys on proof of process death, so when the kill did not reap the pane shell the relay kept forwarding the orphaned agent's hook events as a live agent pane with no tab, and kept publishing its `agentSessionOwners` from `pty.listProcesses` with no liveness check at all. The relay now records the retirement the client stated, drops the pane's cached agent status with it, refuses to forward or replay posts from a retired pane, verifies the kill actually landed instead of assuming it, and reaps any PTY whose pid it can prove is gone before listing it. No wire change: no new RPC, field or stream opcode.
354 lines
14 KiB
TypeScript
354 lines
14 KiB
TypeScript
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { join } from 'node:path'
|
|
|
|
import {
|
|
ORCA_HOOK_PROTOCOL_VERSION,
|
|
ORCA_HOOK_RAW_JSON_TRANSPORT
|
|
} from '../shared/agent-hook-types'
|
|
import {
|
|
clearAllListenerCaches,
|
|
clearPaneCacheState,
|
|
createHookListenerState,
|
|
type HookListenerState
|
|
} from '../shared/agent-hook-listener/listener-state'
|
|
import {
|
|
getEndpointFileName,
|
|
writeEndpointFile
|
|
} from '../shared/agent-hook-listener/endpoint-publication'
|
|
import { HOOK_REQUEST_SLOWLORIS_MS } from '../shared/agent-hook-listener/listener-limits'
|
|
import { normalizeHookPayload } from '../shared/agent-hook-listener'
|
|
import { mergeAgentHookRequestHeaders } from '../shared/agent-hook-listener/hook-envelope'
|
|
import { readRequestBody } from '../shared/agent-hook-listener/request-body'
|
|
import { resolveHookSource } from '../shared/agent-hook-listener/source-routing'
|
|
import type { AgentHookEventPayload } from '../shared/agent-hook-listener/listener-event'
|
|
import {
|
|
createHookTransportInterferenceTracker,
|
|
describeHookTransportInterference,
|
|
isHookRequestTruncatedError
|
|
} from '../shared/agent-hook-transport-interference'
|
|
import {
|
|
isAgentHookSource,
|
|
REMOTE_AGENT_HOOK_ENV,
|
|
type AgentHookRelayEnvelope,
|
|
type AgentHookSource
|
|
} from '../shared/agent-hook-relay'
|
|
import {
|
|
buildSpoolHookBody,
|
|
drainAgentHookSpool,
|
|
type SpoolRecord
|
|
} from '../shared/agent-hook-spool'
|
|
import { buildRelayHookPtyEnv, defaultEndpointDir } from './agent-hook-endpoint-coordinates'
|
|
import { buildRelayHookEnvelope, hookBodyEnv, hookBodyVersion } from './agent-hook-envelope-build'
|
|
import { AgentHookResultRetryScheduler } from './agent-hook-result-retry-scheduler'
|
|
import {
|
|
evictCachedPanesOverCap,
|
|
selectReplayableCachedPanes
|
|
} from './agent-hook-cached-pane-status'
|
|
|
|
export type RelayHookForward = (envelope: AgentHookRelayEnvelope) => void
|
|
|
|
export type RelayHookServerOptions = {
|
|
/** Where to put endpoint.env / endpoint.cmd. Defaults to `$HOME/.orca-relay/agent-hooks`. */
|
|
endpointDir?: string
|
|
/** Env tag forwarded into hook payloads. Defaults to "remote", which main excludes from dev-vs-prod mismatch warnings. */
|
|
env?: string
|
|
/** Fixed auth token. WSL relay passes the host-issued token (already in guest env via WSLENV) so unmodified hook clients authenticate. Defaults to a fresh UUID. */
|
|
token?: string
|
|
/** Preferred bind port. WSL relay passes the Windows listener's port so env-sourced client coords stay truthful; falls back to :0 if occupied. Defaults to :0. */
|
|
preferredPort?: number
|
|
forward: RelayHookForward
|
|
/**
|
|
* True when the host has been told this pane's tab is gone and no PTY has re-bound the paneKey.
|
|
* Posts from such a pane come from a process the user already closed, so they describe no surface
|
|
* any client owns. Defaults to "never retired", which is the pre-existing behaviour — a listener
|
|
* with no PTY handler behind it (the WSL relay) keeps forwarding everything.
|
|
*/
|
|
isPaneSurfaceRetired?: (paneKey: string) => boolean
|
|
}
|
|
|
|
export type RelayHookServerStartOptions = {
|
|
publishEndpoint?: boolean
|
|
}
|
|
|
|
export class RelayAgentHookServer {
|
|
private server: ReturnType<typeof createServer> | null = null
|
|
private port = 0
|
|
private token = ''
|
|
private env: string
|
|
private endpointDir: string
|
|
private endpointFilePath: string
|
|
private endpointFileWritten = false
|
|
private state: HookListenerState = createHookListenerState()
|
|
private transportInterference = createHookTransportInterferenceTracker((report) => {
|
|
process.stderr.write(`${describeHookTransportInterference(report)}\n`)
|
|
})
|
|
// Why: retain envelope metadata so replays match live POSTs.
|
|
// Invariant: keys mirror state.lastStatusByPaneKey, populated/cleared in lockstep.
|
|
private lastEnvelopeMetaByPaneKey = new Map<
|
|
string,
|
|
{ source: AgentHookSource; env?: string; version?: string }
|
|
>()
|
|
private forward: RelayHookForward
|
|
private isPaneSurfaceRetired: (paneKey: string) => boolean
|
|
private fixedToken: string | undefined
|
|
private preferredPort: number
|
|
private portFallbackApplied = false
|
|
private retryScheduler: AgentHookResultRetryScheduler
|
|
|
|
constructor(options: RelayHookServerOptions) {
|
|
this.env = options.env ?? REMOTE_AGENT_HOOK_ENV
|
|
this.endpointDir = options.endpointDir ?? defaultEndpointDir()
|
|
this.endpointFilePath = join(this.endpointDir, getEndpointFileName())
|
|
this.fixedToken = options.token
|
|
this.preferredPort = options.preferredPort ?? 0
|
|
this.forward = options.forward
|
|
this.isPaneSurfaceRetired = options.isPaneSurfaceRetired ?? (() => false)
|
|
this.retryScheduler = new AgentHookResultRetryScheduler({
|
|
state: this.state,
|
|
env: this.env,
|
|
isListening: () => this.server !== null,
|
|
applyEvent: (event, source, env, version) => {
|
|
this.applyEvent(event, source, env, version)
|
|
}
|
|
})
|
|
}
|
|
|
|
async start(options: RelayHookServerStartOptions = {}): Promise<void> {
|
|
if (this.server) {
|
|
return
|
|
}
|
|
this.token = this.fixedToken ?? randomUUID()
|
|
this.endpointFileWritten = false
|
|
this.portFallbackApplied = false
|
|
try {
|
|
drainAgentHookSpool({
|
|
endpointDir: this.endpointDir,
|
|
getPersistedLaunchTokenHash: () => undefined,
|
|
ingest: (record) => this.ingestSpoolRecord(record)
|
|
})
|
|
} catch (err) {
|
|
// Why: a downstream relay failure must not prevent the loopback listener from starting;
|
|
// the untruncated spool file remains available for retry on the next restart.
|
|
process.stderr.write(
|
|
`[relay-hook-server] spool replay failed: ${err instanceof Error ? err.message : String(err)}\n`
|
|
)
|
|
}
|
|
try {
|
|
await this.listenOn(this.preferredPort)
|
|
} catch (err) {
|
|
// Why: fall back to an ephemeral port on EADDRINUSE; clients use the endpoint file.
|
|
if (this.preferredPort > 0 && (err as NodeJS.ErrnoException)?.code === 'EADDRINUSE') {
|
|
this.portFallbackApplied = true
|
|
await this.listenOn(0)
|
|
} else {
|
|
throw err
|
|
}
|
|
}
|
|
if (options.publishEndpoint !== false) {
|
|
this.publishEndpointFile()
|
|
}
|
|
}
|
|
|
|
get usedPortFallback(): boolean {
|
|
return this.portFallbackApplied
|
|
}
|
|
|
|
private listenOn(port: number): Promise<void> {
|
|
this.server = createServer((req, res) => this.handleRequest(req, res))
|
|
return new Promise<void>((resolve, reject) => {
|
|
const onStartupError = (err: Error): void => {
|
|
this.server?.off('listening', onListening)
|
|
// Why: clear failed server refs so later start() calls can retry.
|
|
this.server = null
|
|
reject(err)
|
|
}
|
|
const onListening = (): void => {
|
|
this.server?.off('error', onStartupError)
|
|
this.server?.on('error', (err) => {
|
|
process.stderr.write(`[relay-hook-server] server error: ${err.message}\n`)
|
|
})
|
|
const address = this.server!.address()
|
|
if (address && typeof address === 'object') {
|
|
this.port = address.port
|
|
}
|
|
resolve()
|
|
}
|
|
this.server!.once('error', onStartupError)
|
|
// Why: loopback only — reachable by the in-box agent CLI (127.0.0.1), not from outside the box.
|
|
this.server!.listen(port, '127.0.0.1', onListening)
|
|
})
|
|
}
|
|
|
|
publishEndpointFile(): boolean {
|
|
if (this.port <= 0 || !this.token) {
|
|
this.endpointFileWritten = false
|
|
return false
|
|
}
|
|
this.endpointFileWritten = writeEndpointFile(this.endpointDir, this.endpointFilePath, {
|
|
port: this.port,
|
|
token: this.token,
|
|
env: this.env,
|
|
version: ORCA_HOOK_PROTOCOL_VERSION,
|
|
transport: ORCA_HOOK_RAW_JSON_TRANSPORT
|
|
})
|
|
return this.endpointFileWritten
|
|
}
|
|
|
|
stop(): void {
|
|
this.server?.close()
|
|
this.server = null
|
|
this.port = 0
|
|
this.token = ''
|
|
this.endpointFileWritten = false
|
|
this.retryScheduler.clearAll()
|
|
clearAllListenerCaches(this.state)
|
|
this.lastEnvelopeMetaByPaneKey.clear()
|
|
}
|
|
|
|
/** Request-driven replay: re-forwards each cached paneKey payload as a fresh notification. Forwards are
|
|
* issued before the request handler returns, so the response trails all replayed notifications. */
|
|
replayCachedPayloadsForPanes(): number {
|
|
const replayable = selectReplayableCachedPanes({
|
|
cachedByPaneKey: this.state.lastStatusByPaneKey,
|
|
metaByPaneKey: this.lastEnvelopeMetaByPaneKey,
|
|
isPaneSurfaceRetired: this.isPaneSurfaceRetired,
|
|
dropPane: (paneKey) => this.clearPaneState(paneKey)
|
|
})
|
|
for (const { event, meta } of replayable) {
|
|
this.forward(
|
|
buildRelayHookEnvelope(event, meta.source, meta.env, meta.version, { isReplay: true })
|
|
)
|
|
}
|
|
return replayable.length
|
|
}
|
|
|
|
/** Drop a paneKey's cached entries on PTY exit so a terminated pane can't resurface as a ghost event on reconnect. */
|
|
clearPaneState(paneKey: string): void {
|
|
this.retryScheduler.clearAssistantMessageRetry(paneKey)
|
|
this.retryScheduler.clearCodexSubagentPoll(paneKey)
|
|
clearPaneCacheState(this.state, paneKey)
|
|
this.lastEnvelopeMetaByPaneKey.delete(paneKey)
|
|
}
|
|
|
|
/** Env vars to inject into relay-spawned PTYs so the hook script/plugin POSTs back to this loopback server. */
|
|
buildPtyEnv(): Record<string, string> {
|
|
return buildRelayHookPtyEnv({
|
|
port: this.port,
|
|
token: this.token,
|
|
env: this.env,
|
|
endpointFilePath: this.endpointFilePath,
|
|
endpointFileWritten: this.endpointFileWritten
|
|
})
|
|
}
|
|
|
|
/** Test-only / diagnostics accessor. */
|
|
getCoordinates(): { port: number; token: string; endpointFilePath: string } {
|
|
return { port: this.port, token: this.token, endpointFilePath: this.endpointFilePath }
|
|
}
|
|
|
|
// ─── Private ──────────────────────────────────────────────────────
|
|
|
|
private async handleRequest(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
if (req.method !== 'POST') {
|
|
res.writeHead(404)
|
|
res.end()
|
|
return
|
|
}
|
|
if (req.headers['x-orca-agent-hook-token'] !== this.token) {
|
|
res.writeHead(403)
|
|
res.end()
|
|
return
|
|
}
|
|
// Why: track our own destroy so the slowloris cap can't be misread as outside interference.
|
|
let destroyedBySlowlorisCap = false
|
|
req.setTimeout(HOOK_REQUEST_SLOWLORIS_MS, () => {
|
|
destroyedBySlowlorisCap = true
|
|
req.destroy()
|
|
})
|
|
try {
|
|
const pathname = new URL(req.url ?? '/', 'http://127.0.0.1').pathname
|
|
const source = resolveHookSource(pathname)
|
|
if (!source) {
|
|
res.writeHead(404)
|
|
res.end()
|
|
return
|
|
}
|
|
const body = await readRequestBody(req)
|
|
const hookBody = mergeAgentHookRequestHeaders(body, req.headers)
|
|
const event = normalizeHookPayload(this.state, source, hookBody, this.env, {
|
|
deferCompactOwnershipToClient: true
|
|
})
|
|
if (event) {
|
|
// TODO: once normalizeHookPayload returns validated env/version, drop bodyEnv/bodyVersion and source them from the listener result.
|
|
const env = hookBodyEnv(hookBody)
|
|
const version = hookBodyVersion(hookBody)
|
|
this.applyEvent(event, source, env, version)
|
|
this.retryScheduler.scheduleAssistantMessageRetry(source, hookBody, event, env, version)
|
|
this.retryScheduler.scheduleCodexSubagentPoll(source, hookBody, event, env, version)
|
|
}
|
|
res.writeHead(204)
|
|
res.end()
|
|
} catch (err) {
|
|
// Why (#11217): a remote host can run the same IDS; count truncations here so a blocked SSH
|
|
// relay reports the cause instead of an anonymous "hook request failed".
|
|
if (isHookRequestTruncatedError(err) && !destroyedBySlowlorisCap) {
|
|
this.transportInterference.record({ source: null, error: err })
|
|
}
|
|
// Why: hooks fail open (204 on any error) so a buggy agent never blocks the run; still log so the 204 doesn't mask bugs.
|
|
process.stderr.write(
|
|
`[relay-hook-server] hook request failed: ${err instanceof Error ? err.message : String(err)}\n`
|
|
)
|
|
res.writeHead(204)
|
|
res.end()
|
|
}
|
|
}
|
|
|
|
private applyEvent(
|
|
event: AgentHookEventPayload,
|
|
source: AgentHookSource,
|
|
env?: string,
|
|
version?: string,
|
|
options: { isReplay?: boolean } = {}
|
|
): void {
|
|
// Why: this post came from a process still running inside a pane whose tab the user closed.
|
|
// Caching or forwarding it makes every connected client advertise a live, resumable agent pane
|
|
// that no tab owns — the advertisement that ends up auto-typing a second `--resume` onto a
|
|
// transcript the orphan is still writing (#12447). Drop the stale cache with it.
|
|
if (this.isPaneSurfaceRetired(event.paneKey)) {
|
|
this.clearPaneState(event.paneKey)
|
|
return
|
|
}
|
|
if (event.payload.state !== 'done' || event.payload.lastAssistantMessage) {
|
|
this.retryScheduler.clearAssistantMessageRetry(event.paneKey)
|
|
}
|
|
// Why: keep PostCompact identity in the replay cache so the client can re-run ownership when
|
|
// it reconnects. Stripping it would let a cold relay replay a completion as an ordinary `done`
|
|
// row and resurrect a pane that the client had already retired.
|
|
const cachedEvent = event
|
|
// Why: delete-then-set makes Map insertion order = recency, so the cap below evicts the longest-idle pane.
|
|
this.state.lastStatusByPaneKey.delete(event.paneKey)
|
|
this.state.lastStatusByPaneKey.set(event.paneKey, cachedEvent)
|
|
this.lastEnvelopeMetaByPaneKey.delete(event.paneKey)
|
|
this.lastEnvelopeMetaByPaneKey.set(event.paneKey, { source, env, version })
|
|
evictCachedPanesOverCap(this.state.lastStatusByPaneKey, (key) => this.clearPaneState(key))
|
|
this.forward(buildRelayHookEnvelope(event, source, env, version, options))
|
|
}
|
|
|
|
private ingestSpoolRecord(record: SpoolRecord): void {
|
|
if (!isAgentHookSource(record.source)) {
|
|
return
|
|
}
|
|
const body = buildSpoolHookBody(record)
|
|
const event = normalizeHookPayload(this.state, record.source, body, this.env, {
|
|
deferCompactOwnershipToClient: true
|
|
})
|
|
if (!event) {
|
|
return
|
|
}
|
|
this.applyEvent(event, record.source, hookBodyEnv(body), hookBodyVersion(body), {
|
|
isReplay: true
|
|
})
|
|
}
|
|
}
|