From 5e094889116de9a695009bcbd2cd348b9191dcfb Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Tue, 13 Oct 2026 05:21:49 -0700 Subject: [PATCH] fix(terminal-daemon): status says live, exited or unverifiable, and shows the process ceiling (#28239) * fix(terminal-daemon): status reports live, exited or unverifiable, and the process ceiling `orca terminal-daemon status` printed 'Terminal daemon: OK.' whenever no severed daemon was recorded, including the ~60 s after a kill -9 before the next terminal respawned it. The host now probes the daemon (health check, then its pid record with Linux start-time identity) and returns an additive `daemon` report; the CLI prints the verdict and, for a live daemon, the terminal process ceiling, which no CLI command showed under Electron serve. Fixes BUG-5 and BUG-7 from the real-host Linux test run. * fix(terminal-daemon): load the ceiling description from shared so the packaged CLI can resolve it The packaged CLI cannot load out/main/daemon modules: electron-vite replaces out/main after the CLI build. The ceiling state and its description move to src/shared, which the CLI already ships. * fix(terminal-daemon): a pid the signal proved live is never exited because procfs hides it * fix(terminal-daemon): load the liveness probe lazily so the RPC table stays light Importing the probe eagerly pulled the daemon provider graph into every module that loads the runtime RPC method table, which broke the cross-version terminal wire harness. --------- Co-authored-by: m4air --- src/cli/handlers/terminal-daemon.test.ts | 74 +++++++++++- src/cli/handlers/terminal-daemon.ts | 38 +++++- src/cli/specs/terminal-daemon.ts | 5 +- .../daemon/daemon-liveness-report.test.ts | 103 ++++++++++++++++ src/main/daemon/daemon-liveness-report.ts | 112 ++++++++++++++++++ src/main/daemon/daemon-pid-file-parse.ts | 2 +- src/main/daemon/daemon-spawner.ts | 2 +- .../daemon/pty-process-ceiling-runtime.ts | 2 +- src/main/daemon/pty-process-ceiling.test.ts | 2 +- src/main/daemon/pty-process-ceiling.ts | 4 +- src/main/orcad/orcad-health.ts | 2 +- .../runtime/rpc/methods/terminal-daemon.ts | 9 +- src/main/server/serve-readiness.ts | 2 +- .../pty-process-ceiling-state.ts | 0 14 files changed, 340 insertions(+), 17 deletions(-) create mode 100644 src/main/daemon/daemon-liveness-report.test.ts create mode 100644 src/main/daemon/daemon-liveness-report.ts rename src/{main/daemon => shared}/pty-process-ceiling-state.ts (100%) diff --git a/src/cli/handlers/terminal-daemon.test.ts b/src/cli/handlers/terminal-daemon.test.ts index 4d8961a9ed4..a4d27c080df 100644 --- a/src/cli/handlers/terminal-daemon.test.ts +++ b/src/cli/handlers/terminal-daemon.test.ts @@ -1,14 +1,73 @@ import { describe, expect, it } from 'vitest' import { formatSeveredDaemonStatus } from './terminal-daemon' +const live = { + verdict: 'live' as const, + pid: 4288, + detail: 'answered its health check', + processCeiling: { + state: 'on' as const, + limit: 200, + mechanism: 'systemd-slice' as const, + unit: 'orca_ptys_1.slice' + } +} + describe('terminal-daemon status', () => { - it('says OK when nothing is severed', () => { - expect(formatSeveredDaemonStatus({ severed: null })).toBe('Terminal daemon: OK.') + it('reports a live daemon and its process ceiling', () => { + const text = formatSeveredDaemonStatus({ severed: null, daemon: live }) + expect(text).toContain('Terminal daemon: live (pid 4288)') + expect(text).toContain('Terminal process ceiling: on (200 processes') + expect(text).not.toContain('OK') + }) + + it('reports a ceiling that is off and why', () => { + const text = formatSeveredDaemonStatus({ + severed: null, + daemon: { ...live, processCeiling: { state: 'off', reason: 'no_user_manager' } } + }) + expect(text).toContain('Terminal process ceiling: off (no user manager)') + }) + + it('never says OK for a daemon that is gone', () => { + const text = formatSeveredDaemonStatus({ + severed: null, + daemon: { + verdict: 'exited', + pid: 4288, + detail: 'pid 4288 is gone; Orca starts a new daemon with the next terminal', + processCeiling: null + } + }) + expect(text).toContain('Terminal daemon: exited') + expect(text).toContain('next terminal') + expect(text).not.toContain('OK') + expect(text).not.toContain('ceiling') + }) + + it('reports unverifiable when the host cannot prove either way', () => { + const text = formatSeveredDaemonStatus({ + severed: null, + daemon: { + verdict: 'unverifiable', + pid: 4288, + detail: 'it did not answer', + processCeiling: null + } + }) + expect(text).toContain('Terminal daemon: unverifiable (pid 4288) — it did not answer') + }) + + it('says liveness is unverifiable against a host that does not report it', () => { + const text = formatSeveredDaemonStatus({ severed: null }) + expect(text).toContain('unverifiable') + expect(text).not.toContain('OK') }) it('says a restart ends running processes and only resumable agents come back', () => { const text = formatSeveredDaemonStatus({ - severed: { pid: 4288, linuxStartTicks: '777', bootId: 'b', liveSessionCount: 3 } + severed: { pid: 4288, linuxStartTicks: '777', bootId: 'b', liveSessionCount: 3 }, + daemon: live }) expect(text).toContain('PID 4288') expect(text).toContain('3 running terminals') @@ -16,4 +75,13 @@ describe('terminal-daemon status', () => { expect(text).toContain('Agents that support resume can be resumed') expect(text).toContain('shells and other jobs cannot') }) + + it('does not describe an exited severed daemon as still hosting terminals', () => { + const text = formatSeveredDaemonStatus({ + severed: { pid: 4288, linuxStartTicks: '777', bootId: 'b', liveSessionCount: 0 }, + daemon: { verdict: 'exited', pid: 4288, detail: 'pid 4288 is gone', processCeiling: null } + }) + expect(text).toContain('Terminal daemon: exited') + expect(text).not.toContain('still hosts') + }) }) diff --git a/src/cli/handlers/terminal-daemon.ts b/src/cli/handlers/terminal-daemon.ts index 99a277eab97..e01f0a1d053 100644 --- a/src/cli/handlers/terminal-daemon.ts +++ b/src/cli/handlers/terminal-daemon.ts @@ -1,4 +1,8 @@ import type { CommandHandler } from '../dispatch' +import { + describePtyProcessCeiling, + type PtyProcessCeilingState +} from '../../shared/pty-process-ceiling-state' import { printResult } from '../format' import { RuntimeClientError } from '../runtime-client' @@ -9,7 +13,15 @@ type SeveredDaemonStatus = { liveSessionCount: number | null } -type SeveredStatusResult = { severed: SeveredDaemonStatus | null } +type DaemonLiveness = { + verdict: 'live' | 'unverifiable' | 'exited' + pid: number | null + detail: string + processCeiling: PtyProcessCeilingState | null +} + +/** `daemon` is absent from hosts that predate the liveness report. */ +type SeveredStatusResult = { severed: SeveredDaemonStatus | null; daemon?: DaemonLiveness } type RestartResult = | { status: 'restarted'; killedCount: number } @@ -27,16 +39,34 @@ function describeSessions(count: number | null): string { : `${count} running terminal${count === 1 ? '' : 's'}` } +function formatDaemonLiveness(daemon: DaemonLiveness | undefined): string[] { + if (!daemon) { + return [ + 'Terminal daemon: unverifiable — this Orca server does not report whether it is running.' + ] + } + const lines = [ + `Terminal daemon: ${daemon.verdict}${daemon.pid === null ? '' : ` (pid ${daemon.pid})`} — ${daemon.detail}.` + ] + if (daemon.verdict === 'live') { + lines.push(`Terminal process ceiling: ${describePtyProcessCeiling(daemon.processCeiling)}`) + } + return lines +} + export function formatSeveredDaemonStatus(result: SeveredStatusResult): string { const severed = result.severed - if (!severed) { - return 'Terminal daemon: OK.' + const liveness = formatDaemonLiveness(result.daemon) + // Why: a severed daemon that has since died hosts nothing; the liveness line says so. + if (!severed || result.daemon?.verdict === 'exited') { + return liveness.join('\n') } return [ `Terminal daemon PID ${severed.pid} is running without its program files (the AppImage it started from is gone).`, `It still hosts ${describeSessions(severed.liveSessionCount)}. New terminals are not kept across restarts until it is restarted.`, 'Restarting ENDS those processes. Agents that support resume can be resumed afterwards; shells and other jobs cannot.', - 'Run `orca terminal-daemon restart --yes` when your agents are idle.' + 'Run `orca terminal-daemon restart --yes` when your agents are idle.', + ...liveness ].join('\n') } diff --git a/src/cli/specs/terminal-daemon.ts b/src/cli/specs/terminal-daemon.ts index 0ab468ecbba..cea526b9d87 100644 --- a/src/cli/specs/terminal-daemon.ts +++ b/src/cli/specs/terminal-daemon.ts @@ -4,10 +4,13 @@ import { GLOBAL_FLAGS } from '../args' export const TERMINAL_DAEMON_COMMAND_SPECS: CommandSpec[] = [ { path: ['terminal-daemon', 'status'], - summary: 'Report whether the terminal daemon still has its program files', + summary: + 'Report whether the terminal daemon is running, its process ceiling, and whether it still has its program files', usage: 'orca terminal-daemon status [--json]', allowedFlags: [...GLOBAL_FLAGS], notes: [ + 'The daemon is reported live, exited, or unverifiable. Orca starts a new daemon with the next terminal after one exits, so exited between a crash and that terminal is expected.', + 'For a live daemon it also reports the terminal process ceiling: on with its limit, or off with the reason.', 'On Linux AppImage, a daemon started by an earlier Orca can outlive the files it runs from. Orca keeps it so its terminals keep running, but new terminals are not kept across restarts until it is restarted.' ], examples: ['orca terminal-daemon status --json'] diff --git a/src/main/daemon/daemon-liveness-report.test.ts b/src/main/daemon/daemon-liveness-report.test.ts new file mode 100644 index 00000000000..f416f77be01 --- /dev/null +++ b/src/main/daemon/daemon-liveness-report.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest' +import type { ParsedDaemonPid } from './daemon-pid-file-parse' +import { + inspectDaemonRecordProcess, + readTerminalDaemonLiveness, + type DaemonLivenessDependencies +} from './daemon-liveness-report' + +const record: ParsedDaemonPid = { + pid: 4288, + startedAtMs: null, + entryPath: null, + appVersion: null, + launchNonce: null, + linuxStartTicks: '777', + bootId: 'b', + spawnerExecPath: null, + cgroupUnit: null, + processCeiling: { state: 'on', limit: 200, mechanism: 'systemd-slice', unit: 'orca.slice' } +} + +function deps(overrides: Partial): DaemonLivenessDependencies { + return { + endpoint: () => ({ socketPath: '/run/d.sock', tokenPath: '/run/d.token' }), + checkHealth: async () => ({ verdict: 'healthy', coverage: 'pty-spawn' }), + readRecord: () => record, + inspectPid: async () => 'live', + confirmCeiling: (state) => state, + ...overrides + } +} + +describe('terminal daemon liveness report', () => { + it('is live with the ceiling when the daemon answers', async () => { + await expect(readTerminalDaemonLiveness(deps({}))).resolves.toEqual({ + verdict: 'live', + pid: 4288, + detail: 'it answered its health check', + processCeiling: record.processCeiling + }) + }) + + it('is exited after a kill -9, before anything respawns it', async () => { + const report = await readTerminalDaemonLiveness( + deps({ + checkHealth: async () => ({ verdict: 'unreachable', coverage: 'pty-spawn' }), + inspectPid: async () => 'exited' + }) + ) + expect(report.verdict).toBe('exited') + expect(report.processCeiling).toBeNull() + }) + + it('is unverifiable, never exited, when a silent pid still runs or cannot be checked', async () => { + for (const state of ['live', 'unverifiable'] as const) { + const report = await readTerminalDaemonLiveness( + deps({ + checkHealth: async () => ({ verdict: 'unreachable', coverage: 'pty-spawn' }), + inspectPid: async () => state + }) + ) + expect(report.verdict).toBe('unverifiable') + } + const noRecord = await readTerminalDaemonLiveness( + deps({ + checkHealth: async () => ({ verdict: 'unreachable', coverage: 'pty-spawn' }), + readRecord: () => null + }) + ) + expect(noRecord.verdict).toBe('unverifiable') + }) + + it('is unverifiable on a server with no daemon provider', async () => { + const report = await readTerminalDaemonLiveness(deps({ endpoint: () => null })) + expect(report.verdict).toBe('unverifiable') + }) + + it.runIf(process.platform === 'linux')( + 'reads a pid reused by another process as exited', + async () => { + await expect( + inspectDaemonRecordProcess({ ...record, pid: process.pid, linuxStartTicks: '1' }) + ).resolves.toBe('exited') + } + ) + + it('never reads a signal-live pid as exited because procfs hides its stat', async () => { + await expect( + inspectDaemonRecordProcess({ ...record, pid: process.pid }, 'linux', async () => ({ + status: 'missing' + })) + ).resolves.toBe('unverifiable') + }) + + it('reads a pid with no process as exited', async () => { + await expect(inspectDaemonRecordProcess({ ...record, pid: 2 ** 22 + 12_345 })).resolves.toBe( + 'exited' + ) + await expect( + inspectDaemonRecordProcess({ ...record, pid: process.pid, linuxStartTicks: null }) + ).resolves.toBe('live') + }) +}) diff --git a/src/main/daemon/daemon-liveness-report.ts b/src/main/daemon/daemon-liveness-report.ts new file mode 100644 index 00000000000..48c8a0dd632 --- /dev/null +++ b/src/main/daemon/daemon-liveness-report.ts @@ -0,0 +1,112 @@ +/** + * What `orca terminal-daemon status` reports about the daemon this server uses. The daemon is + * respawned lazily, so between a crash and the next terminal there is none; that window must read + * as `exited`, not as healthy. Vocabulary per the SSH execution boundary: live / unverifiable / + * exited, and only positive proof earns `exited`. + */ +import { parseLinuxStartTicks } from '../agent-hooks/managed-hook-owner-identity' +import { checkDaemonHealthWithCoverage, type DaemonHealthCheck } from './daemon-health' +import { inspectProcessLiveness, readLinuxStat } from './daemon-process-inspection' +import type { ParsedDaemonPid } from './daemon-pid-file-parse' +import { getDaemonEndpointFacts, readDaemonPidRecord } from './daemon-provider-state' +import { + confirmPtyProcessCeiling, + type PtyProcessCeilingState +} from '../../shared/pty-process-ceiling-state' + +export type TerminalDaemonLivenessReport = { + verdict: 'live' | 'unverifiable' | 'exited' + pid: number | null + detail: string + /** Only for a live daemon: a dead one's record says nothing about the next one. */ + processCeiling: PtyProcessCeilingState | null +} + +export type DaemonLivenessDependencies = { + endpoint: () => { socketPath: string; tokenPath: string } | null + checkHealth: (socketPath: string, tokenPath: string) => Promise + readRecord: () => ParsedDaemonPid | null + inspectPid: (record: ParsedDaemonPid) => Promise<'live' | 'exited' | 'unverifiable'> + confirmCeiling: (state: PtyProcessCeilingState | null) => PtyProcessCeilingState | null +} + +const LIVE_DETAIL: Record = { + healthy: 'it answered its health check', + 'pty-spawn-unhealthy': 'it answered, but could not start a test terminal', + rejected: "it answered, but refused this server's handshake", + unreachable: '' +} + +export async function readTerminalDaemonLiveness( + deps: DaemonLivenessDependencies = defaultDependencies +): Promise { + const endpoint = deps.endpoint() + if (!endpoint) { + return { + verdict: 'unverifiable', + pid: null, + detail: 'this server has no terminal daemon provider', + processCeiling: null + } + } + const health = await deps.checkHealth(endpoint.socketPath, endpoint.tokenPath) + const record = deps.readRecord() + const pid = record?.pid ?? null + if (health.verdict !== 'unreachable') { + return { + verdict: 'live', + pid, + detail: LIVE_DETAIL[health.verdict], + processCeiling: deps.confirmCeiling(record?.processCeiling ?? null) + } + } + if (!record) { + return { + verdict: 'unverifiable', + pid: null, + detail: 'it did not answer and left no process record', + processCeiling: null + } + } + const processState = await deps.inspectPid(record) + return processState === 'exited' + ? { + verdict: 'exited', + pid, + detail: `pid ${record.pid} is gone; Orca starts a new daemon with the next terminal`, + processCeiling: null + } + : { + verdict: 'unverifiable', + pid, + detail: `pid ${record.pid} ${processState === 'live' ? 'is running but did not answer' : 'could not be checked'}`, + processCeiling: null + } +} + +/** A reused pid is a different process: on Linux the recorded start time tells them apart. */ +export async function inspectDaemonRecordProcess( + record: ParsedDaemonPid, + platform: NodeJS.Platform = process.platform, + readStat: typeof readLinuxStat = readLinuxStat +): Promise<'live' | 'exited' | 'unverifiable'> { + const signal = inspectProcessLiveness(record.pid).status + if (signal !== 'live' || platform !== 'linux' || !record.linuxStartTicks) { + return signal + } + const stat = await readStat(record.pid) + // Why not exited on a missing stat: the signal just proved the pid live; procfs may hide it. + if (stat.status !== 'present') { + return stat.status === 'missing' ? 'unverifiable' : 'live' + } + const startTicks = parseLinuxStartTicks(stat.value) + return startTicks !== null && startTicks !== record.linuxStartTicks ? 'exited' : 'live' +} + +const defaultDependencies: DaemonLivenessDependencies = { + endpoint: getDaemonEndpointFacts, + checkHealth: checkDaemonHealthWithCoverage, + readRecord: readDaemonPidRecord, + inspectPid: (record) => inspectDaemonRecordProcess(record), + confirmCeiling: (state) => confirmPtyProcessCeiling(state) +} diff --git a/src/main/daemon/daemon-pid-file-parse.ts b/src/main/daemon/daemon-pid-file-parse.ts index 3d05a283e7b..b7d4eac229b 100644 --- a/src/main/daemon/daemon-pid-file-parse.ts +++ b/src/main/daemon/daemon-pid-file-parse.ts @@ -1,7 +1,7 @@ import { parsePtyProcessCeilingState, type PtyProcessCeilingState -} from './pty-process-ceiling-state' +} from '../../shared/pty-process-ceiling-state' export type ParsedDaemonPid = { pid: number startedAtMs: number | null diff --git a/src/main/daemon/daemon-spawner.ts b/src/main/daemon/daemon-spawner.ts index fe96bb56e65..a17aeae0496 100644 --- a/src/main/daemon/daemon-spawner.ts +++ b/src/main/daemon/daemon-spawner.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from 'node:crypto' -import type { PtyProcessCeilingState } from './pty-process-ceiling-state' +import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' import { constants, copyFileSync, diff --git a/src/main/daemon/pty-process-ceiling-runtime.ts b/src/main/daemon/pty-process-ceiling-runtime.ts index aac0c2a2601..8eb53c0c840 100644 --- a/src/main/daemon/pty-process-ceiling-runtime.ts +++ b/src/main/daemon/pty-process-ceiling-runtime.ts @@ -4,7 +4,7 @@ import { runProcessSync } from '@orca/process-host' import { userManagerEnv } from './daemon-cgroup-scope' import { setPtyActiveProcessLimit } from '../windows/windows-pty-job' import { resolvePtyProcessCeiling, type LinuxCeilingRuntime } from './pty-process-ceiling' -import type { PtyProcessCeilingState } from './pty-process-ceiling-state' +import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' let currentState: PtyProcessCeilingState = { state: 'off', reason: 'disabled' } let currentLinux: LinuxCeilingRuntime | null = null diff --git a/src/main/daemon/pty-process-ceiling.test.ts b/src/main/daemon/pty-process-ceiling.test.ts index d995995694a..7040b3d8cb7 100644 --- a/src/main/daemon/pty-process-ceiling.test.ts +++ b/src/main/daemon/pty-process-ceiling.test.ts @@ -14,7 +14,7 @@ import { confirmPtyProcessCeiling, describePtyProcessCeiling, parsePtyProcessCeilingState -} from './pty-process-ceiling-state' +} from '../../shared/pty-process-ceiling-state' import { parseDaemonPidFile } from './daemon-pid-file-parse' const on = { [PTY_PROCESS_CEILING_ENV]: '2048' } diff --git a/src/main/daemon/pty-process-ceiling.ts b/src/main/daemon/pty-process-ceiling.ts index d3cbc4e581e..73d32953b52 100644 --- a/src/main/daemon/pty-process-ceiling.ts +++ b/src/main/daemon/pty-process-ceiling.ts @@ -12,9 +12,9 @@ import { accessSync, constants, readFileSync } from 'node:fs' import { delimiter, isAbsolute, join } from 'node:path' import { runProcessSync } from '@orca/process-host' import { isDurableDaemonScopeSupported, userManagerEnv } from './daemon-cgroup-scope' -import type { PtyProcessCeilingState } from './pty-process-ceiling-state' +import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' -export type { PtyProcessCeilingState } from './pty-process-ceiling-state' +export type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' export const PTY_PROCESS_CEILING_ENV = 'ORCA_TERMINAL_PROCESS_CEILING' /** Below this a shell, its prompt hooks and one build step already collide. */ diff --git a/src/main/orcad/orcad-health.ts b/src/main/orcad/orcad-health.ts index 4e8d869922f..c604ebeb6c4 100644 --- a/src/main/orcad/orcad-health.ts +++ b/src/main/orcad/orcad-health.ts @@ -12,7 +12,7 @@ import { hashOrcadLauncher } from '../../shared/orcad-build-identity' import { confirmPtyProcessCeiling, type PtyProcessCeilingState -} from '../daemon/pty-process-ceiling-state' +} from '../../shared/pty-process-ceiling-state' import process from 'node:process' import { checkDaemonHealthWithCoverage, type DaemonHealth } from '../daemon/daemon-health' import { ptySpawnHealthPlatformCoverage } from '../daemon/daemon-health-identity' diff --git a/src/main/runtime/rpc/methods/terminal-daemon.ts b/src/main/runtime/rpc/methods/terminal-daemon.ts index 98cf2777419..31f5a3fd3a8 100644 --- a/src/main/runtime/rpc/methods/terminal-daemon.ts +++ b/src/main/runtime/rpc/methods/terminal-daemon.ts @@ -10,7 +10,14 @@ export const TERMINAL_DAEMON_METHODS = [ name: 'terminalDaemon.severedStatus', permission: 'workspace', params: null, - handler: async () => ({ severed: await readSeveredDaemonStatus() }) + // Why `daemon` is additive: older CLIs read only `severed`. + handler: async () => ({ + severed: await readSeveredDaemonStatus(), + // Why lazy: the probe pulls in the daemon provider graph, which the RPC table must not load. + daemon: await ( + await import('../../../daemon/daemon-liveness-report') + ).readTerminalDaemonLiveness() + }) }), defineMethod({ name: 'terminalDaemon.restartSevered', diff --git a/src/main/server/serve-readiness.ts b/src/main/server/serve-readiness.ts index a40b2b91ae6..b0d1e82a59f 100644 --- a/src/main/server/serve-readiness.ts +++ b/src/main/server/serve-readiness.ts @@ -1,6 +1,6 @@ import type { PairingOfferUnavailableReason } from '../runtime/runtime-rpc' import type { OrcadHealth } from '../orcad/orcad-health' -import { describePtyProcessCeiling } from '../daemon/pty-process-ceiling-state' +import { describePtyProcessCeiling } from '../../shared/pty-process-ceiling-state' export type ServePairingUnavailableReason = PairingOfferUnavailableReason | 'disabled_by_operator' diff --git a/src/main/daemon/pty-process-ceiling-state.ts b/src/shared/pty-process-ceiling-state.ts similarity index 100% rename from src/main/daemon/pty-process-ceiling-state.ts rename to src/shared/pty-process-ceiling-state.ts