fix(terminal-daemon): status says live, exited or unverifiable, and shows the process ceiling (#28239)

* fix(terminal-daemon): status reports live, exited or unverifiable, and the process ceiling

`orca terminal-daemon status` printed 'Terminal daemon: OK.' whenever no severed daemon was
recorded, including the ~60 s after a kill -9 before the next terminal respawned it. The host
now probes the daemon (health check, then its pid record with Linux start-time identity) and
returns an additive `daemon` report; the CLI prints the verdict and, for a live daemon, the
terminal process ceiling, which no CLI command showed under Electron serve.

Fixes BUG-5 and BUG-7 from the real-host Linux test run.

* fix(terminal-daemon): load the ceiling description from shared so the packaged CLI can resolve it

The packaged CLI cannot load out/main/daemon modules: electron-vite replaces out/main after the
CLI build. The ceiling state and its description move to src/shared, which the CLI already ships.

* fix(terminal-daemon): a pid the signal proved live is never exited because procfs hides it

* fix(terminal-daemon): load the liveness probe lazily so the RPC table stays light

Importing the probe eagerly pulled the daemon provider graph into every module that loads the
runtime RPC method table, which broke the cross-version terminal wire harness.

---------

Co-authored-by: m4air <m4air@Mac.localdomain>

src/cli/handlers/terminal-daemon.test.ts

@@ -1,14 +1,73 @@
 import { describe, expect, it } from 'vitest'
 import { formatSeveredDaemonStatus } from './terminal-daemon'
 
+const live = {
+  verdict: 'live' as const,
+  pid: 4288,
+  detail: 'answered its health check',
+  processCeiling: {
+    state: 'on' as const,
+    limit: 200,
+    mechanism: 'systemd-slice' as const,
+    unit: 'orca_ptys_1.slice'
+  }
+}
+
 describe('terminal-daemon status', () => {
-  it('says OK when nothing is severed', () => {
-    expect(formatSeveredDaemonStatus({ severed: null })).toBe('Terminal daemon: OK.')
+  it('reports a live daemon and its process ceiling', () => {
+    const text = formatSeveredDaemonStatus({ severed: null, daemon: live })
+    expect(text).toContain('Terminal daemon: live (pid 4288)')
+    expect(text).toContain('Terminal process ceiling: on (200 processes')
+    expect(text).not.toContain('OK')
+  })
+
+  it('reports a ceiling that is off and why', () => {
+    const text = formatSeveredDaemonStatus({
+      severed: null,
+      daemon: { ...live, processCeiling: { state: 'off', reason: 'no_user_manager' } }
+    })
+    expect(text).toContain('Terminal process ceiling: off (no user manager)')
+  })
+
+  it('never says OK for a daemon that is gone', () => {
+    const text = formatSeveredDaemonStatus({
+      severed: null,
+      daemon: {
+        verdict: 'exited',
+        pid: 4288,
+        detail: 'pid 4288 is gone; Orca starts a new daemon with the next terminal',
+        processCeiling: null
+      }
+    })
+    expect(text).toContain('Terminal daemon: exited')
+    expect(text).toContain('next terminal')
+    expect(text).not.toContain('OK')
+    expect(text).not.toContain('ceiling')
+  })
+
+  it('reports unverifiable when the host cannot prove either way', () => {
+    const text = formatSeveredDaemonStatus({
+      severed: null,
+      daemon: {
+        verdict: 'unverifiable',
+        pid: 4288,
+        detail: 'it did not answer',
+        processCeiling: null
+      }
+    })
+    expect(text).toContain('Terminal daemon: unverifiable (pid 4288) — it did not answer')
+  })
+
+  it('says liveness is unverifiable against a host that does not report it', () => {
+    const text = formatSeveredDaemonStatus({ severed: null })
+    expect(text).toContain('unverifiable')
+    expect(text).not.toContain('OK')
   })
 
   it('says a restart ends running processes and only resumable agents come back', () => {
     const text = formatSeveredDaemonStatus({
-      severed: { pid: 4288, linuxStartTicks: '777', bootId: 'b', liveSessionCount: 3 }
+      severed: { pid: 4288, linuxStartTicks: '777', bootId: 'b', liveSessionCount: 3 },
+      daemon: live
     })
     expect(text).toContain('PID 4288')
     expect(text).toContain('3 running terminals')
@@ -16,4 +75,13 @@ describe('terminal-daemon status', () => {
     expect(text).toContain('Agents that support resume can be resumed')
     expect(text).toContain('shells and other jobs cannot')
   })
+
+  it('does not describe an exited severed daemon as still hosting terminals', () => {
+    const text = formatSeveredDaemonStatus({
+      severed: { pid: 4288, linuxStartTicks: '777', bootId: 'b', liveSessionCount: 0 },
+      daemon: { verdict: 'exited', pid: 4288, detail: 'pid 4288 is gone', processCeiling: null }
+    })
+    expect(text).toContain('Terminal daemon: exited')
+    expect(text).not.toContain('still hosts')
+  })
 })

src/cli/handlers/terminal-daemon.ts

@@ -1,4 +1,8 @@
 import type { CommandHandler } from '../dispatch'
+import {
+  describePtyProcessCeiling,
+  type PtyProcessCeilingState
+} from '../../shared/pty-process-ceiling-state'
 import { printResult } from '../format'
 import { RuntimeClientError } from '../runtime-client'
 
@@ -9,7 +13,15 @@ type SeveredDaemonStatus = {
   liveSessionCount: number | null
 }
 
-type SeveredStatusResult = { severed: SeveredDaemonStatus | null }
+type DaemonLiveness = {
+  verdict: 'live' | 'unverifiable' | 'exited'
+  pid: number | null
+  detail: string
+  processCeiling: PtyProcessCeilingState | null
+}
+
+/** `daemon` is absent from hosts that predate the liveness report. */
+type SeveredStatusResult = { severed: SeveredDaemonStatus | null; daemon?: DaemonLiveness }
 
 type RestartResult =
   | { status: 'restarted'; killedCount: number }
@@ -27,16 +39,34 @@ function describeSessions(count: number | null): string {
     : `${count} running terminal${count === 1 ? '' : 's'}`
 }
 
+function formatDaemonLiveness(daemon: DaemonLiveness | undefined): string[] {
+  if (!daemon) {
+    return [
+      'Terminal daemon: unverifiable — this Orca server does not report whether it is running.'
+    ]
+  }
+  const lines = [
+    `Terminal daemon: ${daemon.verdict}${daemon.pid === null ? '' : ` (pid ${daemon.pid})`} — ${daemon.detail}.`
+  ]
+  if (daemon.verdict === 'live') {
+    lines.push(`Terminal process ceiling: ${describePtyProcessCeiling(daemon.processCeiling)}`)
+  }
+  return lines
+}
+
 export function formatSeveredDaemonStatus(result: SeveredStatusResult): string {
   const severed = result.severed
-  if (!severed) {
-    return 'Terminal daemon: OK.'
+  const liveness = formatDaemonLiveness(result.daemon)
+  // Why: a severed daemon that has since died hosts nothing; the liveness line says so.
+  if (!severed || result.daemon?.verdict === 'exited') {
+    return liveness.join('\n')
   }
   return [
     `Terminal daemon PID ${severed.pid} is running without its program files (the AppImage it started from is gone).`,
     `It still hosts ${describeSessions(severed.liveSessionCount)}. New terminals are not kept across restarts until it is restarted.`,
     'Restarting ENDS those processes. Agents that support resume can be resumed afterwards; shells and other jobs cannot.',
-    'Run `orca terminal-daemon restart --yes` when your agents are idle.'
+    'Run `orca terminal-daemon restart --yes` when your agents are idle.',
+    ...liveness
   ].join('\n')
 }
 

src/cli/specs/terminal-daemon.ts

@@ -4,10 +4,13 @@ import { GLOBAL_FLAGS } from '../args'
 export const TERMINAL_DAEMON_COMMAND_SPECS: CommandSpec[] = [
   {
     path: ['terminal-daemon', 'status'],
-    summary: 'Report whether the terminal daemon still has its program files',
+    summary:
+      'Report whether the terminal daemon is running, its process ceiling, and whether it still has its program files',
     usage: 'orca terminal-daemon status [--json]',
     allowedFlags: [...GLOBAL_FLAGS],
     notes: [
+      'The daemon is reported live, exited, or unverifiable. Orca starts a new daemon with the next terminal after one exits, so exited between a crash and that terminal is expected.',
+      'For a live daemon it also reports the terminal process ceiling: on with its limit, or off with the reason.',
       'On Linux AppImage, a daemon started by an earlier Orca can outlive the files it runs from. Orca keeps it so its terminals keep running, but new terminals are not kept across restarts until it is restarted.'
     ],
     examples: ['orca terminal-daemon status --json']

src/main/daemon/daemon-liveness-report.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from 'vitest'
+import type { ParsedDaemonPid } from './daemon-pid-file-parse'
+import {
+  inspectDaemonRecordProcess,
+  readTerminalDaemonLiveness,
+  type DaemonLivenessDependencies
+} from './daemon-liveness-report'
+
+const record: ParsedDaemonPid = {
+  pid: 4288,
+  startedAtMs: null,
+  entryPath: null,
+  appVersion: null,
+  launchNonce: null,
+  linuxStartTicks: '777',
+  bootId: 'b',
+  spawnerExecPath: null,
+  cgroupUnit: null,
+  processCeiling: { state: 'on', limit: 200, mechanism: 'systemd-slice', unit: 'orca.slice' }
+}
+
+function deps(overrides: Partial<DaemonLivenessDependencies>): DaemonLivenessDependencies {
+  return {
+    endpoint: () => ({ socketPath: '/run/d.sock', tokenPath: '/run/d.token' }),
+    checkHealth: async () => ({ verdict: 'healthy', coverage: 'pty-spawn' }),
+    readRecord: () => record,
+    inspectPid: async () => 'live',
+    confirmCeiling: (state) => state,
+    ...overrides
+  }
+}
+
+describe('terminal daemon liveness report', () => {
+  it('is live with the ceiling when the daemon answers', async () => {
+    await expect(readTerminalDaemonLiveness(deps({}))).resolves.toEqual({
+      verdict: 'live',
+      pid: 4288,
+      detail: 'it answered its health check',
+      processCeiling: record.processCeiling
+    })
+  })
+
+  it('is exited after a kill -9, before anything respawns it', async () => {
+    const report = await readTerminalDaemonLiveness(
+      deps({
+        checkHealth: async () => ({ verdict: 'unreachable', coverage: 'pty-spawn' }),
+        inspectPid: async () => 'exited'
+      })
+    )
+    expect(report.verdict).toBe('exited')
+    expect(report.processCeiling).toBeNull()
+  })
+
+  it('is unverifiable, never exited, when a silent pid still runs or cannot be checked', async () => {
+    for (const state of ['live', 'unverifiable'] as const) {
+      const report = await readTerminalDaemonLiveness(
+        deps({
+          checkHealth: async () => ({ verdict: 'unreachable', coverage: 'pty-spawn' }),
+          inspectPid: async () => state
+        })
+      )
+      expect(report.verdict).toBe('unverifiable')
+    }
+    const noRecord = await readTerminalDaemonLiveness(
+      deps({
+        checkHealth: async () => ({ verdict: 'unreachable', coverage: 'pty-spawn' }),
+        readRecord: () => null
+      })
+    )
+    expect(noRecord.verdict).toBe('unverifiable')
+  })
+
+  it('is unverifiable on a server with no daemon provider', async () => {
+    const report = await readTerminalDaemonLiveness(deps({ endpoint: () => null }))
+    expect(report.verdict).toBe('unverifiable')
+  })
+
+  it.runIf(process.platform === 'linux')(
+    'reads a pid reused by another process as exited',
+    async () => {
+      await expect(
+        inspectDaemonRecordProcess({ ...record, pid: process.pid, linuxStartTicks: '1' })
+      ).resolves.toBe('exited')
+    }
+  )
+
+  it('never reads a signal-live pid as exited because procfs hides its stat', async () => {
+    await expect(
+      inspectDaemonRecordProcess({ ...record, pid: process.pid }, 'linux', async () => ({
+        status: 'missing'
+      }))
+    ).resolves.toBe('unverifiable')
+  })
+
+  it('reads a pid with no process as exited', async () => {
+    await expect(inspectDaemonRecordProcess({ ...record, pid: 2 ** 22 + 12_345 })).resolves.toBe(
+      'exited'
+    )
+    await expect(
+      inspectDaemonRecordProcess({ ...record, pid: process.pid, linuxStartTicks: null })
+    ).resolves.toBe('live')
+  })
+})

src/main/daemon/daemon-liveness-report.ts

@@ -0,0 +1,112 @@
+/**
+ * What `orca terminal-daemon status` reports about the daemon this server uses. The daemon is
+ * respawned lazily, so between a crash and the next terminal there is none; that window must read
+ * as `exited`, not as healthy. Vocabulary per the SSH execution boundary: live / unverifiable /
+ * exited, and only positive proof earns `exited`.
+ */
+import { parseLinuxStartTicks } from '../agent-hooks/managed-hook-owner-identity'
+import { checkDaemonHealthWithCoverage, type DaemonHealthCheck } from './daemon-health'
+import { inspectProcessLiveness, readLinuxStat } from './daemon-process-inspection'
+import type { ParsedDaemonPid } from './daemon-pid-file-parse'
+import { getDaemonEndpointFacts, readDaemonPidRecord } from './daemon-provider-state'
+import {
+  confirmPtyProcessCeiling,
+  type PtyProcessCeilingState
+} from '../../shared/pty-process-ceiling-state'
+
+export type TerminalDaemonLivenessReport = {
+  verdict: 'live' | 'unverifiable' | 'exited'
+  pid: number | null
+  detail: string
+  /** Only for a live daemon: a dead one's record says nothing about the next one. */
+  processCeiling: PtyProcessCeilingState | null
+}
+
+export type DaemonLivenessDependencies = {
+  endpoint: () => { socketPath: string; tokenPath: string } | null
+  checkHealth: (socketPath: string, tokenPath: string) => Promise<DaemonHealthCheck>
+  readRecord: () => ParsedDaemonPid | null
+  inspectPid: (record: ParsedDaemonPid) => Promise<'live' | 'exited' | 'unverifiable'>
+  confirmCeiling: (state: PtyProcessCeilingState | null) => PtyProcessCeilingState | null
+}
+
+const LIVE_DETAIL: Record<DaemonHealthCheck['verdict'], string> = {
+  healthy: 'it answered its health check',
+  'pty-spawn-unhealthy': 'it answered, but could not start a test terminal',
+  rejected: "it answered, but refused this server's handshake",
+  unreachable: ''
+}
+
+export async function readTerminalDaemonLiveness(
+  deps: DaemonLivenessDependencies = defaultDependencies
+): Promise<TerminalDaemonLivenessReport> {
+  const endpoint = deps.endpoint()
+  if (!endpoint) {
+    return {
+      verdict: 'unverifiable',
+      pid: null,
+      detail: 'this server has no terminal daemon provider',
+      processCeiling: null
+    }
+  }
+  const health = await deps.checkHealth(endpoint.socketPath, endpoint.tokenPath)
+  const record = deps.readRecord()
+  const pid = record?.pid ?? null
+  if (health.verdict !== 'unreachable') {
+    return {
+      verdict: 'live',
+      pid,
+      detail: LIVE_DETAIL[health.verdict],
+      processCeiling: deps.confirmCeiling(record?.processCeiling ?? null)
+    }
+  }
+  if (!record) {
+    return {
+      verdict: 'unverifiable',
+      pid: null,
+      detail: 'it did not answer and left no process record',
+      processCeiling: null
+    }
+  }
+  const processState = await deps.inspectPid(record)
+  return processState === 'exited'
+    ? {
+        verdict: 'exited',
+        pid,
+        detail: `pid ${record.pid} is gone; Orca starts a new daemon with the next terminal`,
+        processCeiling: null
+      }
+    : {
+        verdict: 'unverifiable',
+        pid,
+        detail: `pid ${record.pid} ${processState === 'live' ? 'is running but did not answer' : 'could not be checked'}`,
+        processCeiling: null
+      }
+}
+
+/** A reused pid is a different process: on Linux the recorded start time tells them apart. */
+export async function inspectDaemonRecordProcess(
+  record: ParsedDaemonPid,
+  platform: NodeJS.Platform = process.platform,
+  readStat: typeof readLinuxStat = readLinuxStat
+): Promise<'live' | 'exited' | 'unverifiable'> {
+  const signal = inspectProcessLiveness(record.pid).status
+  if (signal !== 'live' || platform !== 'linux' || !record.linuxStartTicks) {
+    return signal
+  }
+  const stat = await readStat(record.pid)
+  // Why not exited on a missing stat: the signal just proved the pid live; procfs may hide it.
+  if (stat.status !== 'present') {
+    return stat.status === 'missing' ? 'unverifiable' : 'live'
+  }
+  const startTicks = parseLinuxStartTicks(stat.value)
+  return startTicks !== null && startTicks !== record.linuxStartTicks ? 'exited' : 'live'
+}
+
+const defaultDependencies: DaemonLivenessDependencies = {
+  endpoint: getDaemonEndpointFacts,
+  checkHealth: checkDaemonHealthWithCoverage,
+  readRecord: readDaemonPidRecord,
+  inspectPid: (record) => inspectDaemonRecordProcess(record),
+  confirmCeiling: (state) => confirmPtyProcessCeiling(state)
+}

src/main/daemon/daemon-pid-file-parse.ts

@@ -1,7 +1,7 @@
 import {
   parsePtyProcessCeilingState,
   type PtyProcessCeilingState
-} from './pty-process-ceiling-state'
+} from '../../shared/pty-process-ceiling-state'
 export type ParsedDaemonPid = {
   pid: number
   startedAtMs: number | null

src/main/daemon/daemon-spawner.ts

@@ -1,5 +1,5 @@
 import { createHash, randomUUID } from 'node:crypto'
-import type { PtyProcessCeilingState } from './pty-process-ceiling-state'
+import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state'
 import {
   constants,
   copyFileSync,

src/main/daemon/pty-process-ceiling-runtime.ts

@@ -4,7 +4,7 @@ import { runProcessSync } from '@orca/process-host'
 import { userManagerEnv } from './daemon-cgroup-scope'
 import { setPtyActiveProcessLimit } from '../windows/windows-pty-job'
 import { resolvePtyProcessCeiling, type LinuxCeilingRuntime } from './pty-process-ceiling'
-import type { PtyProcessCeilingState } from './pty-process-ceiling-state'
+import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state'
 
 let currentState: PtyProcessCeilingState = { state: 'off', reason: 'disabled' }
 let currentLinux: LinuxCeilingRuntime | null = null

src/main/daemon/pty-process-ceiling.test.ts

@@ -14,7 +14,7 @@ import {
   confirmPtyProcessCeiling,
   describePtyProcessCeiling,
   parsePtyProcessCeilingState
-} from './pty-process-ceiling-state'
+} from '../../shared/pty-process-ceiling-state'
 import { parseDaemonPidFile } from './daemon-pid-file-parse'
 
 const on = { [PTY_PROCESS_CEILING_ENV]: '2048' }

src/main/daemon/pty-process-ceiling.ts

@@ -12,9 +12,9 @@ import { accessSync, constants, readFileSync } from 'node:fs'
 import { delimiter, isAbsolute, join } from 'node:path'
 import { runProcessSync } from '@orca/process-host'
 import { isDurableDaemonScopeSupported, userManagerEnv } from './daemon-cgroup-scope'
-import type { PtyProcessCeilingState } from './pty-process-ceiling-state'
+import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state'
 
-export type { PtyProcessCeilingState } from './pty-process-ceiling-state'
+export type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state'
 
 export const PTY_PROCESS_CEILING_ENV = 'ORCA_TERMINAL_PROCESS_CEILING'
 /** Below this a shell, its prompt hooks and one build step already collide. */

src/main/orcad/orcad-health.ts

@@ -12,7 +12,7 @@ import { hashOrcadLauncher } from '../../shared/orcad-build-identity'
 import {
   confirmPtyProcessCeiling,
   type PtyProcessCeilingState
-} from '../daemon/pty-process-ceiling-state'
+} from '../../shared/pty-process-ceiling-state'
 import process from 'node:process'
 import { checkDaemonHealthWithCoverage, type DaemonHealth } from '../daemon/daemon-health'
 import { ptySpawnHealthPlatformCoverage } from '../daemon/daemon-health-identity'

src/main/runtime/rpc/methods/terminal-daemon.ts

@@ -10,7 +10,14 @@ export const TERMINAL_DAEMON_METHODS = [
     name: 'terminalDaemon.severedStatus',
     permission: 'workspace',
     params: null,
-    handler: async () => ({ severed: await readSeveredDaemonStatus() })
+    // Why `daemon` is additive: older CLIs read only `severed`.
+    handler: async () => ({
+      severed: await readSeveredDaemonStatus(),
+      // Why lazy: the probe pulls in the daemon provider graph, which the RPC table must not load.
+      daemon: await (
+        await import('../../../daemon/daemon-liveness-report')
+      ).readTerminalDaemonLiveness()
+    })
   }),
   defineMethod({
     name: 'terminalDaemon.restartSevered',

src/main/server/serve-readiness.ts

@@ -1,6 +1,6 @@
 import type { PairingOfferUnavailableReason } from '../runtime/runtime-rpc'
 import type { OrcadHealth } from '../orcad/orcad-health'
-import { describePtyProcessCeiling } from '../daemon/pty-process-ceiling-state'
+import { describePtyProcessCeiling } from '../../shared/pty-process-ceiling-state'
 
 export type ServePairingUnavailableReason = PairingOfferUnavailableReason | 'disabled_by_operator'