feat(daemon): pace automatic terminal restores on the desktop too (#28202)

* feat(daemon): pace automatic terminal restores on the desktop too

The desktop daemon now enforces the restore budget that orca serve and orcad
already enforce: the visible pane first, at most 3 automatic restores at once
(1/s), at most 8 live per machine, and the rest stay sleeping until opened.
ORCA_TERMINAL_SPAWN_ADMISSION=observe rolls newly launched daemons back to
count-only.

* fix(daemon): never put a user's own resume back to sleep; keep every refused old-client restore

A user-resume past the per-client allowance is still paced as automatic work
but is no longer deferred for a pause, the live cap or its deadline, so a visible
pane is never put back to sleep in front of the user. The legacy restore record
holds as many refusals as the host could run.

---------

Co-authored-by: m4air <m4air@Mac.localdomain>

src/main/daemon/desktop-restore-admission.test.ts

@@ -0,0 +1,213 @@
+import './mock-descendant-sweep'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir, uptime } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { SpawnIntent } from '../../shared/host-spawn-admission'
+import { DaemonAutomaticWorkGuard } from './daemon-automatic-work-guard'
+import { DaemonProcessGrowthMonitor } from './daemon-process-growth-monitor'
+import type { SubprocessHandle } from './session-subprocess-handle'
+import { TerminalHost, type TerminalHostOptions } from './terminal-host'
+import {
+  resolveTerminalSpawnAdmissionMode,
+  TERMINAL_SPAWN_ADMISSION_ENV_VAR
+} from './terminal-spawn-gate'
+
+type SpawnSubprocess = TerminalHostOptions['spawnSubprocess']
+
+// The desktop app's daemon after a restart: its renderer declares the visible pane
+// `user-resume` and every hidden agent pane `automatic-restore`, with no session claim.
+describe('desktop restore admission', () => {
+  let dir: string
+  let host: TerminalHost
+  let guard: DaemonAutomaticWorkGuard
+  let spawned: string[]
+  let exits: Map<number, (code: number) => void>
+  let nextPid: number
+
+  beforeEach(() => {
+    vi.useFakeTimers({ now: 1_000_000 })
+    dir = mkdtempSync(join(tmpdir(), 'orca-desktop-restore-'))
+    spawned = []
+    exits = new Map()
+    nextPid = 2000
+  })
+
+  afterEach(async () => {
+    await host?.dispose()
+    guard?.stop()
+    vi.useRealTimers()
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  function startDesktopDaemon(): void {
+    const markerPath = join(dir, 'daemon-v1.automatic-work-run.json')
+    guard = new DaemonAutomaticWorkGuard({
+      markerPath,
+      // What the desktop launcher passes with no rollback switch set.
+      mode: resolveTerminalSpawnAdmissionMode({}),
+      log: { log: vi.fn() },
+      listSessions: () => [],
+      ownsEndpoint: () => true,
+      monitor: new DaemonProcessGrowthMonitor({
+        listSessions: () => [],
+        recentAutomaticWork: () => false,
+        sampleWhenIdle: false,
+        onLevelChanged: () => {},
+        onTrip: () => {},
+        readRows: async () => []
+      })
+    })
+    guard.start()
+    const spawnSubprocess = vi.fn<SpawnSubprocess>((opts) => {
+      const pid = nextPid++
+      spawned.push(opts.sessionId)
+      const handle: SubprocessHandle = {
+        pid,
+        getForegroundProcess: vi.fn(() => null),
+        write: vi.fn(),
+        resize: vi.fn(),
+        kill: vi.fn(() => exits.get(pid)?.(0)),
+        terminateOwnedTree: () => 'unavailable' as const,
+        forceKill: vi.fn(() => exits.get(pid)?.(137)),
+        signal: vi.fn(),
+        onData: vi.fn(),
+        onExit: (callback) => {
+          exits.set(pid, callback)
+        },
+        dispose: vi.fn()
+      }
+      return handle
+    })
+    host = new TerminalHost({ spawnSubprocess, spawnGate: guard.gate })
+  }
+
+  function restore(sessionId: string, declaredIntent: SpawnIntent) {
+    return host.createOrAttach({
+      sessionId,
+      cols: 80,
+      rows: 24,
+      historySeedChunks: ['seed'],
+      spawnAdmission: {
+        clientId: 'desktop-main',
+        declaredIntent,
+        // The daemon's cap for a desktop create (30 s request timeout, halved, capped at 12 s).
+        deadlineAt: Date.now() + 12_000
+      },
+      streamClient: { onData: vi.fn(), onExit: vi.fn() }
+    })
+  }
+
+  function shell(sessionId: string) {
+    return host.createOrAttach({
+      sessionId,
+      cols: 80,
+      rows: 24,
+      streamClient: { onData: vi.fn(), onExit: vi.fn() }
+    })
+  }
+
+  it('enforces on desktop unless the rollback switch asks to observe', () => {
+    expect(resolveTerminalSpawnAdmissionMode({})).toBe('enforce')
+    expect(
+      resolveTerminalSpawnAdmissionMode({ [TERMINAL_SPAWN_ADMISSION_ENV_VAR]: 'enforce' })
+    ).toBe('enforce')
+    expect(resolveTerminalSpawnAdmissionMode({ [TERMINAL_SPAWN_ADMISSION_ENV_VAR]: 'bogus' })).toBe(
+      'enforce'
+    )
+    expect(
+      resolveTerminalSpawnAdmissionMode({ [TERMINAL_SPAWN_ADMISSION_ENV_VAR]: ' Observe ' })
+    ).toBe('observe')
+  })
+
+  it('restores the visible pane at once, at most 8 hidden agents, and keeps the rest sleeping', async () => {
+    startDesktopDaemon()
+    const hidden = Array.from({ length: 19 }, (_, i) =>
+      restore(`hidden-${i}`, 'automatic-restore').then(
+        () => 'restored' as const,
+        (error: Error) => error.message
+      )
+    )
+    // Why sent last: the visible pane must not queue behind a storm already in progress.
+    const visible = restore('visible', 'user-resume')
+    await vi.advanceTimersByTimeAsync(0)
+    await expect(visible).resolves.toMatchObject({ isNew: true })
+    expect(spawned).toContain('visible')
+    // Never more than the burst of 3 automatic forks before the first second passes.
+    expect(spawned.filter((id) => id.startsWith('hidden-'))).toHaveLength(3)
+
+    await vi.advanceTimersByTimeAsync(12_000)
+    const outcomes = await Promise.all(hidden)
+    expect(outcomes.filter((outcome) => outcome === 'restored')).toHaveLength(8)
+    expect(outcomes.filter((outcome) => outcome !== 'restored')).toEqual(
+      Array.from({ length: 11 }, () => 'terminal_spawn_deferred:live_cap')
+    )
+    expect(spawned).toHaveLength(9)
+    expect(guard.gate.snapshot()).toMatchObject({ mode: 'enforce', liveAutomatic: 8, waiting: 0 })
+  })
+
+  it('resumes a sleeping pane as soon as the user opens it, even at the live cap', async () => {
+    startDesktopDaemon()
+    const hidden = Array.from({ length: 9 }, (_, i) =>
+      restore(`hidden-${i}`, 'automatic-restore').catch((error: Error) => error.message)
+    )
+    await vi.advanceTimersByTimeAsync(12_000)
+    expect(await hidden[8]).toBe('terminal_spawn_deferred:live_cap')
+    expect(guard.gate.snapshot().liveAutomatic).toBe(8)
+
+    // Opening the sleeping pane retries it as the user's own resume; no timer runs before it forks.
+    await expect(restore('hidden-8', 'user-resume')).resolves.toMatchObject({ isNew: true })
+    expect(spawned.at(-1)).toBe('hidden-8')
+  })
+
+  it('never delays a manual new terminal behind a restore storm', async () => {
+    startDesktopDaemon()
+    const storm = Array.from({ length: 12 }, (_, i) =>
+      restore(`hidden-${i}`, 'automatic-restore').catch(() => null)
+    )
+    await vi.advanceTimersByTimeAsync(0)
+    expect(guard.gate.snapshot().waiting).toBeGreaterThan(0)
+
+    await expect(shell('manual')).resolves.toMatchObject({ isNew: true })
+    expect(spawned).toContain('manual')
+    await vi.advanceTimersByTimeAsync(12_000)
+    await Promise.all(storm)
+  })
+
+  // Windows cannot tell a new boot from the same one (Fast Startup), so it never pauses.
+  it.skipIf(process.platform === 'win32')(
+    'keeps hidden agents sleeping after a same-boot crash, but restores the visible pane',
+    async () => {
+      writeFileSync(
+        join(dir, 'daemon-v1.automatic-work-run.json'),
+        JSON.stringify({
+          schemaVersion: 1,
+          boot: { bootTimeMs: Date.now() - uptime() * 1000, bootId: null },
+          state: 'running',
+          mode: { kind: 'normal' }
+        })
+      )
+      startDesktopDaemon()
+      expect(guard.status()).toMatchObject({
+        admission: 'enforce',
+        mode: { kind: 'paused', reason: 'unclean_host_death' }
+      })
+
+      await expect(restore('hidden', 'automatic-restore')).rejects.toThrow(
+        'terminal_spawn_deferred:paused'
+      )
+      await expect(restore('visible', 'user-resume')).resolves.toMatchObject({ isNew: true })
+      await expect(shell('manual')).resolves.toMatchObject({ isNew: true })
+
+      // A visible split of six agents spends the user allowance of 4; the rest are paced, not slept.
+      const split = Array.from({ length: 5 }, (_, i) => restore(`split-${i}`, 'user-resume'))
+      await vi.advanceTimersByTimeAsync(12_000)
+      await expect(Promise.all(split)).resolves.toHaveLength(5)
+
+      guard.resume()
+      await expect(restore('hidden', 'automatic-restore')).resolves.toMatchObject({ isNew: true })
+      expect(spawned.at(-1)).toBe('hidden')
+      expect(spawned).toHaveLength(8)
+    }
+  )
+})

src/main/daemon/legacy-restore-records.test.ts

@@ -7,6 +7,7 @@ import {
   legacyRestoreRecordsPathFor,
   type LegacyRestoreDetails
 } from './legacy-restore-records'
+import { DEFAULT_MAX_HOST_TERMINALS } from './host-terminal-ledger'
 
 const details = (overrides: Partial<LegacyRestoreDetails> = {}): LegacyRestoreDetails => ({
   sessionId: 'wt-1@@aaaa1111',
@@ -64,11 +65,13 @@ describe('LegacyRestoreRecords', () => {
 
   it('keeps the newest records within its bound and forgets old ones', () => {
     const records = open()
-    for (let i = 0; i < 40; i++) {
+    // A storm refusing as many restores as the host could run keeps every handle.
+    for (let i = 0; i < DEFAULT_MAX_HOST_TERMINALS + 8; i++) {
       records.record(details({ command: `claude --resume ${i}` }), 'pressure')
     }
-    expect(records.list()).toHaveLength(32)
-    expect(records.list()[0]?.command).toBe('claude --resume 39')
+    expect(records.list()).toHaveLength(DEFAULT_MAX_HOST_TERMINALS)
+    expect(records.list()[0]?.command).toBe(`claude --resume ${DEFAULT_MAX_HOST_TERMINALS + 7}`)
+    expect(records.list().at(-1)?.command).toBe('claude --resume 8')
     now += 15 * 24 * 60 * 60_000
     expect(open().list()).toEqual([])
   })

src/main/daemon/legacy-restore-records.ts

@@ -1,13 +1,15 @@
 import { readFileSync } from 'node:fs'
 import { durableWriteTempPath, writeFileDurableSync } from '../durable-file-write'
 import type { LegacyRestoreRecord } from '../../shared/host-spawn-admission'
+import { DEFAULT_MAX_HOST_TERMINALS } from './host-terminal-ledger'
 
 // COMPAT(legacy-restore-intent): added 2026-10-11 (v1.4.214) for clients that predate
 // `spawnIntent` (#27452). Remove after 2027-04-11, with the gate's undeclared-caller branches.
 
 export type LegacyRestoreDetails = Omit<LegacyRestoreRecord, 'reason' | 'refusedAt'>
 
-const MAX_RECORDS = 32
+// Why the host terminal cap: a storm refusing every restore the host could run keeps each handle.
+const MAX_RECORDS = DEFAULT_MAX_HOST_TERMINALS
 const MAX_AGE_MS = 14 * 24 * 60 * 60_000
 const PRIVATE_FILE_MODE = 0o600
 

src/main/daemon/terminal-spawn-gate.ts

@@ -32,6 +32,18 @@ const OTHER_DAEMON_POLL_MS = 500
 
 export const TERMINAL_SPAWN_ADMISSION_ARG = '--spawn-admission'
 
+// Rollback switch: `observe` makes newly launched daemons count decisions without pacing.
+export const TERMINAL_SPAWN_ADMISSION_ENV_VAR = 'ORCA_TERMINAL_SPAWN_ADMISSION'
+
+/** The mode a launcher passes its daemon: enforce, unless the env var asks for `observe`. */
+export function resolveTerminalSpawnAdmissionMode(
+  env: NodeJS.ProcessEnv = process.env
+): TerminalSpawnAdmissionMode {
+  return env[TERMINAL_SPAWN_ADMISSION_ENV_VAR]?.trim().toLowerCase() === 'observe'
+    ? 'observe'
+    : 'enforce'
+}
+
 /** Reads the launcher's `--spawn-admission <mode>`; anything else observes. */
 export function readTerminalSpawnAdmissionArg(argv: readonly string[]): TerminalSpawnAdmissionMode {
   const index = argv.indexOf(TERMINAL_SPAWN_ADMISSION_ARG)

src/main/orcad/orcad-daemon-supervision.ts

@@ -17,6 +17,7 @@ import {
   readDaemonPidRecord
 } from '../daemon/daemon-provider-state'
 import { initDaemonPtyProvider } from '../daemon/daemon-provider-init'
+import { resolveTerminalSpawnAdmissionMode } from '../daemon/terminal-spawn-gate'
 import { errorMessage } from '../../shared/error-message'
 
 const WINDOWS_DAEMON_STARTUP_TIMEOUT_MS = 30_000
@@ -41,7 +42,7 @@ export async function startOrcadDaemon(
   // session dies. An orcad daemon must survive its SSH session ending — that is the point.
   const policy = {
     macosLoginSessionWatch: false,
-    spawnAdmission: 'enforce' as const,
+    spawnAdmission: resolveTerminalSpawnAdmissionMode(),
     // A freshly uploaded node.exe plus conpty can take well over 10 s on its first, AV-scanned exec.
     ...(platform === 'win32' ? { startupTimeoutMs: WINDOWS_DAEMON_STARTUP_TIMEOUT_MS } : {})
   }

src/main/startup/main-process-pty-startup.ts

@@ -4,6 +4,7 @@ import { track } from '../telemetry/client'
 import { getPtyIdForPaneKey } from '../ipc/pty/pane/key-state'
 import { getDaemonProvider, listLiveDaemonPtyIds } from '../daemon/daemon-provider-state'
 import { initDaemonPtyProvider } from '../daemon/daemon-provider-init'
+import { resolveTerminalSpawnAdmissionMode } from '../daemon/terminal-spawn-gate'
 import {
   getCodexPaneAccount,
   hasAnyRecordedLegacyWslCodexPane,
@@ -136,8 +137,8 @@ export function startTerminalRuntimeStartupServices(): WindowsDesktopStartupServ
       // serve daemon must survive its spawning session ending (SSH disconnect).
       await initDaemonPtyProvider(signal, {
         macosLoginSessionWatch: process.platform === 'darwin' && !state.isServeMode,
-        // Why: desktop observes for one release; headless hosts pace automatic restores now.
-        spawnAdmission: state.isServeMode ? 'enforce' : 'observe'
+        // Why both: desktop and `orca serve` share one restore budget per machine (D1 Q1).
+        spawnAdmission: resolveTerminalSpawnAdmissionMode()
       })
       // Why: a retained shell keeps its launch-time Codex home even when the current routing lane changes.
       const hasRetainedManagedHostPane = hasRecordedManagedHostCodexPane()

src/shared/host-spawn-admission.test.ts

@@ -163,6 +163,24 @@ describe('decideSpawnAdmission', () => {
     expect(decide(state, f, 500)).toEqual({ kind: 'admit', charge: 'user' })
   })
 
+  it('paces a user resume past its allowance but never puts it back to sleep', () => {
+    const state = createSpawnAdmissionState(LIMITS, 0)
+    const f = facts({ declaredIntent: 'user-resume', deadlineAt: 5_000 })
+    state.liveAutomatic = LIMITS.maxLiveAutomatic
+    state.automaticBucket = { tokens: 0, updatedAt: 0 }
+    const paused = { kind: 'paused', reason: 'unclean_host_death', since: 0 } as const
+    for (const mode of [{ kind: 'normal' } as const, paused]) {
+      const args = { state, facts: f, charge: 'automatic' as const, pressure: 'ok' as const }
+      expect(decideSpawnAdmission({ ...args, mode, limits: LIMITS, now: 0 })).toMatchObject({
+        kind: 'wait'
+      })
+      expect(decideSpawnAdmission({ ...args, mode, limits: LIMITS, now: 5_000 })).toEqual({
+        kind: 'admit',
+        charge: 'automatic'
+      })
+    }
+  })
+
   it('M-1: random arrivals keep the pacing and live-cap invariants', () => {
     for (let seed = 1; seed <= 25; seed++) {
       const rng = mulberry32(seed)
@@ -226,7 +244,8 @@ describe('decideSpawnAdmission', () => {
           expect(state.inFlight.automatic).toBeLessThanOrEqual(LIMITS.automaticConcurrency)
           // Burst plus one token per second over a 10 s window.
           expect(automaticAdmitsInWindow.length).toBeLessThanOrEqual(LIMITS.automaticBurst + 10)
-          if (f.declaredIntent !== null) {
+          // A user resume past its allowance is paced like an old caller, never capped.
+          if (f.declaredIntent !== null && f.declaredIntent !== 'user-resume') {
             expect(state.liveAutomatic + state.inFlight.automatic).toBeLessThanOrEqual(
               LIMITS.maxLiveAutomatic
             )

src/shared/host-spawn-admission.ts

@@ -242,10 +242,12 @@ export function decideSpawnAdmission(args: {
       ? { kind: 'refuse_retryable' }
       : { kind: 'wait', charge, retryAt: null }
   }
-  if (negotiated && mode.kind === 'paused') {
+  // Why: a user's resume past its allowance is paced, never put back to sleep in front of them.
+  const mayDefer = negotiated && facts.declaredIntent !== 'user-resume'
+  if (mayDefer && mode.kind === 'paused') {
     return { kind: 'defer', reason: 'paused' }
   }
-  if (negotiated && state.liveAutomatic + state.inFlight.automatic >= limits.maxLiveAutomatic) {
+  if (mayDefer && state.liveAutomatic + state.inFlight.automatic >= limits.maxLiveAutomatic) {
     return { kind: 'defer', reason: 'live_cap' }
   }
   const bucket = refill(
@@ -263,7 +265,7 @@ export function decideSpawnAdmission(args: {
     : now + Math.ceil((1 - bucket.tokens) / automaticTokensPerMs(limits))
   if (retryAt !== null ? retryAt > facts.deadlineAt : now >= facts.deadlineAt) {
     // Why: an old caller is admitted at its deadline rather than refused (degraded pacing).
-    return negotiated ? { kind: 'defer', reason: 'deadline' } : { kind: 'admit', charge }
+    return mayDefer ? { kind: 'defer', reason: 'deadline' } : { kind: 'admit', charge }
   }
   return { kind: 'wait', charge, retryAt }
 }