From 8e4e7be869ff2dad6034186eeb811330497cb8d1 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Tue, 13 Oct 2026 00:18:08 -0700 Subject: [PATCH] feat(daemon): pace automatic terminal restores on the desktop too (#28202) * feat(daemon): pace automatic terminal restores on the desktop too The desktop daemon now enforces the restore budget that orca serve and orcad already enforce: the visible pane first, at most 3 automatic restores at once (1/s), at most 8 live per machine, and the rest stay sleeping until opened. ORCA_TERMINAL_SPAWN_ADMISSION=observe rolls newly launched daemons back to count-only. * fix(daemon): never put a user's own resume back to sleep; keep every refused old-client restore A user-resume past the per-client allowance is still paced as automatic work but is no longer deferred for a pause, the live cap or its deadline, so a visible pane is never put back to sleep in front of the user. The legacy restore record holds as many refusals as the host could run. --------- Co-authored-by: m4air --- .../daemon/desktop-restore-admission.test.ts | 213 ++++++++++++++++++ .../daemon/legacy-restore-records.test.ts | 9 +- src/main/daemon/legacy-restore-records.ts | 4 +- src/main/daemon/terminal-spawn-gate.ts | 12 + src/main/orcad/orcad-daemon-supervision.ts | 3 +- src/main/startup/main-process-pty-startup.ts | 5 +- src/shared/host-spawn-admission.test.ts | 21 +- src/shared/host-spawn-admission.ts | 8 +- 8 files changed, 264 insertions(+), 11 deletions(-) create mode 100644 src/main/daemon/desktop-restore-admission.test.ts diff --git a/src/main/daemon/desktop-restore-admission.test.ts b/src/main/daemon/desktop-restore-admission.test.ts new file mode 100644 index 00000000000..2210f346951 --- /dev/null +++ b/src/main/daemon/desktop-restore-admission.test.ts @@ -0,0 +1,213 @@ +import './mock-descendant-sweep' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir, uptime } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { SpawnIntent } from '../../shared/host-spawn-admission' +import { DaemonAutomaticWorkGuard } from './daemon-automatic-work-guard' +import { DaemonProcessGrowthMonitor } from './daemon-process-growth-monitor' +import type { SubprocessHandle } from './session-subprocess-handle' +import { TerminalHost, type TerminalHostOptions } from './terminal-host' +import { + resolveTerminalSpawnAdmissionMode, + TERMINAL_SPAWN_ADMISSION_ENV_VAR +} from './terminal-spawn-gate' + +type SpawnSubprocess = TerminalHostOptions['spawnSubprocess'] + +// The desktop app's daemon after a restart: its renderer declares the visible pane +// `user-resume` and every hidden agent pane `automatic-restore`, with no session claim. +describe('desktop restore admission', () => { + let dir: string + let host: TerminalHost + let guard: DaemonAutomaticWorkGuard + let spawned: string[] + let exits: Map void> + let nextPid: number + + beforeEach(() => { + vi.useFakeTimers({ now: 1_000_000 }) + dir = mkdtempSync(join(tmpdir(), 'orca-desktop-restore-')) + spawned = [] + exits = new Map() + nextPid = 2000 + }) + + afterEach(async () => { + await host?.dispose() + guard?.stop() + vi.useRealTimers() + rmSync(dir, { recursive: true, force: true }) + }) + + function startDesktopDaemon(): void { + const markerPath = join(dir, 'daemon-v1.automatic-work-run.json') + guard = new DaemonAutomaticWorkGuard({ + markerPath, + // What the desktop launcher passes with no rollback switch set. + mode: resolveTerminalSpawnAdmissionMode({}), + log: { log: vi.fn() }, + listSessions: () => [], + ownsEndpoint: () => true, + monitor: new DaemonProcessGrowthMonitor({ + listSessions: () => [], + recentAutomaticWork: () => false, + sampleWhenIdle: false, + onLevelChanged: () => {}, + onTrip: () => {}, + readRows: async () => [] + }) + }) + guard.start() + const spawnSubprocess = vi.fn((opts) => { + const pid = nextPid++ + spawned.push(opts.sessionId) + const handle: SubprocessHandle = { + pid, + getForegroundProcess: vi.fn(() => null), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(() => exits.get(pid)?.(0)), + terminateOwnedTree: () => 'unavailable' as const, + forceKill: vi.fn(() => exits.get(pid)?.(137)), + signal: vi.fn(), + onData: vi.fn(), + onExit: (callback) => { + exits.set(pid, callback) + }, + dispose: vi.fn() + } + return handle + }) + host = new TerminalHost({ spawnSubprocess, spawnGate: guard.gate }) + } + + function restore(sessionId: string, declaredIntent: SpawnIntent) { + return host.createOrAttach({ + sessionId, + cols: 80, + rows: 24, + historySeedChunks: ['seed'], + spawnAdmission: { + clientId: 'desktop-main', + declaredIntent, + // The daemon's cap for a desktop create (30 s request timeout, halved, capped at 12 s). + deadlineAt: Date.now() + 12_000 + }, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + } + + function shell(sessionId: string) { + return host.createOrAttach({ + sessionId, + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + } + + it('enforces on desktop unless the rollback switch asks to observe', () => { + expect(resolveTerminalSpawnAdmissionMode({})).toBe('enforce') + expect( + resolveTerminalSpawnAdmissionMode({ [TERMINAL_SPAWN_ADMISSION_ENV_VAR]: 'enforce' }) + ).toBe('enforce') + expect(resolveTerminalSpawnAdmissionMode({ [TERMINAL_SPAWN_ADMISSION_ENV_VAR]: 'bogus' })).toBe( + 'enforce' + ) + expect( + resolveTerminalSpawnAdmissionMode({ [TERMINAL_SPAWN_ADMISSION_ENV_VAR]: ' Observe ' }) + ).toBe('observe') + }) + + it('restores the visible pane at once, at most 8 hidden agents, and keeps the rest sleeping', async () => { + startDesktopDaemon() + const hidden = Array.from({ length: 19 }, (_, i) => + restore(`hidden-${i}`, 'automatic-restore').then( + () => 'restored' as const, + (error: Error) => error.message + ) + ) + // Why sent last: the visible pane must not queue behind a storm already in progress. + const visible = restore('visible', 'user-resume') + await vi.advanceTimersByTimeAsync(0) + await expect(visible).resolves.toMatchObject({ isNew: true }) + expect(spawned).toContain('visible') + // Never more than the burst of 3 automatic forks before the first second passes. + expect(spawned.filter((id) => id.startsWith('hidden-'))).toHaveLength(3) + + await vi.advanceTimersByTimeAsync(12_000) + const outcomes = await Promise.all(hidden) + expect(outcomes.filter((outcome) => outcome === 'restored')).toHaveLength(8) + expect(outcomes.filter((outcome) => outcome !== 'restored')).toEqual( + Array.from({ length: 11 }, () => 'terminal_spawn_deferred:live_cap') + ) + expect(spawned).toHaveLength(9) + expect(guard.gate.snapshot()).toMatchObject({ mode: 'enforce', liveAutomatic: 8, waiting: 0 }) + }) + + it('resumes a sleeping pane as soon as the user opens it, even at the live cap', async () => { + startDesktopDaemon() + const hidden = Array.from({ length: 9 }, (_, i) => + restore(`hidden-${i}`, 'automatic-restore').catch((error: Error) => error.message) + ) + await vi.advanceTimersByTimeAsync(12_000) + expect(await hidden[8]).toBe('terminal_spawn_deferred:live_cap') + expect(guard.gate.snapshot().liveAutomatic).toBe(8) + + // Opening the sleeping pane retries it as the user's own resume; no timer runs before it forks. + await expect(restore('hidden-8', 'user-resume')).resolves.toMatchObject({ isNew: true }) + expect(spawned.at(-1)).toBe('hidden-8') + }) + + it('never delays a manual new terminal behind a restore storm', async () => { + startDesktopDaemon() + const storm = Array.from({ length: 12 }, (_, i) => + restore(`hidden-${i}`, 'automatic-restore').catch(() => null) + ) + await vi.advanceTimersByTimeAsync(0) + expect(guard.gate.snapshot().waiting).toBeGreaterThan(0) + + await expect(shell('manual')).resolves.toMatchObject({ isNew: true }) + expect(spawned).toContain('manual') + await vi.advanceTimersByTimeAsync(12_000) + await Promise.all(storm) + }) + + // Windows cannot tell a new boot from the same one (Fast Startup), so it never pauses. + it.skipIf(process.platform === 'win32')( + 'keeps hidden agents sleeping after a same-boot crash, but restores the visible pane', + async () => { + writeFileSync( + join(dir, 'daemon-v1.automatic-work-run.json'), + JSON.stringify({ + schemaVersion: 1, + boot: { bootTimeMs: Date.now() - uptime() * 1000, bootId: null }, + state: 'running', + mode: { kind: 'normal' } + }) + ) + startDesktopDaemon() + expect(guard.status()).toMatchObject({ + admission: 'enforce', + mode: { kind: 'paused', reason: 'unclean_host_death' } + }) + + await expect(restore('hidden', 'automatic-restore')).rejects.toThrow( + 'terminal_spawn_deferred:paused' + ) + await expect(restore('visible', 'user-resume')).resolves.toMatchObject({ isNew: true }) + await expect(shell('manual')).resolves.toMatchObject({ isNew: true }) + + // A visible split of six agents spends the user allowance of 4; the rest are paced, not slept. + const split = Array.from({ length: 5 }, (_, i) => restore(`split-${i}`, 'user-resume')) + await vi.advanceTimersByTimeAsync(12_000) + await expect(Promise.all(split)).resolves.toHaveLength(5) + + guard.resume() + await expect(restore('hidden', 'automatic-restore')).resolves.toMatchObject({ isNew: true }) + expect(spawned.at(-1)).toBe('hidden') + expect(spawned).toHaveLength(8) + } + ) +}) diff --git a/src/main/daemon/legacy-restore-records.test.ts b/src/main/daemon/legacy-restore-records.test.ts index 0debaf2800b..e8e25901ebb 100644 --- a/src/main/daemon/legacy-restore-records.test.ts +++ b/src/main/daemon/legacy-restore-records.test.ts @@ -7,6 +7,7 @@ import { legacyRestoreRecordsPathFor, type LegacyRestoreDetails } from './legacy-restore-records' +import { DEFAULT_MAX_HOST_TERMINALS } from './host-terminal-ledger' const details = (overrides: Partial = {}): LegacyRestoreDetails => ({ sessionId: 'wt-1@@aaaa1111', @@ -64,11 +65,13 @@ describe('LegacyRestoreRecords', () => { it('keeps the newest records within its bound and forgets old ones', () => { const records = open() - for (let i = 0; i < 40; i++) { + // A storm refusing as many restores as the host could run keeps every handle. + for (let i = 0; i < DEFAULT_MAX_HOST_TERMINALS + 8; i++) { records.record(details({ command: `claude --resume ${i}` }), 'pressure') } - expect(records.list()).toHaveLength(32) - expect(records.list()[0]?.command).toBe('claude --resume 39') + expect(records.list()).toHaveLength(DEFAULT_MAX_HOST_TERMINALS) + expect(records.list()[0]?.command).toBe(`claude --resume ${DEFAULT_MAX_HOST_TERMINALS + 7}`) + expect(records.list().at(-1)?.command).toBe('claude --resume 8') now += 15 * 24 * 60 * 60_000 expect(open().list()).toEqual([]) }) diff --git a/src/main/daemon/legacy-restore-records.ts b/src/main/daemon/legacy-restore-records.ts index 39d80167a05..0dbd2acd472 100644 --- a/src/main/daemon/legacy-restore-records.ts +++ b/src/main/daemon/legacy-restore-records.ts @@ -1,13 +1,15 @@ import { readFileSync } from 'node:fs' import { durableWriteTempPath, writeFileDurableSync } from '../durable-file-write' import type { LegacyRestoreRecord } from '../../shared/host-spawn-admission' +import { DEFAULT_MAX_HOST_TERMINALS } from './host-terminal-ledger' // COMPAT(legacy-restore-intent): added 2026-10-11 (v1.4.214) for clients that predate // `spawnIntent` (#27452). Remove after 2027-04-11, with the gate's undeclared-caller branches. export type LegacyRestoreDetails = Omit -const MAX_RECORDS = 32 +// Why the host terminal cap: a storm refusing every restore the host could run keeps each handle. +const MAX_RECORDS = DEFAULT_MAX_HOST_TERMINALS const MAX_AGE_MS = 14 * 24 * 60 * 60_000 const PRIVATE_FILE_MODE = 0o600 diff --git a/src/main/daemon/terminal-spawn-gate.ts b/src/main/daemon/terminal-spawn-gate.ts index 461e4116fc4..c75b4c3e3db 100644 --- a/src/main/daemon/terminal-spawn-gate.ts +++ b/src/main/daemon/terminal-spawn-gate.ts @@ -32,6 +32,18 @@ const OTHER_DAEMON_POLL_MS = 500 export const TERMINAL_SPAWN_ADMISSION_ARG = '--spawn-admission' +// Rollback switch: `observe` makes newly launched daemons count decisions without pacing. +export const TERMINAL_SPAWN_ADMISSION_ENV_VAR = 'ORCA_TERMINAL_SPAWN_ADMISSION' + +/** The mode a launcher passes its daemon: enforce, unless the env var asks for `observe`. */ +export function resolveTerminalSpawnAdmissionMode( + env: NodeJS.ProcessEnv = process.env +): TerminalSpawnAdmissionMode { + return env[TERMINAL_SPAWN_ADMISSION_ENV_VAR]?.trim().toLowerCase() === 'observe' + ? 'observe' + : 'enforce' +} + /** Reads the launcher's `--spawn-admission `; anything else observes. */ export function readTerminalSpawnAdmissionArg(argv: readonly string[]): TerminalSpawnAdmissionMode { const index = argv.indexOf(TERMINAL_SPAWN_ADMISSION_ARG) diff --git a/src/main/orcad/orcad-daemon-supervision.ts b/src/main/orcad/orcad-daemon-supervision.ts index 6a87db81f70..aebbf6a0f38 100644 --- a/src/main/orcad/orcad-daemon-supervision.ts +++ b/src/main/orcad/orcad-daemon-supervision.ts @@ -17,6 +17,7 @@ import { readDaemonPidRecord } from '../daemon/daemon-provider-state' import { initDaemonPtyProvider } from '../daemon/daemon-provider-init' +import { resolveTerminalSpawnAdmissionMode } from '../daemon/terminal-spawn-gate' import { errorMessage } from '../../shared/error-message' const WINDOWS_DAEMON_STARTUP_TIMEOUT_MS = 30_000 @@ -41,7 +42,7 @@ export async function startOrcadDaemon( // session dies. An orcad daemon must survive its SSH session ending — that is the point. const policy = { macosLoginSessionWatch: false, - spawnAdmission: 'enforce' as const, + spawnAdmission: resolveTerminalSpawnAdmissionMode(), // A freshly uploaded node.exe plus conpty can take well over 10 s on its first, AV-scanned exec. ...(platform === 'win32' ? { startupTimeoutMs: WINDOWS_DAEMON_STARTUP_TIMEOUT_MS } : {}) } diff --git a/src/main/startup/main-process-pty-startup.ts b/src/main/startup/main-process-pty-startup.ts index 64f7ea41f38..4943ed79b93 100644 --- a/src/main/startup/main-process-pty-startup.ts +++ b/src/main/startup/main-process-pty-startup.ts @@ -4,6 +4,7 @@ import { track } from '../telemetry/client' import { getPtyIdForPaneKey } from '../ipc/pty/pane/key-state' import { getDaemonProvider, listLiveDaemonPtyIds } from '../daemon/daemon-provider-state' import { initDaemonPtyProvider } from '../daemon/daemon-provider-init' +import { resolveTerminalSpawnAdmissionMode } from '../daemon/terminal-spawn-gate' import { getCodexPaneAccount, hasAnyRecordedLegacyWslCodexPane, @@ -136,8 +137,8 @@ export function startTerminalRuntimeStartupServices(): WindowsDesktopStartupServ // serve daemon must survive its spawning session ending (SSH disconnect). await initDaemonPtyProvider(signal, { macosLoginSessionWatch: process.platform === 'darwin' && !state.isServeMode, - // Why: desktop observes for one release; headless hosts pace automatic restores now. - spawnAdmission: state.isServeMode ? 'enforce' : 'observe' + // Why both: desktop and `orca serve` share one restore budget per machine (D1 Q1). + spawnAdmission: resolveTerminalSpawnAdmissionMode() }) // Why: a retained shell keeps its launch-time Codex home even when the current routing lane changes. const hasRetainedManagedHostPane = hasRecordedManagedHostCodexPane() diff --git a/src/shared/host-spawn-admission.test.ts b/src/shared/host-spawn-admission.test.ts index f9eea768a18..2817285f742 100644 --- a/src/shared/host-spawn-admission.test.ts +++ b/src/shared/host-spawn-admission.test.ts @@ -163,6 +163,24 @@ describe('decideSpawnAdmission', () => { expect(decide(state, f, 500)).toEqual({ kind: 'admit', charge: 'user' }) }) + it('paces a user resume past its allowance but never puts it back to sleep', () => { + const state = createSpawnAdmissionState(LIMITS, 0) + const f = facts({ declaredIntent: 'user-resume', deadlineAt: 5_000 }) + state.liveAutomatic = LIMITS.maxLiveAutomatic + state.automaticBucket = { tokens: 0, updatedAt: 0 } + const paused = { kind: 'paused', reason: 'unclean_host_death', since: 0 } as const + for (const mode of [{ kind: 'normal' } as const, paused]) { + const args = { state, facts: f, charge: 'automatic' as const, pressure: 'ok' as const } + expect(decideSpawnAdmission({ ...args, mode, limits: LIMITS, now: 0 })).toMatchObject({ + kind: 'wait' + }) + expect(decideSpawnAdmission({ ...args, mode, limits: LIMITS, now: 5_000 })).toEqual({ + kind: 'admit', + charge: 'automatic' + }) + } + }) + it('M-1: random arrivals keep the pacing and live-cap invariants', () => { for (let seed = 1; seed <= 25; seed++) { const rng = mulberry32(seed) @@ -226,7 +244,8 @@ describe('decideSpawnAdmission', () => { expect(state.inFlight.automatic).toBeLessThanOrEqual(LIMITS.automaticConcurrency) // Burst plus one token per second over a 10 s window. expect(automaticAdmitsInWindow.length).toBeLessThanOrEqual(LIMITS.automaticBurst + 10) - if (f.declaredIntent !== null) { + // A user resume past its allowance is paced like an old caller, never capped. + if (f.declaredIntent !== null && f.declaredIntent !== 'user-resume') { expect(state.liveAutomatic + state.inFlight.automatic).toBeLessThanOrEqual( LIMITS.maxLiveAutomatic ) diff --git a/src/shared/host-spawn-admission.ts b/src/shared/host-spawn-admission.ts index 9fb5f10e559..2795c36236f 100644 --- a/src/shared/host-spawn-admission.ts +++ b/src/shared/host-spawn-admission.ts @@ -242,10 +242,12 @@ export function decideSpawnAdmission(args: { ? { kind: 'refuse_retryable' } : { kind: 'wait', charge, retryAt: null } } - if (negotiated && mode.kind === 'paused') { + // Why: a user's resume past its allowance is paced, never put back to sleep in front of them. + const mayDefer = negotiated && facts.declaredIntent !== 'user-resume' + if (mayDefer && mode.kind === 'paused') { return { kind: 'defer', reason: 'paused' } } - if (negotiated && state.liveAutomatic + state.inFlight.automatic >= limits.maxLiveAutomatic) { + if (mayDefer && state.liveAutomatic + state.inFlight.automatic >= limits.maxLiveAutomatic) { return { kind: 'defer', reason: 'live_cap' } } const bucket = refill( @@ -263,7 +265,7 @@ export function decideSpawnAdmission(args: { : now + Math.ceil((1 - bucket.tokens) / automaticTokensPerMs(limits)) if (retryAt !== null ? retryAt > facts.deadlineAt : now >= facts.deadlineAt) { // Why: an old caller is admitted at its deadline rather than refused (degraded pacing). - return negotiated ? { kind: 'defer', reason: 'deadline' } : { kind: 'admit', charge } + return mayDefer ? { kind: 'defer', reason: 'deadline' } : { kind: 'admit', charge } } return { kind: 'wait', charge, retryAt } }