fix(daemon): bound the whole boot-recovery sequence with one budget (STA-5732) (#17427)

* fix(daemon): bound the whole boot-recovery sequence with one budget (STA-5732)

* fix(daemon): keep socket probes inside recovery budget

* fix(daemon): size the recovery budget against the real post-kill tail

The 24s budget reserved only 9s for everything after the deadline, leaving
27s of the startup PTY gate's fail-open cap unused — and every unused second
is one where a daemon that would have drained gets killed with its live PTYs
instead. Reserve each post-deadline stage's actual hard cap (kill 10.5s, fork
10s, lease 5s) and spend the rest: 24s -> 32s of adopt window.

* fix(daemon): keep the last-resort endpoint rescue outside the recovery budget

The rescue probe in the launcher's outer catch was clamped to the recovery
budget's remainder, but it runs *after* that budget by construction — past
prepareDaemonReplacement, killStaleDaemon, the fork and the adoption lease.
The remainder is therefore essentially always negative, so Math.max(1, ...)
handed a live socket a 1ms connect window. On the loaded machine this path
exists for the probe loses to its own timer, the launcher rethrows, and a
recoverable degraded adoption becomes total daemon loss for the whole run —
the outcome the comment above it exists to prevent. Restore the 1s default
and pin the window with a test that drives the launcher to that catch with
the budget already spent.

Also make the deliberate narrowing legible instead of implicit:

- daemon-recovery-budget.ts: TRANSIENT_WEDGE_DRAIN_MS documented 20s as the
  grace #8697 sized, but #8697's merged second commit (840d3277d1) widened
  it to 11 retries ~= 60s. Record that 20s is the drain estimate and that the
  budget deliberately sits under #8697's shipped grace.
- daemon-init-wedged-daemon-grace.test.ts: pin the trade directly — a wedge
  draining after the budget is replaced and loses its live sessions.
- Rewrite 'preserves a daemon that stays wedged until the LAST allowed grace
  retry' onto the simulated clock. It never mocked Date.now, so its 12 probes
  elapsed ~0ms and asserted a retry grace the wall clock can no longer
  deliver; it now pins the last drain the budget still adopts.

* fix(daemon): name the socket probe default and correct the grace-retry rationale

Answers the review round on the budget accounting: the outer-catch endpoint
rescue is deliberately outside it, and the preflight clamp no longer duplicates
probeDaemonSocket's default as a bare literal.
This commit is contained in:
Neil
2026-08-31 02:41:11 -07:00
committed by GitHub
parent 7cb1db63db
commit e17c98d425
12 changed files with 480 additions and 65 deletions
@@ -60,6 +60,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
function basicClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: vi.fn(async () => ({ sessions: [] })),
disconnect: vi.fn()
}
@@ -144,6 +145,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
throw new Error('adoption unavailable')
}),
ensureConnectedWithin: vi.fn(async () => {
throw new Error('adoption unavailable')
}),
request: vi.fn(),
disconnect
}
@@ -5,6 +5,7 @@ import {
FAKE_RUNTIME_DIR,
FAKE_DAEMON_ENTRY_PATH
} from './daemon-init-test-harness'
import { DAEMON_RECOVERY_BUDGET_MS } from './daemon-recovery-budget'
const {
isPackagedMock,
@@ -16,6 +17,8 @@ const {
killStaleDaemonMock,
replaceDaemonPidFileMock,
daemonClientMock,
netConnectMock,
probeSocketExistsMock,
spawnerInstances,
trackDaemonReplacedMock,
importFresh,
@@ -220,6 +223,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementation(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: vi.fn(async () => ({ sessions: [{ sessionId: 's1', isAlive: true }] })),
disconnect: vi.fn()
}
@@ -243,6 +247,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
events.push('full-pair')
}),
ensureConnectedWithin: vi.fn(async () => {
events.push('full-pair')
}),
request: vi.fn(),
disconnect
}
@@ -279,6 +286,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockAdoptionClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
getDaemonIdentity: vi.fn(() => endpointIdentity),
request: vi.fn(),
disconnect: vi.fn()
@@ -337,6 +345,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockAdoptionClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
getDaemonIdentity: vi.fn(() => endpointIdentity),
request: vi.fn(),
disconnect: vi.fn()
@@ -385,6 +394,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockAdoptionClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
getDaemonIdentity: vi.fn(() => endpointIdentity),
request: vi.fn(),
disconnect: vi.fn()
@@ -456,6 +466,113 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
}
})
it('rescues the endpoint owner with a full probe window after the recovery budget is spent', async () => {
// Why: this last-resort probe runs AFTER the budget — past the kill, the fork and the lease —
// so anything clamped to the remainder is a ~1ms probe that loses to its own timer against a
// live socket, and the rescue that saves every persistent session degrades into total loss.
const mod = await importFresh()
checkDaemonHealthMock.mockResolvedValue('unreachable')
await mod.initDaemonPtyProvider()
const clock = { now: 1_700_000_000_000 }
const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => clock.now)
// The real post-deadline tail: kill, fork and lease all run after recovery has decided.
killStaleDaemonMock.mockImplementationOnce(async () => {
clock.now += DAEMON_RECOVERY_BUDGET_MS + 5_000
return { killed: true, liveOwnerSurvived: false }
})
function basicClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: vi.fn(async () => ({ sessions: [] })),
disconnect: vi.fn()
}
}
daemonClientMock.mockImplementationOnce(basicClient)
daemonClientMock.mockImplementationOnce(basicClient)
// The fresh child lost the endpoint to another daemon: the lease rejects on identity.
daemonClientMock.mockImplementationOnce(function MockPostReadyClient() {
return {
...basicClient(),
getDaemonIdentity: vi.fn(() => ({
pid: 999,
startedAtMs: 900_000,
launchNonce: 'stale-launch'
}))
}
})
const exitHandlers: ((code?: unknown) => void)[] = []
const child = {
pid: 12345,
connected: true,
exitCode: null as number | null,
signalCode: null as NodeJS.Signals | null,
on(event: string, callback: (arg?: unknown) => void) {
if (event === 'exit') {
exitHandlers.push(callback)
}
if (event === 'message') {
queueMicrotask(() => callback({ type: 'ready', startedAtMs: 1_000_000 }))
}
return this
},
once(event: string, callback: (arg?: unknown) => void) {
return child.on(event, callback)
},
off() {
return child
},
kill: vi.fn(() => true),
disconnect: vi.fn(() => {
child.connected = false
}),
unref: vi.fn()
}
forkMock.mockReturnValueOnce(child)
const kill = vi.spyOn(process, 'kill').mockImplementation(() => {
queueMicrotask(() => {
child.exitCode = 0
for (const callback of exitHandlers.slice()) {
callback(0)
}
})
return true
})
probeSocketExistsMock.mockReturnValue(true)
// A loaded host answers late but well inside probeDaemonSocket's own 1s default.
netConnectMock.mockImplementation(() => ({
on(event: string, callback: () => void) {
if (event === 'connect') {
setTimeout(callback, 500)
}
return this
},
removeListener() {
return this
},
destroy() {}
}))
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const launcher = spawnerInstances[0].launcher as (
socketPath: string,
tokenPath: string,
pidPath?: string,
launchNonce?: string
) => Promise<{ mode?: string; releaseAdoptionLease?(): void }>
try {
const handle = await launcher('/fake/socket', '/fake/token', '/fake/daemon.pid', 'launch-new')
expect(handle.mode).toBe('degraded-new-pty-fallback')
handle.releaseAdoptionLease?.()
} finally {
warn.mockRestore()
kill.mockRestore()
nowSpy.mockRestore()
probeSocketExistsMock.mockReturnValue(false)
}
})
it('disconnects every temporary client when healthy adoption fails', async () => {
const mod = await importFresh()
await mod.initDaemonPtyProvider()
@@ -467,6 +584,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
throw new Error('initial adoption failed')
}),
ensureConnectedWithin: vi.fn(async () => {
throw new Error('initial adoption failed')
}),
request: vi.fn(),
disconnect: initialDisconnect
}
@@ -476,6 +596,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
throw new Error('replacement adoption failed')
}),
ensureConnectedWithin: vi.fn(async () => {
throw new Error('replacement adoption failed')
}),
request: vi.fn(),
disconnect: replacementDisconnect
}
@@ -91,6 +91,7 @@ export async function importFreshDaemonInit(state: DaemonInitMockState) {
daemonClientMock.mockImplementation(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
request: vi.fn(async () => ({ sessions: [] })),
disconnect: vi.fn()
@@ -70,6 +70,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: requestMock,
disconnect: disconnectMock
}
@@ -89,7 +90,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
'/fake/token',
FAKE_DAEMON_ENTRY_PATH
)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
expect(disconnectMock).toHaveBeenCalledOnce()
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
@@ -110,6 +111,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: requestMock,
disconnect: disconnectMock
}
@@ -123,7 +125,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
await launcher('/fake/socket', '/fake/token')
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
expect(disconnectMock).toHaveBeenCalledOnce()
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
@@ -208,6 +210,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: requestMock,
disconnect: disconnectMock
}
@@ -222,7 +225,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
await launcher('/fake/socket', '/fake/token')
expect(getMacDaemonSystemResolverHealthMock).toHaveBeenCalledWith('/fake/socket', '/fake/token')
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
expect(disconnectMock).toHaveBeenCalledOnce()
expect(getDaemonLaunchIdentityMock).not.toHaveBeenCalled()
expect(killStaleDaemonMock).not.toHaveBeenCalled()
@@ -246,6 +249,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: requestMock,
disconnect: disconnectMock
}
@@ -259,7 +263,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
await launcher('/fake/socket', '/fake/token')
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
expect(disconnectMock).toHaveBeenCalledOnce()
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
@@ -282,6 +286,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: requestMock,
disconnect: disconnectMock
}
@@ -295,7 +300,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
await launcher('/fake/socket', '/fake/token')
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
expect(disconnectMock).toHaveBeenCalledOnce()
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
@@ -317,6 +322,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: requestMock,
disconnect: vi.fn()
}
@@ -333,7 +339,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
const handle = await launcher('/fake/socket', '/fake/token')
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
expect(handle.mode).toBe('degraded-new-pty-fallback')
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
@@ -348,6 +354,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
throw new Error('daemon is wedged')
}),
ensureConnectedWithin: vi.fn(async () => {
throw new Error('daemon is wedged')
}),
request: vi.fn(),
disconnect: vi.fn()
}
@@ -164,6 +164,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: requestMock,
disconnect: disconnectMock
}
@@ -184,7 +185,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
'/fake/token',
'1.2.3'
)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
expect(disconnectMock).toHaveBeenCalledOnce()
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
@@ -304,6 +304,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
throw new Error('connect ENOENT')
}),
ensureConnectedWithin: vi.fn(async () => {
throw new Error('connect ENOENT')
}),
request: vi.fn(),
disconnect: vi.fn()
}
@@ -102,6 +102,7 @@ function createDaemonInitMockState(): DaemonInitMockState {
const daemonClientMock = vi.fn().mockImplementation(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
request: vi.fn(async () => ({ sessions: [] })),
disconnect: vi.fn()
@@ -212,6 +213,7 @@ export function createDaemonInitMocks(): DaemonInitMocks {
state.daemonClientMock.mockImplementationOnce(function MockAdoptionClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: vi.fn(),
disconnect: vi.fn()
}
@@ -1,6 +1,12 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { WEDGED_DAEMON_GRACE_RETRIES } from './daemon-init'
import { FAKE_RUNTIME_DIR } from './daemon-init-test-harness'
import {
DAEMON_RECOVERY_BUDGET_MS,
DAEMON_RECOVERY_PROBE_MS,
TRANSIENT_WEDGE_DRAIN_MS
} from './daemon-recovery-budget'
import { LOCAL_PTY_STARTUP_FAIL_OPEN_TIMEOUT_MS } from '../startup/first-window-startup-services'
const {
probeSocketExistsMock,
@@ -64,6 +70,121 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
}
}
// The real client's own defaults, which is what an unbudgeted probe pays. Mirrored here so the
// simulated clock below charges a wedge exactly what production would have spent on it.
const CLIENT_CONNECT_TIMEOUT_MS = 5_000
const CLIENT_REQUEST_TIMEOUT_MS = 30_000
const HEALTH_CHECK_TIMEOUT_MS = 3_000
// What still has to fit inside the startup PTY gate once recovery decides, at each stage's own
// hard cap: killStaleDaemon's two identity inspections, SIGTERM wait, SIGKILL confirm and
// endpoint probe (daemon-stale-kill.ts, daemon-process-identity-query.ts,
// daemon-endpoint-probe.ts), then launchDaemonChild's readiness timeout
// (daemon-launched-child.ts), then the adoption lease and adapter connects — the last against a
// daemon that just reported ready, so an allowance rather than the client's unbudgeted 4x5s.
const POST_RECOVERY_KILL_MS = 3_000 + 3_000 + 3_000 + 1_000 + 500
const POST_RECOVERY_FORK_MS = 10_000
const POST_RECOVERY_LEASE_MS = 5_000
const POST_RECOVERY_RELAUNCH_MS =
POST_RECOVERY_KILL_MS + POST_RECOVERY_FORK_MS + POST_RECOVERY_LEASE_MS
/**
* Runs the launcher against a daemon that accepts connections and answers nothing until
* `drainsAfterMs` of simulated elapsed time, on a hand-driven clock: each mocked wait advances
* Date.now by exactly what the real call would have blocked for, or stops at the drain — the
* probe already in flight when the daemon comes back is the one that gets an answer. Returns
* how long the adopt-or-replace decision took.
*/
async function runWedgedRecovery(
wedgeAt: 'handshake' | 'listSessions',
drainsAfterMs = Number.POSITIVE_INFINITY
): Promise<number> {
const mod = await importFresh()
await mod.initDaemonPtyProvider()
const startedAtMs = 1_700_000_000_000
const drainsAtMs = startedAtMs + drainsAfterMs
const clock = { now: startedAtMs }
const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => clock.now)
const wedge = { active: true, drained: false }
const stall = async (timeoutMs: number, message: string): Promise<void> => {
if (drainsAtMs > clock.now + timeoutMs) {
clock.now += timeoutMs
throw new Error(message)
}
clock.now = Math.max(clock.now, drainsAtMs)
wedge.active = false
wedge.drained = true
}
daemonClientMock.mockImplementation(function MockDaemonClient() {
const connect = async (timeoutMs = CLIENT_CONNECT_TIMEOUT_MS): Promise<void> => {
if (wedge.active && wedgeAt === 'handshake') {
await stall(timeoutMs, 'Hello response timed out')
}
}
return {
ensureConnected: vi.fn(() => connect()),
ensureConnectedWithin: vi.fn(connect),
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
request: vi.fn(async (_type: string, _payload: unknown, timeoutMs?: number) => {
if (wedge.active && wedgeAt === 'listSessions') {
await stall(timeoutMs ?? CLIENT_REQUEST_TIMEOUT_MS, 'Request timed out')
}
// A daemon that drained still owns the sessions this grace exists to preserve.
return { sessions: wedge.drained ? [{ sessionId: 'wt-1@@live', isAlive: true }] : [] }
}),
disconnect: vi.fn()
}
})
checkDaemonHealthMock.mockImplementationOnce(async () => {
clock.now += HEALTH_CHECK_TIMEOUT_MS
return 'unreachable'
})
// Ending the wedge on the kill keeps the replacement daemon answering its adoption lease.
killStaleDaemonMock.mockImplementationOnce(async () => {
wedge.active = false
return { killed: true, liveOwnerSurvived: false }
})
probeSocketExistsMock.mockReturnValue(true)
netConnectMock.mockImplementation(stubAliveSocketConnect)
forkMock.mockImplementationOnce(() => ({
pid: 12345,
on(event: string, cb: (arg?: unknown) => void) {
if (event === 'message') {
queueMicrotask(() => cb({ type: 'ready', startedAtMs: 1_000_000 }))
}
return this
},
off() {
return this
},
disconnect: vi.fn(),
unref: vi.fn()
}))
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
const launcher = spawnerInstances[0].launcher as (
socketPath: string,
tokenPath: string
) => Promise<{ shutdown(): Promise<void> }>
try {
await launcher('/fake/socket', '/fake/token')
} finally {
warnSpy.mockRestore()
nowSpy.mockRestore()
daemonClientMock.mockImplementation(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: vi.fn(async () => ({ sessions: [] })),
disconnect: vi.fn()
}
})
}
// Nothing after the adopt-or-replace decision advances the simulated clock, so this is it.
return clock.now - startedAtMs
}
it('adopts a transiently wedged daemon that drains and reports live sessions within the grace window', async () => {
// Why: Windows update-relaunch — post-install load wedges the daemon briefly; it still owns live sessions, so grace-adopt not kill.
const mod = await importFresh()
@@ -75,6 +196,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
throw new Error('Hello response timed out')
}),
ensureConnectedWithin: vi.fn(async () => {
throw new Error('Hello response timed out')
}),
request: vi.fn(),
disconnect: vi.fn()
}
@@ -82,6 +206,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
daemonClientMock.mockImplementationOnce(function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: vi.fn(async () => ({
sessions: [{ sessionId: 'wt-1@@live', isAlive: true }]
})),
@@ -111,6 +236,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
const answeringDefault = function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
ensureConnectedWithin: vi.fn(async () => {}),
request: vi.fn(async () => ({ sessions: [] })),
disconnect: vi.fn()
}
@@ -125,6 +251,11 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
throw new Error('Hello response timed out')
}
}),
ensureConnectedWithin: vi.fn(async () => {
if (daemonClientConstructionCount <= 2 + WEDGED_DAEMON_GRACE_RETRIES) {
throw new Error('Hello response timed out')
}
}),
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
request: vi.fn(),
disconnect: vi.fn()
@@ -183,56 +314,70 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
}
})
it('grace budget is generous enough to ride out a ~60s transient wedge', () => {
// Why: each probe waits the client's 5s hello timeout, so 1 + 11 probes ≈ 60s of drain grace; don't cut without telemetry.
expect(WEDGED_DAEMON_GRACE_RETRIES).toBeGreaterThanOrEqual(11)
it('bounds recovery inside the startup PTY gate without cutting into the drain grace', () => {
// Lower bound: #8697 bought adoption of a daemon that drains within ~20s *with* its live
// sessions; a budget that expires first turns that adoption back into a kill.
expect(DAEMON_RECOVERY_BUDGET_MS).toBeGreaterThan(TRANSIENT_WEDGE_DRAIN_MS)
// Upper bound: recovery is only the first phase inside the gate's fail-open cap — the kill
// and the relaunch that follow it run inside the same cap.
expect(DAEMON_RECOVERY_BUDGET_MS + POST_RECOVERY_RELAUNCH_MS).toBeLessThan(
LOCAL_PTY_STARTUP_FAIL_OPEN_TIMEOUT_MS
)
// Why the probe outruns the health check: it is the second opinion on that check's verdict,
// so on the loaded machine it exists for, the same bar would just reproduce it.
expect(DAEMON_RECOVERY_PROBE_MS).toBeGreaterThan(HEALTH_CHECK_TIMEOUT_MS)
// Why: the count is a spin guard for instantly-failing probes only. If it could bind first, a
// wedge would again be graced for however long its probes happened to take.
expect(WEDGED_DAEMON_GRACE_RETRIES * DAEMON_RECOVERY_PROBE_MS).toBeGreaterThan(
DAEMON_RECOVERY_BUDGET_MS
)
})
it('preserves a daemon that stays wedged until the LAST allowed grace retry', async () => {
// Why: daemon drains only on the last allowed probe (1 + WEDGED_DAEMON_GRACE_RETRIES) — must be preserved, not replaced.
const mod = await importFresh()
await mod.initDaemonPtyProvider()
it('preserves a daemon that drains on the last probe the recovery budget allows', async () => {
// Why rewritten onto the clock: this used to drain on probe 1 + WEDGED_DAEMON_GRACE_RETRIES
// with real Date.now, so its 12 probes elapsed ~0ms and the budget never bound — a grace
// production can no longer deliver, since each failing probe costs up to
// DAEMON_RECOVERY_PROBE_MS. The count's exhaustion side stays pinned by the #8689 test above.
const recoveryMs = await runWedgedRecovery('handshake', DAEMON_RECOVERY_BUDGET_MS - 1_000)
let probe = 0
const answeringDefault = function MockDaemonClient() {
return {
ensureConnected: vi.fn(async () => {}),
request: vi.fn(async () => ({ sessions: [] })),
disconnect: vi.fn()
}
}
daemonClientMock.mockImplementation(function MockDaemonClient() {
probe += 1
const drainsNow = probe >= 1 + WEDGED_DAEMON_GRACE_RETRIES
return {
ensureConnected: vi.fn(async () => {
if (!drainsNow) {
throw new Error('Hello response timed out')
}
}),
request: vi.fn(async () => ({
sessions: drainsNow ? [{ sessionId: 'wt-1@@live', isAlive: true }] : []
})),
disconnect: vi.fn()
}
})
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
expect(recoveryMs).toBeLessThanOrEqual(DAEMON_RECOVERY_BUDGET_MS)
})
const launcher = spawnerInstances[0].launcher as (
socketPath: string,
tokenPath: string
) => Promise<{ shutdown(): Promise<void> }>
checkDaemonHealthMock.mockResolvedValueOnce('unreachable')
probeSocketExistsMock.mockReturnValue(true)
netConnectMock.mockImplementation(stubAliveSocketConnect)
it('ends the grace on the recovery budget when every handshake stalls', async () => {
// Why: unbudgeted, this cost the client's 5s connect default once per probe across 12 probes
// — ~68s with the health check, past the gate's 60s fail-open cap (STA-5732).
expect(await runWedgedRecovery('handshake')).toBeLessThanOrEqual(DAEMON_RECOVERY_BUDGET_MS)
expect(killStaleDaemonMock).toHaveBeenCalled()
})
try {
await launcher('/fake/socket', '/fake/token')
it('ends the grace on the recovery budget when the daemon answers hello and then wedges', async () => {
// Why: this is the shape the ticket reported — listSessions fell back to the client's 30s
// request default, so 12 probes stalled startup for minutes.
expect(await runWedgedRecovery('listSessions')).toBeLessThanOrEqual(DAEMON_RECOVERY_BUDGET_MS)
expect(killStaleDaemonMock).toHaveBeenCalled()
})
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
} finally {
daemonClientMock.mockImplementation(answeringDefault)
}
it('still adopts a wedge that drains at the far edge of the documented window (#8697)', async () => {
// Why: the wall clock now ends the grace, so the budget is the only thing keeping the
// Windows update-relaunch wedge adoptable. It comes back owning live sessions well after the
// probes start failing; recovery has to still be probing then instead of having killed it.
const recoveryMs = await runWedgedRecovery('handshake', TRANSIENT_WEDGE_DRAIN_MS)
expect(killStaleDaemonMock).not.toHaveBeenCalled()
expect(forkMock).not.toHaveBeenCalled()
expect(recoveryMs).toBeGreaterThanOrEqual(TRANSIENT_WEDGE_DRAIN_MS)
})
it('replaces a wedge that drains after the recovery budget (accepted trade vs #8697)', async () => {
// Why pinned rather than left implicit: #8697's merged grace was 11 retries ~= 60s, chosen to
// keep live-session loss near zero. Bounding it at DAEMON_RECOVERY_BUDGET_MS is a deliberate
// narrowing — a Windows update-relaunch wedge that drains after the budget is now replaced and
// its live terminal/agent sessions are destroyed. Only the window size is tunable.
await runWedgedRecovery('handshake', DAEMON_RECOVERY_BUDGET_MS + 5_000)
expect(killStaleDaemonMock).toHaveBeenCalled()
})
it('replaces a hello-rejected daemon even though its pipe accepts connections', async () => {
@@ -245,6 +390,9 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
ensureConnected: vi.fn(async () => {
throw new Error('Hello rejected')
}),
ensureConnectedWithin: vi.fn(async () => {
throw new Error('Hello rejected')
}),
request: vi.fn(),
disconnect: vi.fn()
}
+22 -4
View File
@@ -4,6 +4,8 @@ import { join } from 'node:path'
import { getAppEnvironment } from '../../shared/app-environment'
import { getDaemonLogFilePath } from '../observability/logs-directory'
import { DaemonClient } from './client'
import { daemonRecoveryProbeTimeoutMs } from './daemon-recovery-budget'
import { remainingDaemonRequestTimeoutMs } from './daemon-request-deadline'
import { PROTOCOL_VERSION, type ListSessionsResult } from './types'
export function getDaemonRuntimeDir(): string {
@@ -44,8 +46,14 @@ export function daemonLogArgs(): string[] {
return disabled === '1' || disabled === 'true' ? [] : ['--log-file', getDaemonLogFilePath()]
}
/** Named so a caller clamping this probe to a deadline cannot silently decouple from its default. */
export const DAEMON_SOCKET_PROBE_TIMEOUT_MS = 1_000
// Why: a socket that accepts a connection proves a daemon survived a previous app session and can be reused.
export function probeDaemonSocket(socketPath: string): Promise<boolean> {
export function probeDaemonSocket(
socketPath: string,
timeoutMs = DAEMON_SOCKET_PROBE_TIMEOUT_MS
): Promise<boolean> {
const { promise, resolve } = Promise.withResolvers<boolean>()
if (process.platform !== 'win32' && !existsSync(socketPath)) {
resolve(false)
@@ -69,21 +77,31 @@ export function probeDaemonSocket(socketPath: string): Promise<boolean> {
}
const onConnect = (): void => finish(true, true)
const onError = (): void => finish(false)
timer = setTimeout(() => finish(false, true), 1000)
timer = setTimeout(() => finish(false, true), timeoutMs)
socket.on('connect', onConnect)
socket.on('error', onError)
return promise
}
// Why recoveryDeadlineMs is required: this probe only ever runs on a startup path that has a
// budget, and the client's own defaults are far larger than any of them.
export async function getAliveDaemonSessionCount(
socketPath: string,
tokenPath: string,
recoveryDeadlineMs: number,
protocolVersion = PROTOCOL_VERSION
): Promise<number | null> {
const client = new DaemonClient({ socketPath, tokenPath, protocolVersion })
// Why one slice for both: a wedged handshake must not leave the request its own fresh 30s.
const probeTimeoutMs = daemonRecoveryProbeTimeoutMs(recoveryDeadlineMs)
const probeDeadlineMs = Date.now() + probeTimeoutMs
try {
await client.ensureConnected()
const result = await client.request<ListSessionsResult>('listSessions', undefined)
await client.ensureConnectedWithin(probeTimeoutMs)
const result = await client.request<ListSessionsResult>(
'listSessions',
undefined,
remainingDaemonRequestTimeoutMs(probeDeadlineMs)
)
return result.sessions.filter((session) => session.isAlive).length
} catch {
return null
@@ -14,6 +14,7 @@ import {
} from './daemon-launched-child'
import { getDaemonEntryPath, probeDaemonSocket as probeSocket } from './daemon-launch-paths'
import { materializeRelocatedDaemonHost } from './daemon-host-relocation'
import { DAEMON_RECOVERY_BUDGET_MS, daemonRecoveryProbeTimeoutMs } from './daemon-recovery-budget'
import { cleanupDaemonForProtocol } from './daemon-protocol-cleanup'
import {
getDaemonPidPath,
@@ -56,6 +57,9 @@ export function createOutOfProcessLauncher(
): DaemonLauncher {
return async (socketPath, tokenPath, suppliedPidPath, suppliedLaunchNonce) => {
const entryPath = getDaemonEntryPath()
// Why here: everything up to the fork is one recovery, so the adoption connect and the
// preflight's probes share a single absolute budget rather than each carrying its own.
const recoveryDeadlineMs = Date.now() + DAEMON_RECOVERY_BUDGET_MS
const pidPath = suppliedPidPath ?? getDaemonPidPath(runtimeDir)
const launchNonce = suppliedLaunchNonce ?? randomUUID()
// One-shot: whichever launch consumes it owns the attribution, so a later unrelated launch can't
@@ -69,7 +73,9 @@ export function createOutOfProcessLauncher(
})
try {
// Why: acquire the full pair before control-only probes so an expired inherited deadline can't fire in the probe-to-adoption gap.
await adoptionClient.ensureConnected()
// Why bounded: unbudgeted this grants a fresh 5s to each of four connect/hello steps, so a
// wedged endpoint burns more before recovery starts than recovery itself is allowed.
await adoptionClient.ensureConnectedWithin(daemonRecoveryProbeTimeoutMs(recoveryDeadlineMs))
await reconcileDaemonPidOwnership(adoptionClient, pidPath)
} catch {
adoptionClient.disconnect()
@@ -99,6 +105,7 @@ export function createOutOfProcessLauncher(
socketPath,
tokenPath,
entryPath,
recoveryDeadlineMs,
attributedReason,
releaseAdoptionClient,
preserveDaemon
@@ -193,6 +200,10 @@ export function createOutOfProcessLauncher(
// giving up here costs the user every persistent session for the whole run. Something
// answering the endpoint now is a daemon worth adopting, not a reason to fall back to
// local PTYs.
// Why unbudgeted: the recovery deadline bounds the adopt-or-replace decision, and this runs
// after it — past the kill, the fork and the lease. Clamping to the remainder yields a 1ms
// probe that loses to its own timer against a live socket, turning the rescue into the total
// daemon loss it exists to prevent.
if (await probeSocket(socketPath)) {
console.warn(
'[daemon] DEGRADED MODE: adopting the daemon that owns the endpoint after a replacement could not publish onto it. Existing sessions keep working; fresh terminals run on the local provider WITHOUT daemon persistence until you restart the daemon (Manage Sessions → Restart).'
+56
View File
@@ -0,0 +1,56 @@
/**
* How long a transient wedge takes to drain — the Windows update-relaunch AV/disk-pressure shape
* #8697 measured. The grace exists to adopt that daemon *with* its live sessions instead of
* killing them, so the recovery budget has to outlast this.
*
* Note this is the drain estimate, not the grace #8697 shipped: its merged second commit
* (840d3277d1d) widened WEDGED_DAEMON_GRACE_RETRIES 3 -> 11 (~20s -> ~60s) precisely to keep
* live-session loss near zero on that path. DAEMON_RECOVERY_BUDGET_MS below deliberately sits
* under that ~60s, because the unbounded version blew the startup PTY gate and lost the sessions
* anyway (STA-5732) — so a wedge draining between the budget and ~60s is now replaced where
* #8697 would have adopted it. That trade is pinned in daemon-init-wedged-daemon-grace.test.ts.
*/
export const TRANSIENT_WEDGE_DRAIN_MS = 20_000
/**
* What one connect (+ listSessions) attempt may spend. Deliberately more generous than the 3s
* daemon health check: this probe is the second opinion on that check's verdict, and a machine
* loaded enough to time the check out on a live daemon would time out a probe held to the same
* bar too — turning "could not verify" into "dead". Left to its own defaults the client instead
* grants a fresh 5s to each of four connect/hello steps plus 30s to the request, so one probe of
* a wedged daemon could outlast the entire recovery.
*/
export const DAEMON_RECOVERY_PROBE_MS = 8_000
/**
* One absolute wall-clock budget for adopting-or-replacing whatever daemon already owns the
* endpoint at startup. Every caller of the out-of-process launcher shares it — the desktop
* startup gate, orcad, and user-initiated restart — because a wedged endpoint costs the same
* wherever it is met.
*
* Both bounds matter and daemon-init-wedged-daemon-grace.test.ts pins them. Above
* TRANSIENT_WEDGE_DRAIN_MS, with room for the probe in flight when the daemon finally answers to
* finish its hello + listSessions rather than expire on the deadline. Below what the startup PTY
* gate can still absorb, since the kill, fork and lease that follow run inside the same fail-open
* cap: grace used to be a probe count with no clock at all, so it ran past that cap and the
* sessions were lost anyway, after the user watched the app hang (STA-5732).
*
* Sized against that cap rather than guessed, because every second not spent here is a second a
* daemon that would have drained gets killed instead. What still has to fit after the deadline,
* at each stage's own hard cap: killStaleDaemon 10.5s (two identity inspections at 3s + the 3s
* SIGTERM wait + the 1s SIGKILL confirm + the 0.5s endpoint probe), the fork's 10s readiness
* timeout, and 5s for the adoption lease and adapter connects — those two run against a daemon
* that has just reported ready over IPC, so they get a realistic allowance, not the client's
* unbudgeted 4x5s. 60 - 25.5 leaves 34.5s; take 32s and keep the rest as margin.
*
* Outside that accounting by design: the launcher's outer-catch endpoint rescue, which only
* runs once the replacement has already failed. Clamping it to the remainder is what turned a
* recoverable degraded adoption into total daemon loss, so it keeps its own probe default and
* the gate may fail open ahead of it — no worse than not rescuing, and better whenever it wins.
*/
export const DAEMON_RECOVERY_BUDGET_MS = 32_000
/** One attempt's share of the recovery budget, never reaching past the deadline. */
export function daemonRecoveryProbeTimeoutMs(recoveryDeadlineMs: number): number {
return Math.max(1, Math.min(DAEMON_RECOVERY_PROBE_MS, recoveryDeadlineMs - Date.now()))
}
@@ -3,7 +3,11 @@ import type { DaemonReplaceReason } from '../../shared/daemon-lifecycle-telemetr
import { isDaemonStaleForCurrentBundle } from './daemon-bundle-staleness'
import { DaemonEndpointOwnershipError } from './daemon-endpoint-adoption'
import { checkDaemonHealth, getMacDaemonSystemResolverHealth } from './daemon-health'
import { getAliveDaemonSessionCount, probeDaemonSocket as probeSocket } from './daemon-launch-paths'
import {
DAEMON_SOCKET_PROBE_TIMEOUT_MS,
getAliveDaemonSessionCount,
probeDaemonSocket as probeSocket
} from './daemon-launch-paths'
import { trackDaemonReplaced } from './daemon-lifecycle-event'
import { getDaemonLaunchIdentity } from './daemon-pid-identity'
import { cleanupDaemonForProtocol } from './daemon-protocol-cleanup'
@@ -12,7 +16,9 @@ import { killStaleDaemon } from './daemon-stale-kill'
import { getMacDaemonTccAttributionHealth } from './daemon-tcc-attribution'
import { PROTOCOL_VERSION } from './types'
// Why: extra hello+listSessions probes (~5s each) giving a wedged-but-connectable daemon ~60s grace to answer and keep its live sessions before a permanent wedge (#8689) is replaced; raise only alongside the fail-open cap.
// Why a count on top of the wall clock: DAEMON_RECOVERY_BUDGET_MS ends the grace, but a socket
// that accepts and then resets the hello answers both probes instantly, so without this the loop
// would spin hot for the whole budget.
export const WEDGED_DAEMON_GRACE_RETRIES = 11
type PreserveDaemon = (mode?: 'degraded-new-pty-fallback') => Promise<DaemonProcessHandle>
@@ -22,6 +28,8 @@ type ReplacementPreflightOptions = {
socketPath: string
tokenPath: string
entryPath: string
/** Absolute deadline for the whole adopt-or-replace decision; see DAEMON_RECOVERY_BUDGET_MS. */
recoveryDeadlineMs: number
attributedReason: DaemonReplaceReason | null
releaseAdoptionClient: () => void
preserveDaemon: PreserveDaemon
@@ -35,6 +43,7 @@ export async function prepareDaemonReplacement(
socketPath,
tokenPath,
entryPath,
recoveryDeadlineMs,
attributedReason,
releaseAdoptionClient,
preserveDaemon
@@ -50,7 +59,11 @@ export async function prepareDaemonReplacement(
if (health === 'healthy') {
const resolverHealth = await getMacDaemonSystemResolverHealth(socketPath, tokenPath)
if (resolverHealth === 'unhealthy') {
const liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
const liveSessionCount = await getAliveDaemonSessionCount(
socketPath,
tokenPath,
recoveryDeadlineMs
)
if (liveSessionCount !== 0) {
console.warn(
liveSessionCount === null
@@ -81,7 +94,14 @@ export async function prepareDaemonReplacement(
const replacementLabel = stalePackagedBundle
? 'launched before the current app bundle was installed'
: 'launched from a different app path'
if (await shouldPreserveDaemonWithLiveSessions(socketPath, tokenPath, replacementLabel)) {
if (
await shouldPreserveDaemonWithLiveSessions(
socketPath,
tokenPath,
recoveryDeadlineMs,
replacementLabel
)
) {
return preserveDaemon()
}
console.warn(
@@ -105,7 +125,11 @@ export async function prepareDaemonReplacement(
if (attributionHealth === 'severed') {
// Why: replacing with live sessions would kill them; Settings → Developer
// Permissions surfaces the Manage Sessions → Restart remedy instead.
const liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
const liveSessionCount = await getAliveDaemonSessionCount(
socketPath,
tokenPath,
recoveryDeadlineMs
)
if (liveSessionCount === 0) {
console.warn(
'[daemon] Replacing daemon whose macOS TCC attribution is severed (spawning app binary no longer exists)'
@@ -124,16 +148,26 @@ export async function prepareDaemonReplacement(
}
} else {
// Why: a busy machine can time out the health check on a live daemon; re-verify with a session list before killing its sessions.
let liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
let liveSessionCount = await getAliveDaemonSessionCount(
socketPath,
tokenPath,
recoveryDeadlineMs
)
// Why: a wedged-but-connectable daemon (Windows update relaunch) may still own live sessions, so grace-retry before replacing; a permanent wedge (#8689) exhausts the grace, and 'rejected' skips it (handshake refused = never adoptable).
// Why the clock term: without it the grace is however long the probes happen to take, which
// ran past the startup PTY gate's fail-open cap and hung terminal restore (STA-5732).
let graceRetry = 0
while (
liveSessionCount === null &&
health !== 'rejected' &&
graceRetry < WEDGED_DAEMON_GRACE_RETRIES &&
(await probeSocket(socketPath))
Date.now() < recoveryDeadlineMs &&
(await probeSocket(
socketPath,
Math.max(1, Math.min(DAEMON_SOCKET_PROBE_TIMEOUT_MS, recoveryDeadlineMs - Date.now()))
))
) {
liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath, recoveryDeadlineMs)
graceRetry++
}
if (liveSessionCount !== null && liveSessionCount > 0) {
@@ -213,9 +247,14 @@ export async function prepareDaemonReplacement(
async function shouldPreserveDaemonWithLiveSessions(
socketPath: string,
tokenPath: string,
recoveryDeadlineMs: number,
replacementLabel: string
): Promise<boolean> {
const liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
const liveSessionCount = await getAliveDaemonSessionCount(
socketPath,
tokenPath,
recoveryDeadlineMs
)
if (liveSessionCount === 0) {
return false
}