mirror of
https://github.com/stablyai/orca.git
synced 2026-10-22 00:03:47 +00:00
* fix(daemon): replace a permanently wedged daemon instead of preserving it forever (#8689) A daemon whose socket accepts connections but whose event loop never answers the 'hello' handshake was adopted by the launcher unconditionally and never re-evaluated, so every terminal spawn failed with 'DaemonProtocolError: Hello response timed out' with no recovery. - daemon-init.ts: bound the launcher's 'preserve any unresponsive-but-connectable daemon' with a grace window. A transient wedge (Windows update-relaunch AV/disk pressure) drains within ~20s and is preserved WITH its live sessions; a permanent wedge exhausts the grace and is replaced. Stays well under the 60s local-PTY fail-open cap. - daemon-pty-adapter.ts: isDaemonGoneError now treats 'Hello response timed out' as daemon-gone, so a runtime wedge triggers withDaemonRetry's respawn (re-entering the same grace-bounded launcher) instead of failing every spawn until app restart. Tests pin the grace magnitude so it cannot be silently shrunk. Co-authored-by: Orca <help@stably.ai> * fix(daemon): widen wedged-daemon grace window to ~60s Bump WEDGED_DAEMON_GRACE_RETRIES 3 -> 11 (~20s -> ~60s) to keep live-session loss on the transient-wedge (Windows update-relaunch) path as close to zero as possible. A transient wedge drains early and stays under the 60s fail-open cap; only a permanent wedge runs the full window. Export the constant and pin its floor in tests so it can't be silently shrunk. Co-authored-by: Orca <help@stably.ai> --------- Co-authored-by: Orca <help@stably.ai>
144 lines
5.6 KiB
TypeScript
144 lines
5.6 KiB
TypeScript
// Regression coverage for issue #6814 (terminal lockup after upgrade).
|
|
//
|
|
// Drives the real DaemonServer + checkDaemonHealth client over a real unix
|
|
// socket to lock in how each post-upgrade daemon failure mode is classified,
|
|
// and therefore which ones the degraded-daemon fallback actually rescues:
|
|
// - degraded (answers protocol, cannot spawn) -> pty-spawn-unhealthy -> rescued
|
|
// - wedged (event loop hung, health RPC never returns) -> unreachable -> NOT
|
|
// rescued by the degraded provider; falls to the preserve-or-replace path.
|
|
// This boundary is load-bearing: the degraded fallback only helps a daemon that
|
|
// still answers protocol, so a regression that widened it to wedged daemons
|
|
// would silently strand fresh terminals on a daemon that cannot serve them.
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
|
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
import { createServer } from 'node:net'
|
|
import { tmpdir } from 'node:os'
|
|
import { basename, join } from 'node:path'
|
|
import { DaemonServer } from './daemon-server'
|
|
import { checkDaemonHealth } from './daemon-health'
|
|
import type { SubprocessHandle } from './session'
|
|
|
|
function createMockSubprocess(): SubprocessHandle {
|
|
return {
|
|
pid: 55555,
|
|
getForegroundProcess: () => null,
|
|
write() {},
|
|
resize() {},
|
|
kill() {},
|
|
forceKill() {},
|
|
signal() {},
|
|
onData() {},
|
|
onExit() {},
|
|
dispose() {}
|
|
}
|
|
}
|
|
|
|
function daemonTestSocketPath(dir: string): string {
|
|
return process.platform === 'win32'
|
|
? `\\\\.\\pipe\\${basename(dir)}-daemon.sock`
|
|
: join(dir, 'daemon.sock')
|
|
}
|
|
|
|
describe('issue #6814 repro: daemon failure-mode classification', () => {
|
|
let dir: string
|
|
let socketPath: string
|
|
let tokenPath: string
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(tmpdir(), 'issue-6814-repro-'))
|
|
socketPath = daemonTestSocketPath(dir)
|
|
tokenPath = join(dir, 'daemon.token')
|
|
})
|
|
|
|
afterEach(() => {
|
|
rmSync(dir, { recursive: true, force: true })
|
|
})
|
|
|
|
// The good case: daemon answers hello AND the PTY spawn probe succeeds.
|
|
it('HEALTHY: a daemon that can spawn PTYs classifies as healthy', async () => {
|
|
const server = new DaemonServer({
|
|
socketPath,
|
|
tokenPath,
|
|
ptySpawnHealthCheck: vi.fn(async () => {}),
|
|
spawnSubprocess: () => createMockSubprocess()
|
|
})
|
|
await server.start()
|
|
try {
|
|
await expect(checkDaemonHealth(socketPath, tokenPath)).resolves.toBe('healthy')
|
|
} finally {
|
|
await server.shutdown()
|
|
}
|
|
})
|
|
|
|
// Symptom B, degraded: this is the case #6830 RESCUES. The daemon answers
|
|
// protocol but its PTY spawn probe throws (deleted cwd / stale native PTY
|
|
// after an upgrade), so fresh terminals would open frozen with no cursor.
|
|
it('DEGRADED: protocol-alive daemon that cannot spawn PTYs classifies as pty-spawn-unhealthy', async () => {
|
|
const server = new DaemonServer({
|
|
socketPath,
|
|
tokenPath,
|
|
ptySpawnHealthCheck: vi.fn(async () => {
|
|
throw new Error('chdir(2) failed.: No such file or directory')
|
|
}),
|
|
spawnSubprocess: () => createMockSubprocess()
|
|
})
|
|
await server.start()
|
|
try {
|
|
// -> #6830 marks this daemon degraded and routes fresh spawns to the
|
|
// local provider instead of the no-cursor daemon pane.
|
|
await expect(checkDaemonHealth(socketPath, tokenPath)).resolves.toBe('pty-spawn-unhealthy')
|
|
} finally {
|
|
await server.shutdown()
|
|
}
|
|
})
|
|
|
|
// The limit of #6830: a fully WEDGED daemon (event loop hung — health RPC
|
|
// never returns) cannot be distinguished by a richer status. It times out
|
|
// and classifies as 'unreachable', the SAME bucket as a dead daemon.
|
|
it('WEDGED: a daemon whose health RPC never resolves classifies as unreachable (NOT degraded)', async () => {
|
|
const server = new DaemonServer({
|
|
socketPath,
|
|
tokenPath,
|
|
// Why: simulate a hung event loop — the probe never settles.
|
|
ptySpawnHealthCheck: vi.fn(() => new Promise<void>(() => {})),
|
|
spawnSubprocess: () => createMockSubprocess()
|
|
})
|
|
await server.start()
|
|
try {
|
|
const health = await checkDaemonHealth(socketPath, tokenPath)
|
|
// This is the key finding: wedged != degraded. #6830's degraded fallback
|
|
// does NOT engage here; recovery still depends on the unreachable-path
|
|
// (grace-bounded preserve-if-live-else-replace) logic, not the degraded
|
|
// provider.
|
|
expect(health).toBe('unreachable')
|
|
} finally {
|
|
await server.shutdown()
|
|
}
|
|
}, 15000)
|
|
|
|
// #8689: a daemon whose socket accepts connections but never answers the
|
|
// hello handshake (event loop wedged before the protocol reply) also
|
|
// classifies as 'unreachable' — the launcher's grace-bounded path must
|
|
// eventually replace it rather than preserve it forever.
|
|
it('WEDGED-HELLO: a daemon that accepts connections but never answers hello classifies as unreachable (#8689)', async () => {
|
|
const wedged = createServer((sock) => {
|
|
// Swallow the hello bytes but never write a response — the exact wedge.
|
|
sock.on('data', () => {})
|
|
})
|
|
await new Promise<void>((resolve) => wedged.listen(socketPath, () => resolve()))
|
|
// The health check reads the token before connecting, so it must exist.
|
|
writeFileSync(tokenPath, 'wedged-token')
|
|
try {
|
|
const health = await checkDaemonHealth(socketPath, tokenPath)
|
|
expect(health).toBe('unreachable')
|
|
} finally {
|
|
await new Promise<void>((resolve) => wedged.close(() => resolve()))
|
|
}
|
|
}, 15000)
|
|
|
|
// No daemon at all (or token missing) -> unreachable.
|
|
it('UNREACHABLE: no daemon listening classifies as unreachable', async () => {
|
|
await expect(checkDaemonHealth(socketPath, tokenPath)).resolves.toBe('unreachable')
|
|
})
|
|
})
|