mirror of
https://github.com/stablyai/orca.git
synced 2026-10-19 08:03:13 +00:00
fix(daemon): retire macOS daemons whose login session died (#7936)
A daemon that survives a full macOS logout is unsalvageable: its PAM context can no longer host login(1) spawns (every new PTY becomes a 'Login incorrect' prompt zombie) and its Mach bootstrap namespace has lost the system DNS resolver, so terminals it hosts have no egress. Today it also keeps the #9301 preflight's cached 'accepted' verdict, so it keeps wrapping spawns in login(1) forever; only a manual daemon restart recovers. GUI-spawned daemons now watch for login-session death from the inside: a fresh cache-bypassing PAM probe (triggered by PTY-exit bursts, fresh client hellos, and a slow periodic timer) must conclusively reject three consecutive times AND the in-process system resolver must be degraded; then the daemon exits crash-style so session meta stays unclean and the replacement daemon cold-restores scrollback. A conclusive rejection also flips the spawn-wrapper cache off immediately. Headless serve/SSH daemons never get the watch (they must survive their spawning session ending), and a session that never conclusively accepted login(1) never arms it — a PAM anomaly alone can't kill a healthy daemon (fast user switching keeps accepting, so switched-away sessions are preserved).
This commit is contained in:
@@ -97,4 +97,17 @@ describe('daemon-entry parseArgs', () => {
|
||||
it('still requires --socket and --token when --log-file is given', () => {
|
||||
expect(() => parseArgs(['--log-file', '/tmp/daemon.log'])).toThrow('Usage:')
|
||||
})
|
||||
|
||||
it('parses the GUI-only --login-session-watch flag and omits it when absent', () => {
|
||||
expect(
|
||||
parseArgs(['--socket', '/tmp/t.sock', '--token', '/tmp/t.token', '--login-session-watch'])
|
||||
).toEqual({
|
||||
socketPath: '/tmp/t.sock',
|
||||
tokenPath: '/tmp/t.token',
|
||||
loginSessionWatch: true
|
||||
})
|
||||
expect(parseArgs(['--socket', '/tmp/t.sock', '--token', '/tmp/t.token'])).not.toHaveProperty(
|
||||
'loginSessionWatch'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -12,13 +12,20 @@ import { warmWindowsConptyOnce } from './windows-conpty-warmup'
|
||||
import { warmPwshAvailabilityCache } from '../pwsh'
|
||||
import { createDaemonFileLog, createNoopDaemonFileLog } from './daemon-file-log'
|
||||
import { PROTOCOL_VERSION } from './types'
|
||||
import { prepareMacosTccLoginShell } from '../providers/macos-tcc-login-shell'
|
||||
import {
|
||||
prepareMacosTccLoginShell,
|
||||
probeMacosLoginSessionAlive
|
||||
} from '../providers/macos-tcc-login-shell'
|
||||
import { MacosLoginSessionDeathWatch } from './macos-login-session-death-watch'
|
||||
import { readCurrentProcessMacSystemResolverHealth } from '../network/macos-system-resolver-health'
|
||||
|
||||
export type ParsedDaemonArgs = {
|
||||
socketPath: string
|
||||
tokenPath: string
|
||||
pidPath?: string
|
||||
launchNonce?: string
|
||||
/** GUI-spawned daemons only — headless serve/SSH daemons must survive session loss. */
|
||||
loginSessionWatch?: boolean
|
||||
/** Optional — absent for adopted old daemons and tests, which log nothing. */
|
||||
logFilePath?: string
|
||||
}
|
||||
@@ -29,6 +36,7 @@ export function parseArgs(argv: string[]): ParsedDaemonArgs {
|
||||
let logFilePath = ''
|
||||
let pidPath = ''
|
||||
let launchNonce = ''
|
||||
let loginSessionWatch = false
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--socket' && argv[i + 1]) {
|
||||
@@ -46,6 +54,8 @@ export function parseArgs(argv: string[]): ParsedDaemonArgs {
|
||||
} else if (argv[i] === '--launch-nonce' && argv[i + 1]) {
|
||||
launchNonce = argv[i + 1]
|
||||
i++
|
||||
} else if (argv[i] === '--login-session-watch') {
|
||||
loginSessionWatch = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +71,7 @@ export function parseArgs(argv: string[]): ParsedDaemonArgs {
|
||||
socketPath,
|
||||
tokenPath,
|
||||
...(pidPath ? { pidPath, launchNonce } : {}),
|
||||
...(loginSessionWatch ? { loginSessionWatch } : {}),
|
||||
...(logFilePath ? { logFilePath } : {})
|
||||
}
|
||||
}
|
||||
@@ -73,7 +84,7 @@ async function main(): Promise<void> {
|
||||
// an otherwise healthy detached daemon. Swallow it: stderr is diagnostic only.
|
||||
process.stderr.on('error', () => {})
|
||||
|
||||
const { socketPath, tokenPath, pidPath, launchNonce, logFilePath } = parseArgs(
|
||||
const { socketPath, tokenPath, pidPath, launchNonce, loginSessionWatch, logFilePath } = parseArgs(
|
||||
process.argv.slice(2)
|
||||
)
|
||||
const startedAtMs = Date.now() - process.uptime() * 1000
|
||||
@@ -157,6 +168,24 @@ async function main(): Promise<void> {
|
||||
process.on('SIGTERM', () => void shutdown('SIGTERM'))
|
||||
process.on('SIGINT', () => void shutdown('SIGINT'))
|
||||
|
||||
const deathWatch =
|
||||
loginSessionWatch && process.platform === 'darwin'
|
||||
? new MacosLoginSessionDeathWatch({
|
||||
probeLoginSession: probeMacosLoginSessionAlive,
|
||||
readResolverHealth: readCurrentProcessMacSystemResolverHealth,
|
||||
log: daemonLog,
|
||||
onRetire: (details) => {
|
||||
shuttingDown = true
|
||||
daemonLog.log('login-session-dead-retire', details)
|
||||
daemonLog.close()
|
||||
// Why: crash-style exit (no PTY teardown) keeps session meta unclean so the
|
||||
// replacement daemon cold-restores scrollback; stale socket/pid files ride
|
||||
// the existing dead-endpoint recovery.
|
||||
process.exit(1)
|
||||
}
|
||||
})
|
||||
: null
|
||||
|
||||
daemon = await startDaemon({
|
||||
socketPath,
|
||||
tokenPath,
|
||||
@@ -165,6 +194,12 @@ async function main(): Promise<void> {
|
||||
...(pidPath ? { startedAtMs } : {}),
|
||||
log: daemonLog,
|
||||
preparePtySpawn: runMacosLoginPreflight,
|
||||
...(deathWatch
|
||||
? {
|
||||
onPtySessionExit: () => deathWatch.notifyPtyExit(),
|
||||
onClientHello: () => deathWatch.notifyClientActivity()
|
||||
}
|
||||
: {}),
|
||||
spawnSubprocess: (opts) => createPtySubprocess(opts),
|
||||
onIdleShutdown: () => {
|
||||
shuttingDown = true
|
||||
@@ -173,6 +208,7 @@ async function main(): Promise<void> {
|
||||
process.exit(0)
|
||||
}
|
||||
})
|
||||
deathWatch?.start()
|
||||
|
||||
// Signal readiness to parent via IPC (if available)
|
||||
if (process.send) {
|
||||
|
||||
@@ -321,7 +321,10 @@ async function shouldPreserveDaemonWithLiveSessions(
|
||||
return true
|
||||
}
|
||||
|
||||
function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher {
|
||||
function createOutOfProcessLauncher(
|
||||
runtimeDir: string,
|
||||
macosLoginSessionWatch = false
|
||||
): DaemonLauncher {
|
||||
return async (socketPath, tokenPath, suppliedPidPath, suppliedLaunchNonce) => {
|
||||
const entryPath = getDaemonEntryPath()
|
||||
const pidPath = suppliedPidPath ?? getDaemonPidPath(runtimeDir)
|
||||
@@ -448,6 +451,7 @@ function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher {
|
||||
pidPath,
|
||||
'--launch-nonce',
|
||||
launchNonce,
|
||||
...(macosLoginSessionWatch ? ['--login-session-watch'] : []),
|
||||
...daemonLogArgs()
|
||||
],
|
||||
{
|
||||
@@ -625,7 +629,10 @@ function createOutOfProcessLauncher(runtimeDir: string): DaemonLauncher {
|
||||
}
|
||||
}
|
||||
|
||||
export async function initDaemonPtyProvider(signal?: AbortSignal): Promise<void> {
|
||||
export async function initDaemonPtyProvider(
|
||||
signal?: AbortSignal,
|
||||
options: { macosLoginSessionWatch?: boolean } = {}
|
||||
): Promise<void> {
|
||||
logDaemonMilestone('daemon-init-start')
|
||||
// Why: e2e coverage for the startup PTY gate (#5232) needs a daemon init that deterministically outlasts the first-window timeout.
|
||||
const e2eInitDelayMs = Number(process.env.ORCA_E2E_DAEMON_INIT_DELAY_MS)
|
||||
@@ -636,7 +643,7 @@ export async function initDaemonPtyProvider(signal?: AbortSignal): Promise<void>
|
||||
|
||||
const newSpawner = new DaemonSpawner({
|
||||
runtimeDir,
|
||||
launcher: createOutOfProcessLauncher(runtimeDir)
|
||||
launcher: createOutOfProcessLauncher(runtimeDir, options.macosLoginSessionWatch ?? false)
|
||||
})
|
||||
|
||||
// Why: assign the module-level spawner/adapter only after both succeed, so a failed ensureRunning() leaves no stale spawner.
|
||||
|
||||
@@ -11,6 +11,8 @@ export type DaemonStartOptions = {
|
||||
protocolVersion?: number
|
||||
spawnSubprocess: DaemonServerOptions['spawnSubprocess']
|
||||
preparePtySpawn?: DaemonServerOptions['preparePtySpawn']
|
||||
onPtySessionExit?: DaemonServerOptions['onPtySessionExit']
|
||||
onClientHello?: DaemonServerOptions['onClientHello']
|
||||
log?: DaemonFileLog
|
||||
onIdleShutdown?: () => void
|
||||
initialAdoptionTestConfig?: DaemonServerOptions['initialAdoptionTestConfig']
|
||||
@@ -30,6 +32,8 @@ export async function startDaemon(opts: DaemonStartOptions): Promise<DaemonHandl
|
||||
...(opts.protocolVersion !== undefined ? { protocolVersion: opts.protocolVersion } : {}),
|
||||
spawnSubprocess: opts.spawnSubprocess,
|
||||
...(opts.preparePtySpawn ? { preparePtySpawn: opts.preparePtySpawn } : {}),
|
||||
...(opts.onPtySessionExit ? { onPtySessionExit: opts.onPtySessionExit } : {}),
|
||||
...(opts.onClientHello ? { onClientHello: opts.onClientHello } : {}),
|
||||
...(opts.log ? { log: opts.log } : {}),
|
||||
...(opts.onIdleShutdown ? { onIdleShutdown: opts.onIdleShutdown } : {}),
|
||||
...(opts.initialAdoptionTestConfig
|
||||
|
||||
@@ -53,6 +53,9 @@ export type DaemonServerOptions = {
|
||||
}
|
||||
ptySpawnHealthCheck?: () => Promise<void>
|
||||
preparePtySpawn?: () => Promise<void>
|
||||
// Why: login-session death detection (#7936) probes on PTY-exit bursts and fresh client hellos.
|
||||
onPtySessionExit?: (sessionId: string) => void
|
||||
onClientHello?: () => void
|
||||
log?: DaemonFileLog
|
||||
spawnSubprocess: (opts: {
|
||||
sessionId: string
|
||||
@@ -97,6 +100,7 @@ export class DaemonServer {
|
||||
private startedAtMs: number | null
|
||||
private protocolVersion: number
|
||||
private onIdleShutdown: () => void
|
||||
private onClientHello: () => void
|
||||
private ptySpawnHealthCheck: () => Promise<void>
|
||||
private preparePtySpawn: () => Promise<void>
|
||||
private log: DaemonFileLog
|
||||
@@ -176,7 +180,11 @@ export class DaemonServer {
|
||||
now: () => Date.now()
|
||||
}
|
||||
this.token = randomUUID()
|
||||
this.host = new TerminalHost({ spawnSubprocess: opts.spawnSubprocess })
|
||||
this.onClientHello = opts.onClientHello ?? (() => {})
|
||||
this.host = new TerminalHost({
|
||||
spawnSubprocess: opts.spawnSubprocess,
|
||||
...(opts.onPtySessionExit ? { onSessionReaped: opts.onPtySessionExit } : {})
|
||||
})
|
||||
this.ptySpawnHealthCheck = opts.ptySpawnHealthCheck ?? checkPtySpawnHealth
|
||||
this.preparePtySpawn = opts.preparePtySpawn ?? (() => Promise.resolve())
|
||||
this.stopStreamBacklogProbe = startDaemonStreamBacklogProbe(() => ({
|
||||
@@ -434,6 +442,7 @@ export class DaemonServer {
|
||||
}
|
||||
|
||||
this.log.log('client-hello-accepted', { role: hello.role, clientId: hello.clientId })
|
||||
this.onClientHello()
|
||||
socket.write(
|
||||
encodeNdjson({
|
||||
type: 'hello',
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { LoginPreflightOutcome } from '../providers/macos-tcc-login-shell'
|
||||
import { createNoopDaemonFileLog } from './daemon-file-log'
|
||||
import {
|
||||
MacosLoginSessionDeathWatch,
|
||||
type MacosLoginSessionDeathWatchOptions
|
||||
} from './macos-login-session-death-watch'
|
||||
import type { SystemResolverHealth } from './types'
|
||||
|
||||
const ACCEPTED: LoginPreflightOutcome = { ok: true, conclusive: true, reason: 'accepted' }
|
||||
const REJECTED: LoginPreflightOutcome = { ok: false, conclusive: true, reason: 'rejected' }
|
||||
const INCONCLUSIVE: LoginPreflightOutcome = { ok: false, conclusive: false, reason: 'timeout' }
|
||||
|
||||
type FakeTimer = { at: number; callback: () => void; cleared: boolean }
|
||||
|
||||
class FakeClock {
|
||||
private timers: FakeTimer[] = []
|
||||
private nowMs = 0
|
||||
|
||||
setTimeout = (callback: () => void, delayMs: number): unknown => {
|
||||
const timer: FakeTimer = { at: this.nowMs + delayMs, callback, cleared: false }
|
||||
this.timers.push(timer)
|
||||
return timer
|
||||
}
|
||||
|
||||
clearTimeout = (handle: unknown): void => {
|
||||
;(handle as FakeTimer).cleared = true
|
||||
}
|
||||
|
||||
now = (): number => this.nowMs
|
||||
|
||||
async advance(ms: number): Promise<void> {
|
||||
const target = this.nowMs + ms
|
||||
for (;;) {
|
||||
const due = this.timers
|
||||
.filter((t) => !t.cleared && t.at <= target)
|
||||
.sort((a, b) => a.at - b.at)[0]
|
||||
if (!due) {
|
||||
break
|
||||
}
|
||||
this.nowMs = due.at
|
||||
due.cleared = true
|
||||
due.callback()
|
||||
// Why: probes are async; let their promise chains settle before firing the next timer.
|
||||
await drainMicrotasks()
|
||||
}
|
||||
this.nowMs = target
|
||||
}
|
||||
|
||||
pendingCount(): number {
|
||||
return this.timers.filter((t) => !t.cleared).length
|
||||
}
|
||||
}
|
||||
|
||||
async function drainMicrotasks(): Promise<void> {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await Promise.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
function createWatch(
|
||||
overrides: Partial<MacosLoginSessionDeathWatchOptions> & {
|
||||
outcomes?: (LoginPreflightOutcome | null)[]
|
||||
} = {}
|
||||
): {
|
||||
watch: MacosLoginSessionDeathWatch
|
||||
clock: FakeClock
|
||||
onRetire: ReturnType<typeof vi.fn>
|
||||
probe: ReturnType<typeof vi.fn>
|
||||
setResolverHealth: (health: SystemResolverHealth) => void
|
||||
} {
|
||||
const clock = new FakeClock()
|
||||
const onRetire = vi.fn()
|
||||
const outcomes = overrides.outcomes ?? []
|
||||
// Why length-check, not `??`: an explicit null outcome (wrapper not applicable) must reach the watch.
|
||||
const probe = vi.fn(async () => (outcomes.length ? outcomes.shift()! : ACCEPTED))
|
||||
let resolverHealth: SystemResolverHealth = 'unhealthy'
|
||||
const watch = new MacosLoginSessionDeathWatch({
|
||||
probeLoginSession: overrides.probeLoginSession ?? probe,
|
||||
readResolverHealth: overrides.readResolverHealth ?? (async () => resolverHealth),
|
||||
onRetire: overrides.onRetire ?? onRetire,
|
||||
log: overrides.log ?? createNoopDaemonFileLog(),
|
||||
clock,
|
||||
timing: {
|
||||
periodicProbeMs: 120_000,
|
||||
rejectionRecheckMs: 10_000,
|
||||
ptyExitDebounceMs: 2_000,
|
||||
clientActivityMinGapMs: 30_000,
|
||||
minProbeGapMs: 5_000,
|
||||
...overrides.timing
|
||||
}
|
||||
})
|
||||
return {
|
||||
watch,
|
||||
clock,
|
||||
onRetire,
|
||||
probe,
|
||||
setResolverHealth: (health) => {
|
||||
resolverHealth = health
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('MacosLoginSessionDeathWatch', () => {
|
||||
it('retires after consecutive conclusive rejections once armed, with a degraded resolver', async () => {
|
||||
const { watch, clock, onRetire } = createWatch({
|
||||
outcomes: [ACCEPTED, REJECTED, REJECTED, REJECTED]
|
||||
})
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
expect(onRetire).not.toHaveBeenCalled()
|
||||
|
||||
await clock.advance(120_000) // periodic → rejection 1
|
||||
await clock.advance(10_000) // recheck → rejection 2
|
||||
expect(onRetire).not.toHaveBeenCalled()
|
||||
await clock.advance(10_000) // recheck → rejection 3 → retire
|
||||
expect(onRetire).toHaveBeenCalledWith({ rejections: 3, resolverHealth: 'unhealthy' })
|
||||
})
|
||||
|
||||
it('never retires when the session was never conclusively accepted', async () => {
|
||||
const { watch, clock, onRetire } = createWatch({
|
||||
outcomes: [REJECTED, REJECTED, REJECTED, REJECTED, REJECTED]
|
||||
})
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await clock.advance(120_000)
|
||||
}
|
||||
expect(onRetire).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('resets the rejection streak on a conclusive acceptance', async () => {
|
||||
const { watch, clock, onRetire } = createWatch({
|
||||
outcomes: [ACCEPTED, REJECTED, REJECTED, ACCEPTED, REJECTED, REJECTED]
|
||||
})
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
await clock.advance(120_000) // rejection 1
|
||||
await clock.advance(10_000) // rejection 2
|
||||
await clock.advance(10_000) // acceptance → reset
|
||||
await clock.advance(120_000) // rejection 1
|
||||
await clock.advance(10_000) // rejection 2
|
||||
expect(onRetire).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('ignores inconclusive probes for the rejection streak', async () => {
|
||||
const { watch, clock, onRetire } = createWatch({
|
||||
outcomes: [ACCEPTED, REJECTED, INCONCLUSIVE, REJECTED, INCONCLUSIVE, REJECTED]
|
||||
})
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
await clock.advance(120_000) // rejection 1
|
||||
await clock.advance(10_000) // inconclusive — streak holds at 1, periodic reschedule
|
||||
await clock.advance(120_000) // rejection 2
|
||||
await clock.advance(10_000) // inconclusive
|
||||
await clock.advance(120_000) // rejection 3 → retire
|
||||
expect(onRetire).toHaveBeenCalledTimes(1)
|
||||
expect(onRetire.mock.calls[0][0].rejections).toBe(3)
|
||||
})
|
||||
|
||||
it('suppresses retirement while the system resolver is healthy, then retires when it degrades', async () => {
|
||||
const { watch, clock, onRetire, setResolverHealth } = createWatch({
|
||||
outcomes: [ACCEPTED, REJECTED, REJECTED, REJECTED, REJECTED]
|
||||
})
|
||||
setResolverHealth('healthy')
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
await clock.advance(120_000)
|
||||
await clock.advance(10_000)
|
||||
await clock.advance(10_000) // threshold reached but resolver healthy → suppressed
|
||||
expect(onRetire).not.toHaveBeenCalled()
|
||||
setResolverHealth('unhealthy')
|
||||
await clock.advance(10_000) // re-verdict → retire
|
||||
expect(onRetire).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('debounces PTY-exit bursts into one probe', async () => {
|
||||
const { watch, clock, probe } = createWatch({ outcomes: [ACCEPTED, ACCEPTED] })
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
await clock.advance(60_000)
|
||||
const before = probe.mock.calls.length
|
||||
watch.notifyPtyExit()
|
||||
watch.notifyPtyExit()
|
||||
watch.notifyPtyExit()
|
||||
await clock.advance(2_000)
|
||||
expect(probe.mock.calls.length).toBe(before + 1)
|
||||
})
|
||||
|
||||
it('probes on client activity only after the min gap', async () => {
|
||||
const { watch, clock, probe } = createWatch({
|
||||
outcomes: [ACCEPTED, ACCEPTED, ACCEPTED]
|
||||
})
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
const after = probe.mock.calls.length
|
||||
watch.notifyClientActivity() // too soon after startup probe
|
||||
await drainMicrotasks()
|
||||
expect(probe.mock.calls.length).toBe(after)
|
||||
await clock.advance(30_000)
|
||||
watch.notifyClientActivity()
|
||||
await drainMicrotasks()
|
||||
expect(probe.mock.calls.length).toBe(after + 1)
|
||||
})
|
||||
|
||||
it('disables itself when the wrapper machinery does not apply', async () => {
|
||||
const { watch, clock, probe } = createWatch({ outcomes: [null] })
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
expect(probe).toHaveBeenCalledTimes(1)
|
||||
await clock.advance(600_000)
|
||||
watch.notifyPtyExit()
|
||||
watch.notifyClientActivity()
|
||||
await clock.advance(600_000)
|
||||
expect(probe).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('stop() cancels all pending timers', async () => {
|
||||
const { watch, clock } = createWatch({ outcomes: [ACCEPTED] })
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
watch.notifyPtyExit()
|
||||
watch.stop()
|
||||
expect(clock.pendingCount()).toBe(0)
|
||||
})
|
||||
|
||||
it('logs and reschedules when a probe throws instead of surfacing a rejection', async () => {
|
||||
const outcomes: (() => Promise<LoginPreflightOutcome>)[] = [
|
||||
async () => ACCEPTED,
|
||||
async () => {
|
||||
throw new Error('launchctl exploded')
|
||||
},
|
||||
async () => ACCEPTED
|
||||
]
|
||||
const probe = vi.fn(() => (outcomes.shift() ?? (async () => ACCEPTED))())
|
||||
const { watch, clock, onRetire } = createWatch({ probeLoginSession: probe })
|
||||
watch.start()
|
||||
await drainMicrotasks()
|
||||
await clock.advance(120_000) // throwing probe
|
||||
await clock.advance(120_000) // rescheduled probe still runs
|
||||
expect(probe).toHaveBeenCalledTimes(3)
|
||||
expect(onRetire).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,231 @@
|
||||
import type { LoginPreflightOutcome } from '../providers/macos-tcc-login-shell'
|
||||
import type { DaemonFileLog } from './daemon-file-log'
|
||||
import type { SystemResolverHealth } from './types'
|
||||
|
||||
// Why: three conclusive PAM verdicts spread over recheck intervals keep a transient
|
||||
// rejection storm (PAM db reload, OS update) from retiring a daemon whose login
|
||||
// session is still alive; a real logout rejects conclusively on every probe.
|
||||
const REQUIRED_CONSECUTIVE_REJECTIONS = 3
|
||||
const PERIODIC_PROBE_MS = 120_000
|
||||
const REJECTION_RECHECK_MS = 10_000
|
||||
const PTY_EXIT_DEBOUNCE_MS = 2_000
|
||||
// Why: a client hello right after login is the fastest death signal for a stale
|
||||
// daemon, but steady reconnects must not turn hellos into a PAM probe storm.
|
||||
const CLIENT_ACTIVITY_MIN_GAP_MS = 30_000
|
||||
const MIN_PROBE_GAP_MS = 5_000
|
||||
|
||||
type WatchClock = {
|
||||
setTimeout(callback: () => void, delayMs: number): unknown
|
||||
clearTimeout(handle: unknown): void
|
||||
now(): number
|
||||
}
|
||||
|
||||
export type MacosLoginSessionDeathWatchOptions = {
|
||||
/** Fresh PAM probe (cache-bypassing); null when the login wrapper doesn't apply on this host. */
|
||||
probeLoginSession: () => Promise<LoginPreflightOutcome | null>
|
||||
readResolverHealth: () => Promise<SystemResolverHealth>
|
||||
onRetire: (details: { rejections: number; resolverHealth: SystemResolverHealth }) => void
|
||||
log: DaemonFileLog
|
||||
/** Direct-construction seam for deterministic tests; production uses real timers. */
|
||||
clock?: WatchClock
|
||||
timing?: Partial<{
|
||||
periodicProbeMs: number
|
||||
rejectionRecheckMs: number
|
||||
ptyExitDebounceMs: number
|
||||
clientActivityMinGapMs: number
|
||||
minProbeGapMs: number
|
||||
}>
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects that the macOS GUI login session this daemon was born into has died
|
||||
* (full logout / WindowServer session teardown) and retires the daemon so the
|
||||
* next app start cold-starts a replacement inside the live session (#7936).
|
||||
*
|
||||
* A daemon in a dead login session is unsalvageable: its PAM context can no
|
||||
* longer host `login(1)` spawns ("Login incorrect" zombies) and its Mach
|
||||
* bootstrap namespace has lost the system DNS resolver, so every terminal it
|
||||
* hosts has no egress. Retirement is the only converging heal.
|
||||
*
|
||||
* The oracle is the existing TCC login-shell PAM probe: it conclusively accepts
|
||||
* while the session is valid — including fast-user-switched-away sessions — and
|
||||
* conclusively rejects once the session is destroyed. Retirement additionally
|
||||
* requires the in-process system resolver to be degraded, so a PAM anomaly
|
||||
* alone can never kill a healthy daemon.
|
||||
*/
|
||||
export class MacosLoginSessionDeathWatch {
|
||||
private readonly probeLoginSession: MacosLoginSessionDeathWatchOptions['probeLoginSession']
|
||||
private readonly readResolverHealth: MacosLoginSessionDeathWatchOptions['readResolverHealth']
|
||||
private readonly onRetire: MacosLoginSessionDeathWatchOptions['onRetire']
|
||||
private readonly log: DaemonFileLog
|
||||
private readonly clock: WatchClock
|
||||
private readonly periodicProbeMs: number
|
||||
private readonly rejectionRecheckMs: number
|
||||
private readonly ptyExitDebounceMs: number
|
||||
private readonly clientActivityMinGapMs: number
|
||||
private readonly minProbeGapMs: number
|
||||
|
||||
// Why: retire only a daemon that once proved its session could host login(1);
|
||||
// a host where the wrapper never worked has no death signal to trust.
|
||||
private armed = false
|
||||
private consecutiveRejections = 0
|
||||
private lastProbeStartedAtMs: number | null = null
|
||||
private probeInFlight = false
|
||||
private stopped = false
|
||||
private retired = false
|
||||
private scheduledProbeTimer: unknown | null = null
|
||||
private ptyExitDebounceTimer: unknown | null = null
|
||||
|
||||
constructor(opts: MacosLoginSessionDeathWatchOptions) {
|
||||
this.probeLoginSession = opts.probeLoginSession
|
||||
this.readResolverHealth = opts.readResolverHealth
|
||||
this.onRetire = opts.onRetire
|
||||
this.log = opts.log
|
||||
this.clock = opts.clock ?? {
|
||||
setTimeout: (callback, delayMs) => {
|
||||
const timer = setTimeout(callback, delayMs)
|
||||
timer.unref()
|
||||
return timer
|
||||
},
|
||||
clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
||||
now: () => Date.now()
|
||||
}
|
||||
this.periodicProbeMs = opts.timing?.periodicProbeMs ?? PERIODIC_PROBE_MS
|
||||
this.rejectionRecheckMs = opts.timing?.rejectionRecheckMs ?? REJECTION_RECHECK_MS
|
||||
this.ptyExitDebounceMs = opts.timing?.ptyExitDebounceMs ?? PTY_EXIT_DEBOUNCE_MS
|
||||
this.clientActivityMinGapMs = opts.timing?.clientActivityMinGapMs ?? CLIENT_ACTIVITY_MIN_GAP_MS
|
||||
this.minProbeGapMs = opts.timing?.minProbeGapMs ?? MIN_PROBE_GAP_MS
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.stopped) {
|
||||
return
|
||||
}
|
||||
void this.runProbe('startup')
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true
|
||||
if (this.scheduledProbeTimer !== null) {
|
||||
this.clock.clearTimeout(this.scheduledProbeTimer)
|
||||
this.scheduledProbeTimer = null
|
||||
}
|
||||
if (this.ptyExitDebounceTimer !== null) {
|
||||
this.clock.clearTimeout(this.ptyExitDebounceTimer)
|
||||
this.ptyExitDebounceTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
/** A mass PTY-exit burst is what a logout's SIGHUP sweep looks like from inside the daemon. */
|
||||
notifyPtyExit(): void {
|
||||
if (this.stopped || this.ptyExitDebounceTimer !== null) {
|
||||
return
|
||||
}
|
||||
this.ptyExitDebounceTimer = this.clock.setTimeout(() => {
|
||||
this.ptyExitDebounceTimer = null
|
||||
void this.runProbe('pty-exit')
|
||||
}, this.ptyExitDebounceMs)
|
||||
}
|
||||
|
||||
notifyClientActivity(): void {
|
||||
if (this.stopped) {
|
||||
return
|
||||
}
|
||||
if (
|
||||
this.lastProbeStartedAtMs !== null &&
|
||||
this.clock.now() - this.lastProbeStartedAtMs < this.clientActivityMinGapMs
|
||||
) {
|
||||
return
|
||||
}
|
||||
void this.runProbe('client-hello')
|
||||
}
|
||||
|
||||
private scheduleNextProbe(delayMs: number): void {
|
||||
if (this.stopped) {
|
||||
return
|
||||
}
|
||||
if (this.scheduledProbeTimer !== null) {
|
||||
this.clock.clearTimeout(this.scheduledProbeTimer)
|
||||
}
|
||||
this.scheduledProbeTimer = this.clock.setTimeout(() => {
|
||||
this.scheduledProbeTimer = null
|
||||
void this.runProbe('periodic')
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
private async runProbe(trigger: string): Promise<void> {
|
||||
if (this.stopped || this.retired || this.probeInFlight) {
|
||||
return
|
||||
}
|
||||
if (
|
||||
trigger !== 'startup' &&
|
||||
this.lastProbeStartedAtMs !== null &&
|
||||
this.clock.now() - this.lastProbeStartedAtMs < this.minProbeGapMs
|
||||
) {
|
||||
return
|
||||
}
|
||||
this.probeInFlight = true
|
||||
this.lastProbeStartedAtMs = this.clock.now()
|
||||
try {
|
||||
const outcome = await this.probeLoginSession()
|
||||
if (this.stopped || this.retired) {
|
||||
return
|
||||
}
|
||||
if (outcome === null) {
|
||||
// Why: no wrapper machinery means no PAM oracle — watching would only ever misfire.
|
||||
this.log.log('login-session-watch-disabled', { trigger })
|
||||
this.stop()
|
||||
return
|
||||
}
|
||||
if (!outcome.conclusive) {
|
||||
this.scheduleNextProbe(this.periodicProbeMs)
|
||||
return
|
||||
}
|
||||
if (outcome.ok) {
|
||||
if (!this.armed) {
|
||||
this.log.log('login-session-watch-armed', { trigger })
|
||||
}
|
||||
this.armed = true
|
||||
this.consecutiveRejections = 0
|
||||
this.scheduleNextProbe(this.periodicProbeMs)
|
||||
return
|
||||
}
|
||||
if (!this.armed) {
|
||||
// Session never hosted login(1) here; the preflight already degraded spawns.
|
||||
this.scheduleNextProbe(this.periodicProbeMs)
|
||||
return
|
||||
}
|
||||
this.consecutiveRejections++
|
||||
this.log.log('login-session-probe-rejected', {
|
||||
trigger,
|
||||
rejections: this.consecutiveRejections
|
||||
})
|
||||
if (this.consecutiveRejections < REQUIRED_CONSECUTIVE_REJECTIONS) {
|
||||
this.scheduleNextProbe(this.rejectionRecheckMs)
|
||||
return
|
||||
}
|
||||
const resolverHealth = await this.readResolverHealth()
|
||||
if (this.stopped || this.retired) {
|
||||
return
|
||||
}
|
||||
if (resolverHealth === 'healthy') {
|
||||
// Why: a live session always has DNS configuration; PAM rejecting while the
|
||||
// resolver works means the anomaly is PAM-side, so hold at the threshold and
|
||||
// keep re-verdicting instead of killing live terminals.
|
||||
this.log.log('login-session-retire-suppressed', { resolverHealth })
|
||||
this.consecutiveRejections = REQUIRED_CONSECUTIVE_REJECTIONS - 1
|
||||
this.scheduleNextProbe(this.rejectionRecheckMs)
|
||||
return
|
||||
}
|
||||
this.retired = true
|
||||
this.onRetire({ rejections: this.consecutiveRejections, resolverHealth })
|
||||
} catch (error) {
|
||||
// Why: a probe failure is diagnostic only; an escaped rejection would trip the
|
||||
// daemon's fatal unhandled-error path and kill live terminals.
|
||||
this.log.log('login-session-probe-error', { message: (error as Error)?.message })
|
||||
this.scheduleNextProbe(this.periodicProbeMs)
|
||||
} finally {
|
||||
this.probeInFlight = false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,8 @@ export type TerminalHostOptions = {
|
||||
terminalWindowsWslDistro?: string | null
|
||||
terminalWindowsPowerShellImplementation?: 'auto' | 'powershell.exe' | 'pwsh.exe'
|
||||
}) => SubprocessHandle
|
||||
// Why: login-session death detection (#7936) needs subprocess exits even when no client is attached.
|
||||
onSessionReaped?: (sessionId: string) => void
|
||||
// Why: graceful shutdown checkpoints must finish in-process before teardown.
|
||||
onFinalCheckpoint?: (
|
||||
sessionId: string,
|
||||
|
||||
@@ -27,6 +27,7 @@ export class TerminalHost {
|
||||
private sessionTeardown = new TerminalSessionTeardown(this.sessions)
|
||||
private killedTombstones = new Map<string, number>()
|
||||
private spawnSubprocess: TerminalHostOptions['spawnSubprocess']
|
||||
private onSessionReaped: TerminalHostOptions['onSessionReaped']
|
||||
private onFinalCheckpoint: TerminalHostOptions['onFinalCheckpoint']
|
||||
private maxTombstones: number
|
||||
private creationFenced = false
|
||||
@@ -34,6 +35,7 @@ export class TerminalHost {
|
||||
|
||||
constructor(opts: TerminalHostOptions) {
|
||||
this.spawnSubprocess = opts.spawnSubprocess
|
||||
this.onSessionReaped = opts.onSessionReaped
|
||||
this.onFinalCheckpoint = opts.onFinalCheckpoint
|
||||
this.maxTombstones = opts.maxTombstones ?? DEFAULT_MAX_TOMBSTONES
|
||||
}
|
||||
@@ -201,6 +203,7 @@ export class TerminalHost {
|
||||
}
|
||||
session.dispose()
|
||||
this.sessions.delete(sessionId)
|
||||
this.onSessionReaped?.(sessionId)
|
||||
}
|
||||
|
||||
signal(sessionId: string, sig: string): void {
|
||||
|
||||
+5
-1
@@ -665,7 +665,11 @@ function startTerminalRuntimeStartupServices(): Promise<void> {
|
||||
// Why: both desktop and headless serve must adopt the same persistent provider before creating terminals or a renderer.
|
||||
startDaemonPtyProvider: async (signal) => {
|
||||
logStartupMilestone('startup-service-start', { service: 'daemon-pty-provider' })
|
||||
await initDaemonPtyProvider(signal)
|
||||
// Why: only GUI-spawned macOS daemons watch for login-session death; a headless
|
||||
// serve daemon must survive its spawning session ending (SSH disconnect).
|
||||
await initDaemonPtyProvider(signal, {
|
||||
macosLoginSessionWatch: process.platform === 'darwin' && !isServeMode
|
||||
})
|
||||
logStartupMilestone('startup-service-done', { service: 'daemon-pty-provider' })
|
||||
},
|
||||
// Why: PTY spawn env reads ORCA_AGENT_HOOK_* from live server state, so the renderer awaits this before restored terminals reconnect.
|
||||
|
||||
@@ -13,6 +13,7 @@ vi.mock('node:child_process', () => ({ execFile: execFileMock }))
|
||||
|
||||
import {
|
||||
prepareMacosTccLoginShell,
|
||||
probeMacosLoginSessionAlive,
|
||||
resetMacosLoginShellPreflightForTests,
|
||||
wrapShellSpawnForMacosTccAttribution
|
||||
} from './macos-tcc-login-shell'
|
||||
@@ -340,3 +341,89 @@ describe('wrapShellSpawnForMacosTccAttribution', () => {
|
||||
expect(execFileMock).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('probeMacosLoginSessionAlive', () => {
|
||||
let origPlatform: PropertyDescriptor | undefined
|
||||
let origDisable: string | undefined
|
||||
|
||||
function setPlatform(value: string): void {
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value })
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
origPlatform = Object.getOwnPropertyDescriptor(process, 'platform')
|
||||
origDisable = process.env.ORCA_DISABLE_MACOS_LOGIN_SHELL
|
||||
delete process.env.ORCA_DISABLE_MACOS_LOGIN_SHELL
|
||||
existsSyncMock.mockReturnValue(true)
|
||||
userInfoMock.mockReturnValue({ username: 'ada', homedir: '/Users/ada' })
|
||||
execFileMock.mockImplementation(
|
||||
(_file: string, _args: string[], _options: unknown, callback: ExecFileCallback) => {
|
||||
callback(null, 'ORCA_LOGIN_PREFLIGHT_OK', '')
|
||||
return { stdin: { end: stdinEndMock } }
|
||||
}
|
||||
)
|
||||
resetMacosLoginShellPreflightForTests()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if (origPlatform) {
|
||||
Object.defineProperty(process, 'platform', origPlatform)
|
||||
}
|
||||
if (origDisable === undefined) {
|
||||
delete process.env.ORCA_DISABLE_MACOS_LOGIN_SHELL
|
||||
} else {
|
||||
process.env.ORCA_DISABLE_MACOS_LOGIN_SHELL = origDisable
|
||||
}
|
||||
vi.restoreAllMocks()
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('re-probes even after a cached acceptance', async () => {
|
||||
setPlatform('darwin')
|
||||
await prepareMacosTccLoginShell()
|
||||
expect(execFileMock).toHaveBeenCalledTimes(1)
|
||||
const outcome = await probeMacosLoginSessionAlive()
|
||||
expect(outcome).toEqual({ ok: true, conclusive: true, reason: 'accepted' })
|
||||
expect(execFileMock).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('flips the spawn wrapper off when a fresh probe conclusively rejects (dead login session)', async () => {
|
||||
setPlatform('darwin')
|
||||
await prepareMacosTccLoginShell()
|
||||
expect(wrapShellSpawnForMacosTccAttribution('/bin/zsh', ['-l']).file).toBe('/usr/bin/login')
|
||||
|
||||
execFileMock.mockImplementation(
|
||||
(_file: string, _args: string[], _options: unknown, callback: ExecFileCallback) => {
|
||||
callback(Object.assign(new Error('login incorrect'), { code: 1 }), '', '')
|
||||
return { stdin: { end: stdinEndMock } }
|
||||
}
|
||||
)
|
||||
const outcome = await probeMacosLoginSessionAlive()
|
||||
expect(outcome).toEqual({ ok: false, conclusive: true, reason: 'rejected' })
|
||||
// The dead-session daemon must stop minting login(1) prompt zombies (#7936).
|
||||
expect(wrapShellSpawnForMacosTccAttribution('/bin/zsh', ['-l']).file).toBe('/bin/zsh')
|
||||
})
|
||||
|
||||
it('does not overwrite the cached verdict on an inconclusive probe', async () => {
|
||||
setPlatform('darwin')
|
||||
await prepareMacosTccLoginShell()
|
||||
execFileMock.mockImplementation(
|
||||
(_file: string, _args: string[], _options: unknown, callback: ExecFileCallback) => {
|
||||
callback(Object.assign(new Error('killed'), { killed: true }), '', '')
|
||||
return { stdin: { end: stdinEndMock } }
|
||||
}
|
||||
)
|
||||
const outcome = await probeMacosLoginSessionAlive()
|
||||
expect(outcome).toEqual({ ok: false, conclusive: false, reason: 'timeout' })
|
||||
expect(wrapShellSpawnForMacosTccAttribution('/bin/zsh', ['-l']).file).toBe('/usr/bin/login')
|
||||
})
|
||||
|
||||
it('returns null off macOS and when disabled', async () => {
|
||||
setPlatform('linux')
|
||||
expect(await probeMacosLoginSessionAlive()).toBeNull()
|
||||
setPlatform('darwin')
|
||||
process.env.ORCA_DISABLE_MACOS_LOGIN_SHELL = '1'
|
||||
expect(await probeMacosLoginSessionAlive()).toBeNull()
|
||||
expect(execFileMock).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -194,6 +194,37 @@ export function resetMacosLoginShellPreflightForTests(): void {
|
||||
transientLoginPreflightFailure = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Fresh PAM probe for login-session death detection (#7936): bypasses the
|
||||
* cached verdict and the transient backoff, and writes any conclusive verdict
|
||||
* back into the cache — so a daemon whose login session died stops wrapping
|
||||
* spawns in `login(1)` (which would only mint "Login incorrect" zombies) even
|
||||
* before retirement completes. Returns null when the wrapper doesn't apply.
|
||||
*/
|
||||
export async function probeMacosLoginSessionAlive(): Promise<LoginPreflightOutcome | null> {
|
||||
if (process.platform !== 'darwin' || isDisabledByEnv() || !existsSync(MACOS_LOGIN_PATH)) {
|
||||
return null
|
||||
}
|
||||
let username: string
|
||||
let accountHome: string
|
||||
try {
|
||||
const account = userInfo()
|
||||
username = account.username
|
||||
accountHome = account.homedir
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (!username || !accountHome) {
|
||||
return null
|
||||
}
|
||||
const outcome = await runLoginPreflight(username, accountHome)
|
||||
if (outcome.conclusive) {
|
||||
cachedLoginPreflightResult = outcome.ok
|
||||
transientLoginPreflightFailure = null
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a macOS shell spawn in `/usr/bin/login -flpq <user> …` so terminal children
|
||||
* get their own TCC identity instead of collapsing into Orca's bundle id — signed
|
||||
|
||||
Reference in New Issue
Block a user