Fix terminal daemon restart leaving older Codex conversations locked (#29216)

* Recover retired macOS daemon trees during terminal restart

* fix(daemon): never let retired-job recovery failure block restart

A wedged retired job (unkillable descendant, launchctl timeout, recycled
root) previously aborted every untargeted Restart Terminal Daemon, leaving
no way to replace the current daemon short of a reboot. Log the failure
and continue with the ordinary restart instead.
This commit is contained in:
Neil
2026-10-15 18:08:22 -07:00
committed by GitHub
parent 9fa50a3d6d
commit ac015cd3b2
10 changed files with 689 additions and 8 deletions
+82
View File
@@ -10,6 +10,88 @@
}
},
"gates": [
{
"id": "terminal-provider.retired-macos-daemon-recovery",
"title": "Restart releases retired macOS daemon conversation owners across upgrades",
"maturity": "experimental",
"protection": "partial",
"owner": "daemon-terminal",
"layer": "macos-daemon-lifecycle",
"surfaces": [
"Manage Sessions Restart",
"agent conversation locks",
"legacy terminal daemons"
],
"platforms": ["macos"],
"providers": ["local-daemon"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["local-daemon"],
"coverageNotes": "Packaged GUI recovery is tested with deterministic job/process evidence and an isolated native launchd service holding a real advisory writer lock. Linux, Windows, Node/SSH hosts, WSL and mobile/relay do not execute GUI-service recovery; healthy legacy routing is unchanged.",
"motivatingLinks": ["https://github.com/stablyai/orca/pull/29130"],
"invariant": "An explicit Restart may recover same-profile GUI services with retired endpoint ownership, including older protocols, only after exact job and root identity checks. It must verify descendants exited before releasing the job. Healthy, newer, foreign-profile, starting or unverifiable owners must not be stopped. Recovery failure is logged and must not block restarting the current daemon.",
"oracle": "Recover an old-protocol conversation owner and require descendants to exit before the root is killed and its label released. Reject connected/unknown endpoints, present/unreadable ownership records, new processes, replaced jobs, recycled roots and failed descendant verification. Require recovery before current-session fanout, and a completed current-daemon restart if recovery fails.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/daemon/macos-retired-daemon-jobs.test.ts src/main/daemon/macos-retired-daemon-recovery.test.ts src/main/daemon/daemon-init-restart-sequence.test.ts src/main/daemon/terminal-descendant-shutdown.test.ts src/main/daemon/macos-daemon-launchd.test.ts"
],
"testFiles": [
"src/main/daemon/macos-retired-daemon-jobs.test.ts",
"src/main/daemon/macos-retired-daemon-recovery.test.ts",
"src/main/daemon/daemon-init-restart-sequence.test.ts",
"src/main/daemon/terminal-descendant-shutdown.test.ts",
"src/main/daemon/macos-daemon-launchd.test.ts"
],
"assertionRefs": [
{
"file": "src/main/daemon/macos-retired-daemon-recovery.test.ts",
"assertions": [
"releases retired old-protocol conversation owners before returning from Restart",
"never signals a root recycled during descendant teardown"
]
},
{
"file": "src/main/daemon/daemon-init-restart-sequence.test.ts",
"assertions": [
"still restarts the current daemon when retired-daemon recovery cannot prove cleanup"
]
}
],
"evidenceRuns": [
{
"date": "2026-10-15",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/daemon/macos-retired-daemon-jobs.test.ts src/main/daemon/macos-retired-daemon-recovery.test.ts src/main/daemon/daemon-init-restart-sequence.test.ts src/main/daemon/terminal-descendant-shutdown.test.ts src/main/daemon/macos-daemon-launchd.test.ts",
"result": "passed",
"durationSeconds": 2.27,
"summary": "79 tests passed. A separate native launchd fixture preserved its fresh service, then recovered the retired tree and released a real writer lock in 306 ms. Field recovery released 80 locks held by the previous daemon."
}
],
"runtimeBudget": {
"p95Seconds": 5,
"scope": "deterministic retired-job recovery and restart contract"
},
"flakeHistory": {
"status": "unknown",
"evidence": "Focused local tests and native smoke pass; CI soak history is not yet collected."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "The installed build restarted protocol 46 while a retired protocol 45 process retained 80 writer locks. The isolated native fixture now releases its real writer lock; automated baseline mutation evidence is not collected."
},
"performanceBudget": {
"required": true,
"evidence": "No startup work or periodic scans. One bounded launchctl list per explicit restart; at most 32 same-profile supported services. Only verified retired candidates invoke the existing bounded descendant verifier. Tests assert one discovery and one initial descendant capture per recovered job."
},
"promotionCriteria": [
"Collect 100 focused CI passes or 14 days of soak history.",
"Capture packaged upgrade and Restart recovery with multiple retired jobs."
],
"knownGaps": [
"Full packaged GUI upgrade journey and multi-job native recovery remain uncollected.",
"Recovery is specific to signed-helper launchd jobs; fork-launched and already reparented processes have no equivalent retained ownership evidence."
],
"demotionRule": "Keep experimental or demote if recovery signals an unverified owner, treats loss of contact as exit, reports success before descendant exit, adds background scanning, or lets a recovery failure block the current-daemon restart."
},
{
"id": "agent-session.delayed-terminal-launch",
"title": "Delayed terminal launches retain their harness preparation and recover process discovery",
@@ -126,6 +126,13 @@ keeps about 124 MiB.
RPC and signal shutdown share a five-second process-exit deadline, armed before terminal teardown.
A stalled teardown must not leave the endpoint retired while the launchd job stays alive forever.
An explicit, untargeted Restart also recovers signed-helper jobs left by older builds after they
retired their PID record and endpoint but failed to exit. Recovery uses exact launchd labels and
arguments for this profile, ignores newer protocols, and preserves live or unreadable endpoints
and services still publishing a PID record. It verifies descendant exits before killing the old
root and releasing its label. Startup never runs this cleanup, and a failed recovery is logged
without blocking the current daemon's restart. Node/SSH hosts do not perform GUI-service recovery.
Running old daemons keep their terminals; settings reports app and terminal-host Full Disk Access
separately, and a denied terminal is recovered by closing it or restarting the daemon. The packaging
hook also gives the main bundle and every Helper the Desktop, Documents and Downloads descriptions,
@@ -1,5 +1,7 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { PROTOCOL_VERSION } from './types'
const { recoverRetired } = vi.hoisted(() => ({ recoverRetired: vi.fn(async () => {}) }))
vi.mock('./macos-retired-daemon-recovery', () => ({ recoverRetiredMacDaemonJobs: recoverRetired }))
const {
probeSocketExistsMock,
@@ -45,6 +47,7 @@ vi.mock('../ipc/pty/provider/listener-lifecycle', () => moduleFactories.ptyListe
describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
beforeEach(() => {
recoverRetired.mockReset().mockResolvedValue(undefined)
installDefaultNetConnectStub()
})
@@ -79,6 +82,22 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
})
// Ordering invariant: synthetic exits must reach the renderer *before* listeners are torn down (Step 1 before 2).
expect(order).toEqual(['fanout', 'unbind'])
expect(recoverRetired).toHaveBeenCalledOnce()
expect(recoverRetired.mock.invocationCallOrder[0]).toBeLessThan(
originalAdapter.fanoutSyntheticExits.mock.invocationCallOrder[0]
)
})
it('still restarts the current daemon when retired-daemon recovery cannot prove cleanup', async () => {
const mod = await importFresh()
await mod.initDaemonPtyProvider()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
recoverRetired.mockRejectedValueOnce(new Error('retired descendants unverifiable'))
await expect(mod.restartDaemon()).resolves.toBeDefined()
expect(warn).toHaveBeenCalledWith(expect.stringContaining('retired descendants unverifiable'))
expect(adapterInstances[0].fanoutSyntheticExits).toHaveBeenCalled()
expect(spawnerInstances[0].resetHandle).toHaveBeenCalled()
warn.mockRestore()
})
it('refuses a targeted restart before ending anything when another daemon owns the endpoint', async () => {
@@ -101,6 +120,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
expect(originalAdapter.fanoutSyntheticExits).not.toHaveBeenCalled()
expect(unbindLocalProviderListenersMock).not.toHaveBeenCalled()
expect(recoverRetired).not.toHaveBeenCalled()
})
it('uses daemon-owned idle retirement after a failed manual-restart adoption', async () => {
@@ -42,6 +42,7 @@ import {
import { DaemonClient } from './client'
import { clearSeveredDaemon } from './daemon-host-image-state'
import { PROTOCOL_VERSION } from './types'
import { recoverRetiredMacDaemonJobs } from './macos-retired-daemon-recovery'
export type { RestartDaemonResult } from './daemon-restart-state'
@@ -60,6 +61,14 @@ async function runRestartDaemon(expectedDaemonPid?: number): Promise<RestartDaem
}
const runtimeDir = getRuntimeDir()
if (expectedDaemonPid === undefined) {
// Why not abort: a wedged leftover must never disable the restart that replaces the current daemon.
await recoverRetiredMacDaemonJobs(runtimeDir).catch((error: unknown) => {
console.warn(
`[daemon] Retired macOS terminal service recovery failed: ${error instanceof Error ? error.message : String(error)}`
)
})
}
if (expectedDaemonPid !== undefined) {
// Why before any teardown: a targeted restart must not end a single terminal when the
// endpoint now belongs to a replacement daemon.
+12
View File
@@ -0,0 +1,12 @@
import { createHash } from 'node:crypto'
import { resolve } from 'node:path'
import { PROTOCOL_VERSION } from './daemon-protocol-version'
/** One job per profile and protocol, including jobs retained across app upgrades. */
export function macDaemonJobLabel(
userDataPath: string,
protocolVersion = PROTOCOL_VERSION
): string {
const profileKey = createHash('sha256').update(resolve(userDataPath)).digest('hex').slice(0, 16)
return `com.stablyai.orca.terminal.${profileKey}.v${protocolVersion}`
}
+3 -8
View File
@@ -1,6 +1,5 @@
import { createHash } from 'node:crypto'
import { writeFile } from 'node:fs/promises'
import { join, resolve } from 'node:path'
import { join } from 'node:path'
import { setTimeout as delay } from 'node:timers/promises'
import { getAppEnvironment } from '../../shared/app-environment'
import { LOCAL_BUILD_COMPATIBILITY_CONTRACT } from '../../shared/local-build-compatibility-contract'
@@ -11,7 +10,7 @@ import { compareDottedVersions } from '../orcad/native-host-abi'
import { DaemonClient } from './client'
import { DaemonEndpointOwnershipError, holdDaemonAdoptionLease } from './daemon-endpoint-adoption'
import { buildDaemonScriptArgs, type DaemonChildSpawnOptions } from './daemon-launched-child-spawn'
import { PROTOCOL_VERSION } from './daemon-protocol-version'
import { macDaemonJobLabel } from './macos-daemon-job-label'
import type { DaemonProcessHandle } from './daemon-spawner'
import { DAEMON_SHUTDOWN_TIMEOUT_MS } from './daemon-shutdown-deadline'
import {
@@ -43,11 +42,7 @@ const BOOTSTRAP_TIMEOUT_MS = 10_000
const STARTUP_TIMEOUT_MS = 10_000
const JOB_CHECK_INTERVAL_MS = 250
/** One job per profile and protocol, matching the one daemon each of them may run. */
export function macDaemonJobLabel(userDataPath: string): string {
const profileKey = createHash('sha256').update(resolve(userDataPath)).digest('hex').slice(0, 16)
return `com.stablyai.orca.terminal.${profileKey}.v${PROTOCOL_VERSION}`
}
export { macDaemonJobLabel } from './macos-daemon-job-label'
function buildMacDaemonLaunchJob(
options: MacDaemonLaunchOptions,
@@ -0,0 +1,119 @@
import { beforeEach, expect, it, vi } from 'vitest'
import type { ProcessResult, ProcessSpec } from '@orca/process-host/process-spec'
import { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from './daemon-spawner'
import { PROTOCOL_VERSION } from './daemon-protocol-version'
import { macDaemonJobLabel } from './macos-daemon-job-label'
const { run } = vi.hoisted(() => ({ run: vi.fn<(spec: ProcessSpec) => Promise<ProcessResult>>() }))
vi.mock('@orca/process-host', () => ({ runProcess: run }))
import { listMacDaemonJobVersions, readMacDaemonRecoveryJob } from './macos-retired-daemon-jobs'
const ok: ProcessResult = { code: 0, signal: null, stdout: '', stderr: '', timedOut: false }
const profile = '/profiles/a'
const runtime = '/profiles/a/daemon'
const version = PROTOCOL_VERSION - 1
const args = [
'/copy/host',
'/copy/daemon-entry.js',
'--socket',
getDaemonSocketPath(runtime, version),
'--token',
getDaemonTokenPath(runtime, version),
'--pid-record',
getDaemonPidPath(runtime, version),
'--launch-nonce',
'old-nonce',
'--login-session-watch'
]
const job = (argv = args): string =>
`service = {\n\tstate = running\n\tpid = 12345\n\targuments = {\n${argv.map((v) => `\t\t${v}\n`).join('')}\t}\n}\n`
beforeEach(() => run.mockReset())
it('discovers only exact same-profile supported labels in one bounded command', async () => {
run.mockResolvedValue({
...ok,
stdout: [
`12345\t0\t${macDaemonJobLabel(profile, version)}`,
`12346\t0\t${macDaemonJobLabel(profile)}`,
`12347\t0\t${macDaemonJobLabel(profile, PROTOCOL_VERSION + 1)}`,
`12348\t0\t${macDaemonJobLabel('/profiles/other', version)}`,
`12349\t0\t${macDaemonJobLabel(profile, version)}.extra`
].join('\n')
})
expect(await listMacDaemonJobVersions(profile)).toEqual([version, PROTOCOL_VERSION])
expect(run).toHaveBeenCalledOnce()
expect(run).toHaveBeenCalledWith(
expect.objectContaining({ args: ['list'], timeoutMs: 3_000, maxOutputBytes: 128 * 1024 })
)
})
it('requires exact endpoint arguments from the top-level launchd job', async () => {
run.mockResolvedValue({ ...ok, stdout: job() })
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toEqual({
service: `gui/${process.getuid?.()}/${macDaemonJobLabel(profile, version)}`,
protocolVersion: version,
pid: 12345,
launchNonce: 'old-nonce',
socketPath: getDaemonSocketPath(runtime, version),
pidPath: getDaemonPidPath(runtime, version)
})
})
it.each(
[
args.filter((a) => a !== '--login-session-watch'),
[...args, '--socket', getDaemonSocketPath(runtime, version)],
args.map((a) => (a === getDaemonSocketPath(runtime, version) ? `${a}.other` : a)),
args.map((a) => (a === getDaemonTokenPath(runtime, version) ? '/another/token' : a)),
args.map((a) => (a === getDaemonPidPath(runtime, version) ? '/another/pid' : a)),
args.filter((a) => a !== 'old-nonce')
].map((argv) => ({ argv }))
)('rejects ambiguous or foreign ownership: $argv', async ({ argv }) => {
run.mockResolvedValue({ ...ok, stdout: job(argv) })
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull()
})
it('ignores a job that exited between list and print', async () => {
run.mockResolvedValue({ ...ok, code: 113, stderr: 'Could not find service "old"' })
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull()
})
it.each([
{ timedOut: true },
{ outputTruncated: true },
{ code: 1 },
{ code: 113, stderr: 'Could not find service; Permission denied' }
])('fails closed on unreadable or incomplete job output: %j', async (override) => {
run.mockResolvedValue({ ...ok, stdout: job(), ...override })
await expect(readMacDaemonRecoveryJob(profile, runtime, version)).rejects.toThrow(
'Could not inspect'
)
})
it('never treats a nested state or pid as the running job', async () => {
run.mockResolvedValue({
...ok,
stdout: job().replace('\tstate = running', '\t\tstate = running')
})
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull()
})
it('bounds job discovery and rejects an oversized profile inventory', async () => {
run.mockResolvedValue({
...ok,
stdout: Array.from(
{ length: 33 },
(_, index) => `12345\t0\t${macDaemonJobLabel(profile, index + 1)}`
).join('\n')
})
await expect(listMacDaemonJobVersions(profile)).rejects.toThrow('Too many')
expect(run).toHaveBeenCalledOnce()
})
it('never inspects newer or invalid protocol jobs', async () => {
for (const unsupported of [0, -1, 1.5, PROTOCOL_VERSION + 1]) {
expect(await readMacDaemonRecoveryJob(profile, runtime, unsupported)).toBeNull()
}
expect(run).not.toHaveBeenCalled()
})
@@ -0,0 +1,102 @@
import { runProcess } from '@orca/process-host'
import { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from './daemon-spawner'
import { PROTOCOL_VERSION } from './daemon-protocol-version'
import { macDaemonJobLabel } from './macos-daemon-job-label'
export type RetiredMacDaemonJob = {
service: string
protocolVersion: number
pid: number
launchNonce: string
socketPath: string
pidPath: string
}
async function launchctl(args: string[]): Promise<string | null> {
const result = await runProcess({
program: '/bin/launchctl',
args,
timeoutMs: 3_000,
maxOutputBytes: 128 * 1024
})
if (
args[0] === 'print' &&
result.code === 113 &&
!result.timedOut &&
!result.outputTruncated &&
result.stderr.includes('Could not find service') &&
!/Operation not permitted|Permission denied/i.test(result.stderr)
) {
return null
}
if (result.code !== 0 || result.timedOut || result.outputTruncated) {
throw new Error('Could not inspect macOS terminal services for restart recovery')
}
return result.stdout
}
/** Reads only exact profile labels; newer protocols remain owned by their newer app. */
export async function listMacDaemonJobVersions(userDataPath: string): Promise<number[]> {
const prefix = macDaemonJobLabel(userDataPath, 0).slice(0, -1)
const versions = new Set<number>()
for (const line of (await launchctl(['list']))?.split('\n') ?? []) {
const label = line.trim().split(/\s+/)[2]
if (!label?.startsWith(prefix)) {
continue
}
const suffix = label.slice(prefix.length)
const version = Number(suffix)
if (/^[1-9]\d*$/.test(suffix) && Number.isSafeInteger(version) && version <= PROTOCOL_VERSION) {
versions.add(version)
}
}
if (versions.size > 32) {
throw new Error('Too many macOS terminal services to recover safely')
}
return [...versions]
}
/** launchctl's structured argument block provides exact endpoint ownership, without parsing argv text. */
export async function readMacDaemonRecoveryJob(
userDataPath: string,
runtimeDir: string,
protocolVersion: number
): Promise<RetiredMacDaemonJob | null> {
if (
!Number.isSafeInteger(protocolVersion) ||
protocolVersion < 1 ||
protocolVersion > PROTOCOL_VERSION
) {
return null
}
const service = `gui/${process.getuid?.()}/${macDaemonJobLabel(userDataPath, protocolVersion)}`
const output = await launchctl(['print', service])
if (!output || !/^\tstate = running$/m.test(output)) {
return null
}
const pid = Number(/^\tpid = (\d+)$/m.exec(output)?.[1])
const block = /^\targuments = \{\n((?:\t\t[^\n]*\n)+)\t\}$/m.exec(output)?.[1]
const args =
block
?.split('\n')
.filter(Boolean)
.map((line) => line.slice(2)) ?? []
const argument = (flag: string): string | undefined =>
args.filter((value) => value === flag).length === 1 ? args[args.indexOf(flag) + 1] : undefined
const socketPath = getDaemonSocketPath(runtimeDir, protocolVersion)
const pidPath = getDaemonPidPath(runtimeDir, protocolVersion)
const launchNonce = argument('--launch-nonce')
if (
!Number.isSafeInteger(pid) ||
pid <= 1 ||
!launchNonce ||
launchNonce.startsWith('--') ||
!args.includes('--login-session-watch') ||
argument('--socket') !== socketPath ||
argument('--token') !== getDaemonTokenPath(runtimeDir, protocolVersion) ||
argument('--pid-record') !== pidPath
) {
return null
}
return { service, protocolVersion, pid, launchNonce, socketPath, pidPath }
}
@@ -0,0 +1,225 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import type { DescendantSnapshot } from '../pty-descendant-termination'
import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification'
import type { RetiredMacDaemonJob } from './macos-retired-daemon-jobs'
import type { ProcessIdentity } from './daemon-process-identity-query'
import type { SocketProbeOutcome } from './daemon-endpoint-probe'
const mocks = vi.hoisted(() => ({
packaged: true,
stat: vi.fn(),
versions: vi.fn<() => Promise<number[]>>(),
job: vi.fn<() => Promise<RetiredMacDaemonJob | null>>(),
probe: vi.fn<() => Promise<SocketProbeOutcome>>(),
identity: vi.fn<() => Promise<ProcessIdentity | null>>(),
capture: vi.fn<() => Promise<DescendantSnapshot | null>>(),
terminate: vi.fn<() => Promise<DescendantTreeVerdict>>(),
release: vi.fn<() => Promise<boolean>>()
}))
vi.mock('node:fs/promises', () => ({ lstat: mocks.stat }))
vi.mock('../../shared/app-environment', () => ({
getAppEnvironment: () => ({ isPackaged: () => mocks.packaged, getPath: () => '/profile' })
}))
vi.mock('../pty-descendant-termination', () => ({ captureDescendantSnapshot: mocks.capture }))
vi.mock('./macos-retired-daemon-jobs', () => ({
listMacDaemonJobVersions: mocks.versions,
readMacDaemonRecoveryJob: mocks.job
}))
vi.mock('./daemon-endpoint-probe', () => ({
endpointIsProvenDead: (outcome: string) => outcome === 'missing' || outcome === 'refused',
probeSocketConnect: mocks.probe
}))
vi.mock('./daemon-process-identity-query', () => ({ getPsProcessIdentityAsync: mocks.identity }))
vi.mock('./terminal-descendant-shutdown', () => ({ terminateShutdownDescendants: mocks.terminate }))
vi.mock('./macos-daemon-job-state', () => ({ releaseMacDaemonJob: mocks.release }))
import { recoverRetiredMacDaemonJobs } from './macos-retired-daemon-recovery'
const startedAt = 'Mon Oct 12 16:28:32 2026'
const identity = { commandLine: 'daemon', startedAtMs: Date.parse(startedAt) }
const job: RetiredMacDaemonJob = {
service: 'gui/501/profile.v45',
protocolVersion: 45,
pid: 7621,
launchNonce: 'old',
socketPath: '/profile/daemon/v45.sock',
pidPath: '/profile/daemon/v45.pid'
}
const snapshot: DescendantSnapshot = {
root: { pid: job.pid, startedAt },
rootPgid: job.pid,
descendants: [{ pid: 88912, ppid: job.pid, pgid: 88912, startedAt }],
capturedAtMs: Date.parse(startedAt) + 60_000
}
const typeDescriptor = Object.getOwnPropertyDescriptor(process, 'type')
const kill = vi.fn<typeof process.kill>()
beforeEach(() => {
vi.resetAllMocks()
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
Object.defineProperty(process, 'type', { configurable: true, value: 'browser' })
vi.spyOn(Date, 'now').mockReturnValue(identity.startedAtMs + 60_000)
vi.spyOn(console, 'info').mockImplementation(() => {})
kill.mockReturnValue(true)
vi.spyOn(process, 'kill').mockImplementation(kill)
mocks.packaged = true
mocks.stat.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' }))
mocks.versions.mockResolvedValue([45])
mocks.job.mockResolvedValue(job)
mocks.probe.mockResolvedValue('refused')
mocks.identity.mockResolvedValue(identity)
mocks.capture.mockResolvedValue(snapshot)
mocks.terminate.mockResolvedValue('exited')
mocks.release.mockResolvedValue(true)
})
afterEach(() => {
vi.restoreAllMocks()
if (typeDescriptor) {
Object.defineProperty(process, 'type', typeDescriptor)
} else {
Reflect.deleteProperty(process, 'type')
}
})
it('releases retired old-protocol conversation owners before returning from Restart', async () => {
mocks.identity
.mockResolvedValueOnce(identity)
.mockResolvedValueOnce(identity)
.mockResolvedValueOnce(identity)
.mockResolvedValueOnce(null)
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(mocks.terminate).toHaveBeenCalledExactlyOnceWith(snapshot)
expect(kill.mock.calls).toEqual([[job.pid, 'SIGKILL']])
expect(mocks.terminate.mock.invocationCallOrder[0]).toBeLessThan(kill.mock.invocationCallOrder[0])
expect(mocks.release).toHaveBeenCalledExactlyOnceWith(job.service, Date.now() + 3_000)
expect(mocks.versions).toHaveBeenCalledOnce()
expect(mocks.capture).toHaveBeenCalledOnce()
})
it.each(['connected', 'unknown'] as const)('preserves endpoints that read %s', async (outcome) => {
mocks.probe.mockResolvedValue(outcome)
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(kill).not.toHaveBeenCalled()
expect(mocks.capture).not.toHaveBeenCalled()
})
it.each(['present', 'permission-denied'] as const)(
'preserves non-retired PID records: %s',
async (state) => {
if (state === 'present') {
mocks.stat.mockResolvedValue({})
} else {
mocks.stat.mockRejectedValue(Object.assign(new Error('denied'), { code: 'EACCES' }))
}
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(kill).not.toHaveBeenCalled()
expect(mocks.probe).not.toHaveBeenCalled()
}
)
it.each([null, { ...identity, startedAtMs: identity.startedAtMs + 59_000 }])(
'preserves uninspectable or newly starting processes: %j',
async (value) => {
mocks.identity.mockResolvedValue(value)
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(kill).not.toHaveBeenCalled()
}
)
it('never signals a reused root PID or a replacement service', async () => {
mocks.job.mockResolvedValueOnce(job).mockResolvedValueOnce({ ...job, launchNonce: 'new' })
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(kill).not.toHaveBeenCalled()
mocks.job.mockResolvedValue(job)
mocks.identity
.mockResolvedValueOnce(identity)
.mockResolvedValueOnce({ ...identity, startedAtMs: 1 })
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(kill).not.toHaveBeenCalled()
})
it('signals nothing and refuses success when process-table capture fails', async () => {
mocks.capture.mockResolvedValue(null)
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
expect(kill).not.toHaveBeenCalled()
expect(mocks.terminate).not.toHaveBeenCalled()
})
it('rechecks retirement after capturing the descendant tree', async () => {
mocks.probe.mockResolvedValueOnce('refused').mockResolvedValueOnce('connected')
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
expect(mocks.terminate).not.toHaveBeenCalled()
expect(kill).not.toHaveBeenCalled()
})
it('never sweeps a snapshot naming another root incarnation', async () => {
mocks.capture.mockResolvedValue({
...snapshot,
root: { pid: job.pid, startedAt: 'Tue Oct 13 16:28:32 2026' }
})
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
expect(mocks.terminate).not.toHaveBeenCalled()
expect(kill).not.toHaveBeenCalled()
})
it('preserves a service that republishes its PID record during inspection', async () => {
mocks.stat
.mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' }))
.mockResolvedValue({})
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
expect(mocks.terminate).not.toHaveBeenCalled()
expect(kill).not.toHaveBeenCalled()
})
it.each(['live', 'unverifiable'] as const)(
'fails closed when descendants are %s',
async (verdict) => {
mocks.terminate.mockResolvedValue(verdict)
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow(verdict)
expect(kill).not.toHaveBeenCalled()
expect(mocks.release).not.toHaveBeenCalled()
}
)
it('never signals a root recycled during descendant teardown', async () => {
mocks.identity
.mockResolvedValueOnce(identity)
.mockResolvedValueOnce(identity)
.mockResolvedValue({ ...identity, startedAtMs: 1 })
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('changed')
expect(kill.mock.calls).toEqual([[job.pid, 0]])
expect(mocks.release).not.toHaveBeenCalled()
})
it('does not report success while a service registration remains', async () => {
mocks.release.mockResolvedValue(false)
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not release')
})
it('accepts a root that exits naturally after its descendants', async () => {
mocks.identity
.mockResolvedValueOnce(identity)
.mockResolvedValueOnce(identity)
.mockResolvedValueOnce(null)
kill.mockImplementation(() => {
throw Object.assign(new Error('gone'), { code: 'ESRCH' })
})
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(kill.mock.calls).toEqual([[job.pid, 0]])
expect(mocks.release).toHaveBeenCalledOnce()
})
it.each(['linux', 'win32'] as const)('does no discovery on %s', async (platform) => {
vi.spyOn(process, 'platform', 'get').mockReturnValue(platform)
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(mocks.versions).not.toHaveBeenCalled()
})
it('does no discovery for Node/SSH execution hosts or unpackaged builds', async () => {
Object.defineProperty(process, 'type', { configurable: true, value: undefined })
await recoverRetiredMacDaemonJobs('/profile/daemon')
Object.defineProperty(process, 'type', { configurable: true, value: 'browser' })
mocks.packaged = false
await recoverRetiredMacDaemonJobs('/profile/daemon')
expect(mocks.versions).not.toHaveBeenCalled()
})
@@ -0,0 +1,110 @@
import { lstat } from 'node:fs/promises'
import { getAppEnvironment } from '../../shared/app-environment'
import { captureDescendantSnapshot } from '../pty-descendant-termination'
import { endpointIsProvenDead, probeSocketConnect } from './daemon-endpoint-probe'
import { getPsProcessIdentityAsync } from './daemon-process-identity-query'
import { hasErrorCode, inspectProcessLiveness } from './daemon-process-inspection'
import { releaseMacDaemonJob } from './macos-daemon-job-state'
import {
listMacDaemonJobVersions,
readMacDaemonRecoveryJob,
type RetiredMacDaemonJob
} from './macos-retired-daemon-jobs'
import { terminateShutdownDescendants } from './terminal-descendant-shutdown'
async function pidRecordWasRetired(path: string): Promise<boolean> {
try {
await lstat(path)
return false
} catch (error) {
return hasErrorCode(error, 'ENOENT')
}
}
async function endpointWasRetired(job: RetiredMacDaemonJob): Promise<boolean> {
return (
(await pidRecordWasRetired(job.pidPath)) &&
endpointIsProvenDead(await probeSocketConnect(job.socketPath))
)
}
async function sameJob(
job: RetiredMacDaemonJob,
userDataPath: string,
runtimeDir: string
): Promise<boolean> {
const current = await readMacDaemonRecoveryJob(userDataPath, runtimeDir, job.protocolVersion)
return current?.pid === job.pid && current.launchNonce === job.launchNonce
}
async function recoverJob(
job: RetiredMacDaemonJob,
userDataPath: string,
runtimeDir: string
): Promise<void> {
if (!(await endpointWasRetired(job))) {
return
}
const identity = await getPsProcessIdentityAsync(job.pid)
// A newly bootstrapping service may not have published yet; it is never a recovery target.
if (!identity?.startedAtMs || Date.now() - identity.startedAtMs < 10_000) {
return
}
if (!(await sameJob(job, userDataPath, runtimeDir))) {
return
}
const beforeCapture = await getPsProcessIdentityAsync(job.pid)
if (beforeCapture?.startedAtMs !== identity.startedAtMs) {
return
}
// Capture before signalling; let the daemon reap children while their exits are verified.
const snapshot = await captureDescendantSnapshot(job.pid)
if (
!snapshot?.root ||
Date.parse(snapshot.root.startedAt) !== identity.startedAtMs ||
!(await sameJob(job, userDataPath, runtimeDir)) ||
!(await endpointWasRetired(job))
) {
throw new Error('Could not verify the retired macOS terminal service for recovery')
}
const verdict = await terminateShutdownDescendants(snapshot)
if (verdict !== 'exited') {
throw new Error(`Retired macOS terminal service descendants are ${verdict}`)
}
const remaining = await getPsProcessIdentityAsync(job.pid)
if (remaining?.startedAtMs === identity.startedAtMs) {
try {
process.kill(job.pid, 'SIGKILL')
} catch (error) {
if (!hasErrorCode(error, 'ESRCH')) {
throw error
}
}
} else if (inspectProcessLiveness(job.pid).status !== 'exited') {
throw new Error('Retired macOS terminal service changed during recovery')
}
if (!(await releaseMacDaemonJob(job.service, Date.now() + 3_000))) {
throw new Error('Could not release the retired macOS terminal service registration')
}
console.info(
`[daemon] Recovered retired macOS terminal service protocol=${job.protocolVersion} pid=${job.pid} descendants=${snapshot.descendants.length}`
)
}
/** Explicit Restart only: healthy legacy daemons and newer app protocols retain their sessions. */
export async function recoverRetiredMacDaemonJobs(runtimeDir: string): Promise<void> {
if (
process.platform !== 'darwin' ||
process.type !== 'browser' ||
!getAppEnvironment().isPackaged()
) {
return
}
const userDataPath = getAppEnvironment().getPath('userData')
for (const version of await listMacDaemonJobVersions(userDataPath)) {
const job = await readMacDaemonRecoveryJob(userDataPath, runtimeDir, version)
if (job) {
await recoverJob(job, userDataPath, runtimeDir)
}
}
}