mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
Fix terminal daemon restart leaving older Codex conversations locked (#29216)
* Recover retired macOS daemon trees during terminal restart * fix(daemon): never let retired-job recovery failure block restart A wedged retired job (unkillable descendant, launchctl timeout, recycled root) previously aborted every untargeted Restart Terminal Daemon, leaving no way to replace the current daemon short of a reboot. Log the failure and continue with the ordinary restart instead.
This commit is contained in:
@@ -10,6 +10,88 @@
|
||||
}
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"id": "terminal-provider.retired-macos-daemon-recovery",
|
||||
"title": "Restart releases retired macOS daemon conversation owners across upgrades",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "daemon-terminal",
|
||||
"layer": "macos-daemon-lifecycle",
|
||||
"surfaces": [
|
||||
"Manage Sessions Restart",
|
||||
"agent conversation locks",
|
||||
"legacy terminal daemons"
|
||||
],
|
||||
"platforms": ["macos"],
|
||||
"providers": ["local-daemon"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": ["local-daemon"],
|
||||
"coverageNotes": "Packaged GUI recovery is tested with deterministic job/process evidence and an isolated native launchd service holding a real advisory writer lock. Linux, Windows, Node/SSH hosts, WSL and mobile/relay do not execute GUI-service recovery; healthy legacy routing is unchanged.",
|
||||
"motivatingLinks": ["https://github.com/stablyai/orca/pull/29130"],
|
||||
"invariant": "An explicit Restart may recover same-profile GUI services with retired endpoint ownership, including older protocols, only after exact job and root identity checks. It must verify descendants exited before releasing the job. Healthy, newer, foreign-profile, starting or unverifiable owners must not be stopped. Recovery failure is logged and must not block restarting the current daemon.",
|
||||
"oracle": "Recover an old-protocol conversation owner and require descendants to exit before the root is killed and its label released. Reject connected/unknown endpoints, present/unreadable ownership records, new processes, replaced jobs, recycled roots and failed descendant verification. Require recovery before current-session fanout, and a completed current-daemon restart if recovery fails.",
|
||||
"commands": [
|
||||
"ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/daemon/macos-retired-daemon-jobs.test.ts src/main/daemon/macos-retired-daemon-recovery.test.ts src/main/daemon/daemon-init-restart-sequence.test.ts src/main/daemon/terminal-descendant-shutdown.test.ts src/main/daemon/macos-daemon-launchd.test.ts"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/main/daemon/macos-retired-daemon-jobs.test.ts",
|
||||
"src/main/daemon/macos-retired-daemon-recovery.test.ts",
|
||||
"src/main/daemon/daemon-init-restart-sequence.test.ts",
|
||||
"src/main/daemon/terminal-descendant-shutdown.test.ts",
|
||||
"src/main/daemon/macos-daemon-launchd.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "src/main/daemon/macos-retired-daemon-recovery.test.ts",
|
||||
"assertions": [
|
||||
"releases retired old-protocol conversation owners before returning from Restart",
|
||||
"never signals a root recycled during descendant teardown"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/daemon/daemon-init-restart-sequence.test.ts",
|
||||
"assertions": [
|
||||
"still restarts the current daemon when retired-daemon recovery cannot prove cleanup"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-10-15",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/daemon/macos-retired-daemon-jobs.test.ts src/main/daemon/macos-retired-daemon-recovery.test.ts src/main/daemon/daemon-init-restart-sequence.test.ts src/main/daemon/terminal-descendant-shutdown.test.ts src/main/daemon/macos-daemon-launchd.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 2.27,
|
||||
"summary": "79 tests passed. A separate native launchd fixture preserved its fresh service, then recovered the retired tree and released a real writer lock in 306 ms. Field recovery released 80 locks held by the previous daemon."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 5,
|
||||
"scope": "deterministic retired-job recovery and restart contract"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "unknown",
|
||||
"evidence": "Focused local tests and native smoke pass; CI soak history is not yet collected."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "partial",
|
||||
"evidence": "The installed build restarted protocol 46 while a retired protocol 45 process retained 80 writer locks. The isolated native fixture now releases its real writer lock; automated baseline mutation evidence is not collected."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "No startup work or periodic scans. One bounded launchctl list per explicit restart; at most 32 same-profile supported services. Only verified retired candidates invoke the existing bounded descendant verifier. Tests assert one discovery and one initial descendant capture per recovered job."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Collect 100 focused CI passes or 14 days of soak history.",
|
||||
"Capture packaged upgrade and Restart recovery with multiple retired jobs."
|
||||
],
|
||||
"knownGaps": [
|
||||
"Full packaged GUI upgrade journey and multi-job native recovery remain uncollected.",
|
||||
"Recovery is specific to signed-helper launchd jobs; fork-launched and already reparented processes have no equivalent retained ownership evidence."
|
||||
],
|
||||
"demotionRule": "Keep experimental or demote if recovery signals an unverified owner, treats loss of contact as exit, reports success before descendant exit, adds background scanning, or lets a recovery failure block the current-daemon restart."
|
||||
},
|
||||
{
|
||||
"id": "agent-session.delayed-terminal-launch",
|
||||
"title": "Delayed terminal launches retain their harness preparation and recover process discovery",
|
||||
|
||||
@@ -126,6 +126,13 @@ keeps about 124 MiB.
|
||||
RPC and signal shutdown share a five-second process-exit deadline, armed before terminal teardown.
|
||||
A stalled teardown must not leave the endpoint retired while the launchd job stays alive forever.
|
||||
|
||||
An explicit, untargeted Restart also recovers signed-helper jobs left by older builds after they
|
||||
retired their PID record and endpoint but failed to exit. Recovery uses exact launchd labels and
|
||||
arguments for this profile, ignores newer protocols, and preserves live or unreadable endpoints
|
||||
and services still publishing a PID record. It verifies descendant exits before killing the old
|
||||
root and releasing its label. Startup never runs this cleanup, and a failed recovery is logged
|
||||
without blocking the current daemon's restart. Node/SSH hosts do not perform GUI-service recovery.
|
||||
|
||||
Running old daemons keep their terminals; settings reports app and terminal-host Full Disk Access
|
||||
separately, and a denied terminal is recovered by closing it or restarting the daemon. The packaging
|
||||
hook also gives the main bundle and every Helper the Desktop, Documents and Downloads descriptions,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { PROTOCOL_VERSION } from './types'
|
||||
const { recoverRetired } = vi.hoisted(() => ({ recoverRetired: vi.fn(async () => {}) }))
|
||||
vi.mock('./macos-retired-daemon-recovery', () => ({ recoverRetiredMacDaemonJobs: recoverRetired }))
|
||||
|
||||
const {
|
||||
probeSocketExistsMock,
|
||||
@@ -45,6 +47,7 @@ vi.mock('../ipc/pty/provider/listener-lifecycle', () => moduleFactories.ptyListe
|
||||
|
||||
describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
beforeEach(() => {
|
||||
recoverRetired.mockReset().mockResolvedValue(undefined)
|
||||
installDefaultNetConnectStub()
|
||||
})
|
||||
|
||||
@@ -79,6 +82,22 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
})
|
||||
// Ordering invariant: synthetic exits must reach the renderer *before* listeners are torn down (Step 1 before 2).
|
||||
expect(order).toEqual(['fanout', 'unbind'])
|
||||
expect(recoverRetired).toHaveBeenCalledOnce()
|
||||
expect(recoverRetired.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
originalAdapter.fanoutSyntheticExits.mock.invocationCallOrder[0]
|
||||
)
|
||||
})
|
||||
|
||||
it('still restarts the current daemon when retired-daemon recovery cannot prove cleanup', async () => {
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
recoverRetired.mockRejectedValueOnce(new Error('retired descendants unverifiable'))
|
||||
await expect(mod.restartDaemon()).resolves.toBeDefined()
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining('retired descendants unverifiable'))
|
||||
expect(adapterInstances[0].fanoutSyntheticExits).toHaveBeenCalled()
|
||||
expect(spawnerInstances[0].resetHandle).toHaveBeenCalled()
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('refuses a targeted restart before ending anything when another daemon owns the endpoint', async () => {
|
||||
@@ -101,6 +120,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
|
||||
expect(originalAdapter.fanoutSyntheticExits).not.toHaveBeenCalled()
|
||||
expect(unbindLocalProviderListenersMock).not.toHaveBeenCalled()
|
||||
expect(recoverRetired).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses daemon-owned idle retirement after a failed manual-restart adoption', async () => {
|
||||
|
||||
@@ -42,6 +42,7 @@ import {
|
||||
import { DaemonClient } from './client'
|
||||
import { clearSeveredDaemon } from './daemon-host-image-state'
|
||||
import { PROTOCOL_VERSION } from './types'
|
||||
import { recoverRetiredMacDaemonJobs } from './macos-retired-daemon-recovery'
|
||||
|
||||
export type { RestartDaemonResult } from './daemon-restart-state'
|
||||
|
||||
@@ -60,6 +61,14 @@ async function runRestartDaemon(expectedDaemonPid?: number): Promise<RestartDaem
|
||||
}
|
||||
|
||||
const runtimeDir = getRuntimeDir()
|
||||
if (expectedDaemonPid === undefined) {
|
||||
// Why not abort: a wedged leftover must never disable the restart that replaces the current daemon.
|
||||
await recoverRetiredMacDaemonJobs(runtimeDir).catch((error: unknown) => {
|
||||
console.warn(
|
||||
`[daemon] Retired macOS terminal service recovery failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
})
|
||||
}
|
||||
if (expectedDaemonPid !== undefined) {
|
||||
// Why before any teardown: a targeted restart must not end a single terminal when the
|
||||
// endpoint now belongs to a replacement daemon.
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { resolve } from 'node:path'
|
||||
import { PROTOCOL_VERSION } from './daemon-protocol-version'
|
||||
|
||||
/** One job per profile and protocol, including jobs retained across app upgrades. */
|
||||
export function macDaemonJobLabel(
|
||||
userDataPath: string,
|
||||
protocolVersion = PROTOCOL_VERSION
|
||||
): string {
|
||||
const profileKey = createHash('sha256').update(resolve(userDataPath)).digest('hex').slice(0, 16)
|
||||
return `com.stablyai.orca.terminal.${profileKey}.v${protocolVersion}`
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { writeFile } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { join } from 'node:path'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { getAppEnvironment } from '../../shared/app-environment'
|
||||
import { LOCAL_BUILD_COMPATIBILITY_CONTRACT } from '../../shared/local-build-compatibility-contract'
|
||||
@@ -11,7 +10,7 @@ import { compareDottedVersions } from '../orcad/native-host-abi'
|
||||
import { DaemonClient } from './client'
|
||||
import { DaemonEndpointOwnershipError, holdDaemonAdoptionLease } from './daemon-endpoint-adoption'
|
||||
import { buildDaemonScriptArgs, type DaemonChildSpawnOptions } from './daemon-launched-child-spawn'
|
||||
import { PROTOCOL_VERSION } from './daemon-protocol-version'
|
||||
import { macDaemonJobLabel } from './macos-daemon-job-label'
|
||||
import type { DaemonProcessHandle } from './daemon-spawner'
|
||||
import { DAEMON_SHUTDOWN_TIMEOUT_MS } from './daemon-shutdown-deadline'
|
||||
import {
|
||||
@@ -43,11 +42,7 @@ const BOOTSTRAP_TIMEOUT_MS = 10_000
|
||||
const STARTUP_TIMEOUT_MS = 10_000
|
||||
const JOB_CHECK_INTERVAL_MS = 250
|
||||
|
||||
/** One job per profile and protocol, matching the one daemon each of them may run. */
|
||||
export function macDaemonJobLabel(userDataPath: string): string {
|
||||
const profileKey = createHash('sha256').update(resolve(userDataPath)).digest('hex').slice(0, 16)
|
||||
return `com.stablyai.orca.terminal.${profileKey}.v${PROTOCOL_VERSION}`
|
||||
}
|
||||
export { macDaemonJobLabel } from './macos-daemon-job-label'
|
||||
|
||||
function buildMacDaemonLaunchJob(
|
||||
options: MacDaemonLaunchOptions,
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { beforeEach, expect, it, vi } from 'vitest'
|
||||
import type { ProcessResult, ProcessSpec } from '@orca/process-host/process-spec'
|
||||
import { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from './daemon-spawner'
|
||||
import { PROTOCOL_VERSION } from './daemon-protocol-version'
|
||||
import { macDaemonJobLabel } from './macos-daemon-job-label'
|
||||
|
||||
const { run } = vi.hoisted(() => ({ run: vi.fn<(spec: ProcessSpec) => Promise<ProcessResult>>() }))
|
||||
vi.mock('@orca/process-host', () => ({ runProcess: run }))
|
||||
import { listMacDaemonJobVersions, readMacDaemonRecoveryJob } from './macos-retired-daemon-jobs'
|
||||
|
||||
const ok: ProcessResult = { code: 0, signal: null, stdout: '', stderr: '', timedOut: false }
|
||||
const profile = '/profiles/a'
|
||||
const runtime = '/profiles/a/daemon'
|
||||
const version = PROTOCOL_VERSION - 1
|
||||
const args = [
|
||||
'/copy/host',
|
||||
'/copy/daemon-entry.js',
|
||||
'--socket',
|
||||
getDaemonSocketPath(runtime, version),
|
||||
'--token',
|
||||
getDaemonTokenPath(runtime, version),
|
||||
'--pid-record',
|
||||
getDaemonPidPath(runtime, version),
|
||||
'--launch-nonce',
|
||||
'old-nonce',
|
||||
'--login-session-watch'
|
||||
]
|
||||
const job = (argv = args): string =>
|
||||
`service = {\n\tstate = running\n\tpid = 12345\n\targuments = {\n${argv.map((v) => `\t\t${v}\n`).join('')}\t}\n}\n`
|
||||
|
||||
beforeEach(() => run.mockReset())
|
||||
|
||||
it('discovers only exact same-profile supported labels in one bounded command', async () => {
|
||||
run.mockResolvedValue({
|
||||
...ok,
|
||||
stdout: [
|
||||
`12345\t0\t${macDaemonJobLabel(profile, version)}`,
|
||||
`12346\t0\t${macDaemonJobLabel(profile)}`,
|
||||
`12347\t0\t${macDaemonJobLabel(profile, PROTOCOL_VERSION + 1)}`,
|
||||
`12348\t0\t${macDaemonJobLabel('/profiles/other', version)}`,
|
||||
`12349\t0\t${macDaemonJobLabel(profile, version)}.extra`
|
||||
].join('\n')
|
||||
})
|
||||
expect(await listMacDaemonJobVersions(profile)).toEqual([version, PROTOCOL_VERSION])
|
||||
expect(run).toHaveBeenCalledOnce()
|
||||
expect(run).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ args: ['list'], timeoutMs: 3_000, maxOutputBytes: 128 * 1024 })
|
||||
)
|
||||
})
|
||||
|
||||
it('requires exact endpoint arguments from the top-level launchd job', async () => {
|
||||
run.mockResolvedValue({ ...ok, stdout: job() })
|
||||
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toEqual({
|
||||
service: `gui/${process.getuid?.()}/${macDaemonJobLabel(profile, version)}`,
|
||||
protocolVersion: version,
|
||||
pid: 12345,
|
||||
launchNonce: 'old-nonce',
|
||||
socketPath: getDaemonSocketPath(runtime, version),
|
||||
pidPath: getDaemonPidPath(runtime, version)
|
||||
})
|
||||
})
|
||||
|
||||
it.each(
|
||||
[
|
||||
args.filter((a) => a !== '--login-session-watch'),
|
||||
[...args, '--socket', getDaemonSocketPath(runtime, version)],
|
||||
args.map((a) => (a === getDaemonSocketPath(runtime, version) ? `${a}.other` : a)),
|
||||
args.map((a) => (a === getDaemonTokenPath(runtime, version) ? '/another/token' : a)),
|
||||
args.map((a) => (a === getDaemonPidPath(runtime, version) ? '/another/pid' : a)),
|
||||
args.filter((a) => a !== 'old-nonce')
|
||||
].map((argv) => ({ argv }))
|
||||
)('rejects ambiguous or foreign ownership: $argv', async ({ argv }) => {
|
||||
run.mockResolvedValue({ ...ok, stdout: job(argv) })
|
||||
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull()
|
||||
})
|
||||
|
||||
it('ignores a job that exited between list and print', async () => {
|
||||
run.mockResolvedValue({ ...ok, code: 113, stderr: 'Could not find service "old"' })
|
||||
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull()
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ timedOut: true },
|
||||
{ outputTruncated: true },
|
||||
{ code: 1 },
|
||||
{ code: 113, stderr: 'Could not find service; Permission denied' }
|
||||
])('fails closed on unreadable or incomplete job output: %j', async (override) => {
|
||||
run.mockResolvedValue({ ...ok, stdout: job(), ...override })
|
||||
await expect(readMacDaemonRecoveryJob(profile, runtime, version)).rejects.toThrow(
|
||||
'Could not inspect'
|
||||
)
|
||||
})
|
||||
|
||||
it('never treats a nested state or pid as the running job', async () => {
|
||||
run.mockResolvedValue({
|
||||
...ok,
|
||||
stdout: job().replace('\tstate = running', '\t\tstate = running')
|
||||
})
|
||||
expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull()
|
||||
})
|
||||
|
||||
it('bounds job discovery and rejects an oversized profile inventory', async () => {
|
||||
run.mockResolvedValue({
|
||||
...ok,
|
||||
stdout: Array.from(
|
||||
{ length: 33 },
|
||||
(_, index) => `12345\t0\t${macDaemonJobLabel(profile, index + 1)}`
|
||||
).join('\n')
|
||||
})
|
||||
await expect(listMacDaemonJobVersions(profile)).rejects.toThrow('Too many')
|
||||
expect(run).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('never inspects newer or invalid protocol jobs', async () => {
|
||||
for (const unsupported of [0, -1, 1.5, PROTOCOL_VERSION + 1]) {
|
||||
expect(await readMacDaemonRecoveryJob(profile, runtime, unsupported)).toBeNull()
|
||||
}
|
||||
expect(run).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -0,0 +1,102 @@
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from './daemon-spawner'
|
||||
import { PROTOCOL_VERSION } from './daemon-protocol-version'
|
||||
import { macDaemonJobLabel } from './macos-daemon-job-label'
|
||||
|
||||
export type RetiredMacDaemonJob = {
|
||||
service: string
|
||||
protocolVersion: number
|
||||
pid: number
|
||||
launchNonce: string
|
||||
socketPath: string
|
||||
pidPath: string
|
||||
}
|
||||
|
||||
async function launchctl(args: string[]): Promise<string | null> {
|
||||
const result = await runProcess({
|
||||
program: '/bin/launchctl',
|
||||
args,
|
||||
timeoutMs: 3_000,
|
||||
maxOutputBytes: 128 * 1024
|
||||
})
|
||||
if (
|
||||
args[0] === 'print' &&
|
||||
result.code === 113 &&
|
||||
!result.timedOut &&
|
||||
!result.outputTruncated &&
|
||||
result.stderr.includes('Could not find service') &&
|
||||
!/Operation not permitted|Permission denied/i.test(result.stderr)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
if (result.code !== 0 || result.timedOut || result.outputTruncated) {
|
||||
throw new Error('Could not inspect macOS terminal services for restart recovery')
|
||||
}
|
||||
return result.stdout
|
||||
}
|
||||
|
||||
/** Reads only exact profile labels; newer protocols remain owned by their newer app. */
|
||||
export async function listMacDaemonJobVersions(userDataPath: string): Promise<number[]> {
|
||||
const prefix = macDaemonJobLabel(userDataPath, 0).slice(0, -1)
|
||||
const versions = new Set<number>()
|
||||
for (const line of (await launchctl(['list']))?.split('\n') ?? []) {
|
||||
const label = line.trim().split(/\s+/)[2]
|
||||
if (!label?.startsWith(prefix)) {
|
||||
continue
|
||||
}
|
||||
const suffix = label.slice(prefix.length)
|
||||
const version = Number(suffix)
|
||||
if (/^[1-9]\d*$/.test(suffix) && Number.isSafeInteger(version) && version <= PROTOCOL_VERSION) {
|
||||
versions.add(version)
|
||||
}
|
||||
}
|
||||
if (versions.size > 32) {
|
||||
throw new Error('Too many macOS terminal services to recover safely')
|
||||
}
|
||||
return [...versions]
|
||||
}
|
||||
|
||||
/** launchctl's structured argument block provides exact endpoint ownership, without parsing argv text. */
|
||||
export async function readMacDaemonRecoveryJob(
|
||||
userDataPath: string,
|
||||
runtimeDir: string,
|
||||
protocolVersion: number
|
||||
): Promise<RetiredMacDaemonJob | null> {
|
||||
if (
|
||||
!Number.isSafeInteger(protocolVersion) ||
|
||||
protocolVersion < 1 ||
|
||||
protocolVersion > PROTOCOL_VERSION
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const service = `gui/${process.getuid?.()}/${macDaemonJobLabel(userDataPath, protocolVersion)}`
|
||||
const output = await launchctl(['print', service])
|
||||
if (!output || !/^\tstate = running$/m.test(output)) {
|
||||
return null
|
||||
}
|
||||
const pid = Number(/^\tpid = (\d+)$/m.exec(output)?.[1])
|
||||
const block = /^\targuments = \{\n((?:\t\t[^\n]*\n)+)\t\}$/m.exec(output)?.[1]
|
||||
const args =
|
||||
block
|
||||
?.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => line.slice(2)) ?? []
|
||||
const argument = (flag: string): string | undefined =>
|
||||
args.filter((value) => value === flag).length === 1 ? args[args.indexOf(flag) + 1] : undefined
|
||||
const socketPath = getDaemonSocketPath(runtimeDir, protocolVersion)
|
||||
const pidPath = getDaemonPidPath(runtimeDir, protocolVersion)
|
||||
const launchNonce = argument('--launch-nonce')
|
||||
if (
|
||||
!Number.isSafeInteger(pid) ||
|
||||
pid <= 1 ||
|
||||
!launchNonce ||
|
||||
launchNonce.startsWith('--') ||
|
||||
!args.includes('--login-session-watch') ||
|
||||
argument('--socket') !== socketPath ||
|
||||
argument('--token') !== getDaemonTokenPath(runtimeDir, protocolVersion) ||
|
||||
argument('--pid-record') !== pidPath
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return { service, protocolVersion, pid, launchNonce, socketPath, pidPath }
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
import type { DescendantSnapshot } from '../pty-descendant-termination'
|
||||
import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification'
|
||||
import type { RetiredMacDaemonJob } from './macos-retired-daemon-jobs'
|
||||
import type { ProcessIdentity } from './daemon-process-identity-query'
|
||||
import type { SocketProbeOutcome } from './daemon-endpoint-probe'
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
packaged: true,
|
||||
stat: vi.fn(),
|
||||
versions: vi.fn<() => Promise<number[]>>(),
|
||||
job: vi.fn<() => Promise<RetiredMacDaemonJob | null>>(),
|
||||
probe: vi.fn<() => Promise<SocketProbeOutcome>>(),
|
||||
identity: vi.fn<() => Promise<ProcessIdentity | null>>(),
|
||||
capture: vi.fn<() => Promise<DescendantSnapshot | null>>(),
|
||||
terminate: vi.fn<() => Promise<DescendantTreeVerdict>>(),
|
||||
release: vi.fn<() => Promise<boolean>>()
|
||||
}))
|
||||
vi.mock('node:fs/promises', () => ({ lstat: mocks.stat }))
|
||||
vi.mock('../../shared/app-environment', () => ({
|
||||
getAppEnvironment: () => ({ isPackaged: () => mocks.packaged, getPath: () => '/profile' })
|
||||
}))
|
||||
vi.mock('../pty-descendant-termination', () => ({ captureDescendantSnapshot: mocks.capture }))
|
||||
vi.mock('./macos-retired-daemon-jobs', () => ({
|
||||
listMacDaemonJobVersions: mocks.versions,
|
||||
readMacDaemonRecoveryJob: mocks.job
|
||||
}))
|
||||
vi.mock('./daemon-endpoint-probe', () => ({
|
||||
endpointIsProvenDead: (outcome: string) => outcome === 'missing' || outcome === 'refused',
|
||||
probeSocketConnect: mocks.probe
|
||||
}))
|
||||
vi.mock('./daemon-process-identity-query', () => ({ getPsProcessIdentityAsync: mocks.identity }))
|
||||
vi.mock('./terminal-descendant-shutdown', () => ({ terminateShutdownDescendants: mocks.terminate }))
|
||||
vi.mock('./macos-daemon-job-state', () => ({ releaseMacDaemonJob: mocks.release }))
|
||||
import { recoverRetiredMacDaemonJobs } from './macos-retired-daemon-recovery'
|
||||
|
||||
const startedAt = 'Mon Oct 12 16:28:32 2026'
|
||||
const identity = { commandLine: 'daemon', startedAtMs: Date.parse(startedAt) }
|
||||
const job: RetiredMacDaemonJob = {
|
||||
service: 'gui/501/profile.v45',
|
||||
protocolVersion: 45,
|
||||
pid: 7621,
|
||||
launchNonce: 'old',
|
||||
socketPath: '/profile/daemon/v45.sock',
|
||||
pidPath: '/profile/daemon/v45.pid'
|
||||
}
|
||||
const snapshot: DescendantSnapshot = {
|
||||
root: { pid: job.pid, startedAt },
|
||||
rootPgid: job.pid,
|
||||
descendants: [{ pid: 88912, ppid: job.pid, pgid: 88912, startedAt }],
|
||||
capturedAtMs: Date.parse(startedAt) + 60_000
|
||||
}
|
||||
const typeDescriptor = Object.getOwnPropertyDescriptor(process, 'type')
|
||||
const kill = vi.fn<typeof process.kill>()
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks()
|
||||
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
|
||||
Object.defineProperty(process, 'type', { configurable: true, value: 'browser' })
|
||||
vi.spyOn(Date, 'now').mockReturnValue(identity.startedAtMs + 60_000)
|
||||
vi.spyOn(console, 'info').mockImplementation(() => {})
|
||||
kill.mockReturnValue(true)
|
||||
vi.spyOn(process, 'kill').mockImplementation(kill)
|
||||
mocks.packaged = true
|
||||
mocks.stat.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' }))
|
||||
mocks.versions.mockResolvedValue([45])
|
||||
mocks.job.mockResolvedValue(job)
|
||||
mocks.probe.mockResolvedValue('refused')
|
||||
mocks.identity.mockResolvedValue(identity)
|
||||
mocks.capture.mockResolvedValue(snapshot)
|
||||
mocks.terminate.mockResolvedValue('exited')
|
||||
mocks.release.mockResolvedValue(true)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
if (typeDescriptor) {
|
||||
Object.defineProperty(process, 'type', typeDescriptor)
|
||||
} else {
|
||||
Reflect.deleteProperty(process, 'type')
|
||||
}
|
||||
})
|
||||
|
||||
it('releases retired old-protocol conversation owners before returning from Restart', async () => {
|
||||
mocks.identity
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValueOnce(null)
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(mocks.terminate).toHaveBeenCalledExactlyOnceWith(snapshot)
|
||||
expect(kill.mock.calls).toEqual([[job.pid, 'SIGKILL']])
|
||||
expect(mocks.terminate.mock.invocationCallOrder[0]).toBeLessThan(kill.mock.invocationCallOrder[0])
|
||||
expect(mocks.release).toHaveBeenCalledExactlyOnceWith(job.service, Date.now() + 3_000)
|
||||
expect(mocks.versions).toHaveBeenCalledOnce()
|
||||
expect(mocks.capture).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it.each(['connected', 'unknown'] as const)('preserves endpoints that read %s', async (outcome) => {
|
||||
mocks.probe.mockResolvedValue(outcome)
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
expect(mocks.capture).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['present', 'permission-denied'] as const)(
|
||||
'preserves non-retired PID records: %s',
|
||||
async (state) => {
|
||||
if (state === 'present') {
|
||||
mocks.stat.mockResolvedValue({})
|
||||
} else {
|
||||
mocks.stat.mockRejectedValue(Object.assign(new Error('denied'), { code: 'EACCES' }))
|
||||
}
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
expect(mocks.probe).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
|
||||
it.each([null, { ...identity, startedAtMs: identity.startedAtMs + 59_000 }])(
|
||||
'preserves uninspectable or newly starting processes: %j',
|
||||
async (value) => {
|
||||
mocks.identity.mockResolvedValue(value)
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
|
||||
it('never signals a reused root PID or a replacement service', async () => {
|
||||
mocks.job.mockResolvedValueOnce(job).mockResolvedValueOnce({ ...job, launchNonce: 'new' })
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
mocks.job.mockResolvedValue(job)
|
||||
mocks.identity
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValueOnce({ ...identity, startedAtMs: 1 })
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('signals nothing and refuses success when process-table capture fails', async () => {
|
||||
mocks.capture.mockResolvedValue(null)
|
||||
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
expect(mocks.terminate).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rechecks retirement after capturing the descendant tree', async () => {
|
||||
mocks.probe.mockResolvedValueOnce('refused').mockResolvedValueOnce('connected')
|
||||
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
|
||||
expect(mocks.terminate).not.toHaveBeenCalled()
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('never sweeps a snapshot naming another root incarnation', async () => {
|
||||
mocks.capture.mockResolvedValue({
|
||||
...snapshot,
|
||||
root: { pid: job.pid, startedAt: 'Tue Oct 13 16:28:32 2026' }
|
||||
})
|
||||
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
|
||||
expect(mocks.terminate).not.toHaveBeenCalled()
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('preserves a service that republishes its PID record during inspection', async () => {
|
||||
mocks.stat
|
||||
.mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' }))
|
||||
.mockResolvedValue({})
|
||||
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify')
|
||||
expect(mocks.terminate).not.toHaveBeenCalled()
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['live', 'unverifiable'] as const)(
|
||||
'fails closed when descendants are %s',
|
||||
async (verdict) => {
|
||||
mocks.terminate.mockResolvedValue(verdict)
|
||||
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow(verdict)
|
||||
expect(kill).not.toHaveBeenCalled()
|
||||
expect(mocks.release).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
|
||||
it('never signals a root recycled during descendant teardown', async () => {
|
||||
mocks.identity
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValue({ ...identity, startedAtMs: 1 })
|
||||
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('changed')
|
||||
expect(kill.mock.calls).toEqual([[job.pid, 0]])
|
||||
expect(mocks.release).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not report success while a service registration remains', async () => {
|
||||
mocks.release.mockResolvedValue(false)
|
||||
await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not release')
|
||||
})
|
||||
|
||||
it('accepts a root that exits naturally after its descendants', async () => {
|
||||
mocks.identity
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValueOnce(identity)
|
||||
.mockResolvedValueOnce(null)
|
||||
kill.mockImplementation(() => {
|
||||
throw Object.assign(new Error('gone'), { code: 'ESRCH' })
|
||||
})
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(kill.mock.calls).toEqual([[job.pid, 0]])
|
||||
expect(mocks.release).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it.each(['linux', 'win32'] as const)('does no discovery on %s', async (platform) => {
|
||||
vi.spyOn(process, 'platform', 'get').mockReturnValue(platform)
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(mocks.versions).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does no discovery for Node/SSH execution hosts or unpackaged builds', async () => {
|
||||
Object.defineProperty(process, 'type', { configurable: true, value: undefined })
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
Object.defineProperty(process, 'type', { configurable: true, value: 'browser' })
|
||||
mocks.packaged = false
|
||||
await recoverRetiredMacDaemonJobs('/profile/daemon')
|
||||
expect(mocks.versions).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -0,0 +1,110 @@
|
||||
import { lstat } from 'node:fs/promises'
|
||||
import { getAppEnvironment } from '../../shared/app-environment'
|
||||
import { captureDescendantSnapshot } from '../pty-descendant-termination'
|
||||
import { endpointIsProvenDead, probeSocketConnect } from './daemon-endpoint-probe'
|
||||
import { getPsProcessIdentityAsync } from './daemon-process-identity-query'
|
||||
import { hasErrorCode, inspectProcessLiveness } from './daemon-process-inspection'
|
||||
import { releaseMacDaemonJob } from './macos-daemon-job-state'
|
||||
import {
|
||||
listMacDaemonJobVersions,
|
||||
readMacDaemonRecoveryJob,
|
||||
type RetiredMacDaemonJob
|
||||
} from './macos-retired-daemon-jobs'
|
||||
import { terminateShutdownDescendants } from './terminal-descendant-shutdown'
|
||||
|
||||
async function pidRecordWasRetired(path: string): Promise<boolean> {
|
||||
try {
|
||||
await lstat(path)
|
||||
return false
|
||||
} catch (error) {
|
||||
return hasErrorCode(error, 'ENOENT')
|
||||
}
|
||||
}
|
||||
|
||||
async function endpointWasRetired(job: RetiredMacDaemonJob): Promise<boolean> {
|
||||
return (
|
||||
(await pidRecordWasRetired(job.pidPath)) &&
|
||||
endpointIsProvenDead(await probeSocketConnect(job.socketPath))
|
||||
)
|
||||
}
|
||||
|
||||
async function sameJob(
|
||||
job: RetiredMacDaemonJob,
|
||||
userDataPath: string,
|
||||
runtimeDir: string
|
||||
): Promise<boolean> {
|
||||
const current = await readMacDaemonRecoveryJob(userDataPath, runtimeDir, job.protocolVersion)
|
||||
return current?.pid === job.pid && current.launchNonce === job.launchNonce
|
||||
}
|
||||
|
||||
async function recoverJob(
|
||||
job: RetiredMacDaemonJob,
|
||||
userDataPath: string,
|
||||
runtimeDir: string
|
||||
): Promise<void> {
|
||||
if (!(await endpointWasRetired(job))) {
|
||||
return
|
||||
}
|
||||
const identity = await getPsProcessIdentityAsync(job.pid)
|
||||
// A newly bootstrapping service may not have published yet; it is never a recovery target.
|
||||
if (!identity?.startedAtMs || Date.now() - identity.startedAtMs < 10_000) {
|
||||
return
|
||||
}
|
||||
if (!(await sameJob(job, userDataPath, runtimeDir))) {
|
||||
return
|
||||
}
|
||||
const beforeCapture = await getPsProcessIdentityAsync(job.pid)
|
||||
if (beforeCapture?.startedAtMs !== identity.startedAtMs) {
|
||||
return
|
||||
}
|
||||
// Capture before signalling; let the daemon reap children while their exits are verified.
|
||||
const snapshot = await captureDescendantSnapshot(job.pid)
|
||||
if (
|
||||
!snapshot?.root ||
|
||||
Date.parse(snapshot.root.startedAt) !== identity.startedAtMs ||
|
||||
!(await sameJob(job, userDataPath, runtimeDir)) ||
|
||||
!(await endpointWasRetired(job))
|
||||
) {
|
||||
throw new Error('Could not verify the retired macOS terminal service for recovery')
|
||||
}
|
||||
const verdict = await terminateShutdownDescendants(snapshot)
|
||||
if (verdict !== 'exited') {
|
||||
throw new Error(`Retired macOS terminal service descendants are ${verdict}`)
|
||||
}
|
||||
const remaining = await getPsProcessIdentityAsync(job.pid)
|
||||
if (remaining?.startedAtMs === identity.startedAtMs) {
|
||||
try {
|
||||
process.kill(job.pid, 'SIGKILL')
|
||||
} catch (error) {
|
||||
if (!hasErrorCode(error, 'ESRCH')) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
} else if (inspectProcessLiveness(job.pid).status !== 'exited') {
|
||||
throw new Error('Retired macOS terminal service changed during recovery')
|
||||
}
|
||||
if (!(await releaseMacDaemonJob(job.service, Date.now() + 3_000))) {
|
||||
throw new Error('Could not release the retired macOS terminal service registration')
|
||||
}
|
||||
console.info(
|
||||
`[daemon] Recovered retired macOS terminal service protocol=${job.protocolVersion} pid=${job.pid} descendants=${snapshot.descendants.length}`
|
||||
)
|
||||
}
|
||||
|
||||
/** Explicit Restart only: healthy legacy daemons and newer app protocols retain their sessions. */
|
||||
export async function recoverRetiredMacDaemonJobs(runtimeDir: string): Promise<void> {
|
||||
if (
|
||||
process.platform !== 'darwin' ||
|
||||
process.type !== 'browser' ||
|
||||
!getAppEnvironment().isPackaged()
|
||||
) {
|
||||
return
|
||||
}
|
||||
const userDataPath = getAppEnvironment().getPath('userData')
|
||||
for (const version of await listMacDaemonJobVersions(userDataPath)) {
|
||||
const job = await readMacDaemonRecoveryJob(userDataPath, runtimeDir, version)
|
||||
if (job) {
|
||||
await recoverJob(job, userDataPath, runtimeDir)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user