diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 30fbf9336e7..027e8dfd907 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10,6 +10,88 @@ } }, "gates": [ + { + "id": "terminal-provider.retired-macos-daemon-recovery", + "title": "Restart releases retired macOS daemon conversation owners across upgrades", + "maturity": "experimental", + "protection": "partial", + "owner": "daemon-terminal", + "layer": "macos-daemon-lifecycle", + "surfaces": [ + "Manage Sessions Restart", + "agent conversation locks", + "legacy terminal daemons" + ], + "platforms": ["macos"], + "providers": ["local-daemon"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local-daemon"], + "coverageNotes": "Packaged GUI recovery is tested with deterministic job/process evidence and an isolated native launchd service holding a real advisory writer lock. Linux, Windows, Node/SSH hosts, WSL and mobile/relay do not execute GUI-service recovery; healthy legacy routing is unchanged.", + "motivatingLinks": ["https://github.com/stablyai/orca/pull/29130"], + "invariant": "An explicit Restart may recover same-profile GUI services with retired endpoint ownership, including older protocols, only after exact job and root identity checks. It must verify descendants exited before releasing the job. Healthy, newer, foreign-profile, starting or unverifiable owners must not be stopped. Recovery failure is logged and must not block restarting the current daemon.", + "oracle": "Recover an old-protocol conversation owner and require descendants to exit before the root is killed and its label released. Reject connected/unknown endpoints, present/unreadable ownership records, new processes, replaced jobs, recycled roots and failed descendant verification. Require recovery before current-session fanout, and a completed current-daemon restart if recovery fails.", + "commands": [ + "ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/daemon/macos-retired-daemon-jobs.test.ts src/main/daemon/macos-retired-daemon-recovery.test.ts src/main/daemon/daemon-init-restart-sequence.test.ts src/main/daemon/terminal-descendant-shutdown.test.ts src/main/daemon/macos-daemon-launchd.test.ts" + ], + "testFiles": [ + "src/main/daemon/macos-retired-daemon-jobs.test.ts", + "src/main/daemon/macos-retired-daemon-recovery.test.ts", + "src/main/daemon/daemon-init-restart-sequence.test.ts", + "src/main/daemon/terminal-descendant-shutdown.test.ts", + "src/main/daemon/macos-daemon-launchd.test.ts" + ], + "assertionRefs": [ + { + "file": "src/main/daemon/macos-retired-daemon-recovery.test.ts", + "assertions": [ + "releases retired old-protocol conversation owners before returning from Restart", + "never signals a root recycled during descendant teardown" + ] + }, + { + "file": "src/main/daemon/daemon-init-restart-sequence.test.ts", + "assertions": [ + "still restarts the current daemon when retired-daemon recovery cannot prove cleanup" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-10-15", + "runner": "local", + "platform": "macos", + "command": "ORCA_BACKGROUND_LAUNCH=1 pnpm test src/main/daemon/macos-retired-daemon-jobs.test.ts src/main/daemon/macos-retired-daemon-recovery.test.ts src/main/daemon/daemon-init-restart-sequence.test.ts src/main/daemon/terminal-descendant-shutdown.test.ts src/main/daemon/macos-daemon-launchd.test.ts", + "result": "passed", + "durationSeconds": 2.27, + "summary": "79 tests passed. A separate native launchd fixture preserved its fresh service, then recovered the retired tree and released a real writer lock in 306 ms. Field recovery released 80 locks held by the previous daemon." + } + ], + "runtimeBudget": { + "p95Seconds": 5, + "scope": "deterministic retired-job recovery and restart contract" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "Focused local tests and native smoke pass; CI soak history is not yet collected." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "The installed build restarted protocol 46 while a retired protocol 45 process retained 80 writer locks. The isolated native fixture now releases its real writer lock; automated baseline mutation evidence is not collected." + }, + "performanceBudget": { + "required": true, + "evidence": "No startup work or periodic scans. One bounded launchctl list per explicit restart; at most 32 same-profile supported services. Only verified retired candidates invoke the existing bounded descendant verifier. Tests assert one discovery and one initial descendant capture per recovered job." + }, + "promotionCriteria": [ + "Collect 100 focused CI passes or 14 days of soak history.", + "Capture packaged upgrade and Restart recovery with multiple retired jobs." + ], + "knownGaps": [ + "Full packaged GUI upgrade journey and multi-job native recovery remain uncollected.", + "Recovery is specific to signed-helper launchd jobs; fork-launched and already reparented processes have no equivalent retained ownership evidence." + ], + "demotionRule": "Keep experimental or demote if recovery signals an unverified owner, treats loss of contact as exit, reports success before descendant exit, adds background scanning, or lets a recovery failure block the current-daemon restart." + }, { "id": "agent-session.delayed-terminal-launch", "title": "Delayed terminal launches retain their harness preparation and recover process discovery", diff --git a/docs/reference/macos-daemon-tcc-attribution.md b/docs/reference/macos-daemon-tcc-attribution.md index f180c029e44..a160b853183 100644 --- a/docs/reference/macos-daemon-tcc-attribution.md +++ b/docs/reference/macos-daemon-tcc-attribution.md @@ -126,6 +126,13 @@ keeps about 124 MiB. RPC and signal shutdown share a five-second process-exit deadline, armed before terminal teardown. A stalled teardown must not leave the endpoint retired while the launchd job stays alive forever. +An explicit, untargeted Restart also recovers signed-helper jobs left by older builds after they +retired their PID record and endpoint but failed to exit. Recovery uses exact launchd labels and +arguments for this profile, ignores newer protocols, and preserves live or unreadable endpoints +and services still publishing a PID record. It verifies descendant exits before killing the old +root and releasing its label. Startup never runs this cleanup, and a failed recovery is logged +without blocking the current daemon's restart. Node/SSH hosts do not perform GUI-service recovery. + Running old daemons keep their terminals; settings reports app and terminal-host Full Disk Access separately, and a denied terminal is recovered by closing it or restarting the daemon. The packaging hook also gives the main bundle and every Helper the Desktop, Documents and Downloads descriptions, diff --git a/src/main/daemon/daemon-init-restart-sequence.test.ts b/src/main/daemon/daemon-init-restart-sequence.test.ts index e9cfcd704e9..59ffbf0d704 100644 --- a/src/main/daemon/daemon-init-restart-sequence.test.ts +++ b/src/main/daemon/daemon-init-restart-sequence.test.ts @@ -1,5 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { PROTOCOL_VERSION } from './types' +const { recoverRetired } = vi.hoisted(() => ({ recoverRetired: vi.fn(async () => {}) })) +vi.mock('./macos-retired-daemon-recovery', () => ({ recoverRetiredMacDaemonJobs: recoverRetired })) const { probeSocketExistsMock, @@ -45,6 +47,7 @@ vi.mock('../ipc/pty/provider/listener-lifecycle', () => moduleFactories.ptyListe describe('daemon-init: runRestartDaemon (7-step sequence)', () => { beforeEach(() => { + recoverRetired.mockReset().mockResolvedValue(undefined) installDefaultNetConnectStub() }) @@ -79,6 +82,22 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => { }) // Ordering invariant: synthetic exits must reach the renderer *before* listeners are torn down (Step 1 before 2). expect(order).toEqual(['fanout', 'unbind']) + expect(recoverRetired).toHaveBeenCalledOnce() + expect(recoverRetired.mock.invocationCallOrder[0]).toBeLessThan( + originalAdapter.fanoutSyntheticExits.mock.invocationCallOrder[0] + ) + }) + + it('still restarts the current daemon when retired-daemon recovery cannot prove cleanup', async () => { + const mod = await importFresh() + await mod.initDaemonPtyProvider() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + recoverRetired.mockRejectedValueOnce(new Error('retired descendants unverifiable')) + await expect(mod.restartDaemon()).resolves.toBeDefined() + expect(warn).toHaveBeenCalledWith(expect.stringContaining('retired descendants unverifiable')) + expect(adapterInstances[0].fanoutSyntheticExits).toHaveBeenCalled() + expect(spawnerInstances[0].resetHandle).toHaveBeenCalled() + warn.mockRestore() }) it('refuses a targeted restart before ending anything when another daemon owns the endpoint', async () => { @@ -101,6 +120,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => { expect(originalAdapter.fanoutSyntheticExits).not.toHaveBeenCalled() expect(unbindLocalProviderListenersMock).not.toHaveBeenCalled() + expect(recoverRetired).not.toHaveBeenCalled() }) it('uses daemon-owned idle retirement after a failed manual-restart adoption', async () => { diff --git a/src/main/daemon/daemon-provider-restart.ts b/src/main/daemon/daemon-provider-restart.ts index 83f476c8396..5a279e51fa8 100644 --- a/src/main/daemon/daemon-provider-restart.ts +++ b/src/main/daemon/daemon-provider-restart.ts @@ -42,6 +42,7 @@ import { import { DaemonClient } from './client' import { clearSeveredDaemon } from './daemon-host-image-state' import { PROTOCOL_VERSION } from './types' +import { recoverRetiredMacDaemonJobs } from './macos-retired-daemon-recovery' export type { RestartDaemonResult } from './daemon-restart-state' @@ -60,6 +61,14 @@ async function runRestartDaemon(expectedDaemonPid?: number): Promise { + console.warn( + `[daemon] Retired macOS terminal service recovery failed: ${error instanceof Error ? error.message : String(error)}` + ) + }) + } if (expectedDaemonPid !== undefined) { // Why before any teardown: a targeted restart must not end a single terminal when the // endpoint now belongs to a replacement daemon. diff --git a/src/main/daemon/macos-daemon-job-label.ts b/src/main/daemon/macos-daemon-job-label.ts new file mode 100644 index 00000000000..80b436d2c3f --- /dev/null +++ b/src/main/daemon/macos-daemon-job-label.ts @@ -0,0 +1,12 @@ +import { createHash } from 'node:crypto' +import { resolve } from 'node:path' +import { PROTOCOL_VERSION } from './daemon-protocol-version' + +/** One job per profile and protocol, including jobs retained across app upgrades. */ +export function macDaemonJobLabel( + userDataPath: string, + protocolVersion = PROTOCOL_VERSION +): string { + const profileKey = createHash('sha256').update(resolve(userDataPath)).digest('hex').slice(0, 16) + return `com.stablyai.orca.terminal.${profileKey}.v${protocolVersion}` +} diff --git a/src/main/daemon/macos-daemon-launchd.ts b/src/main/daemon/macos-daemon-launchd.ts index 2feb0cf4d66..d6de5b16a6b 100644 --- a/src/main/daemon/macos-daemon-launchd.ts +++ b/src/main/daemon/macos-daemon-launchd.ts @@ -1,6 +1,5 @@ -import { createHash } from 'node:crypto' import { writeFile } from 'node:fs/promises' -import { join, resolve } from 'node:path' +import { join } from 'node:path' import { setTimeout as delay } from 'node:timers/promises' import { getAppEnvironment } from '../../shared/app-environment' import { LOCAL_BUILD_COMPATIBILITY_CONTRACT } from '../../shared/local-build-compatibility-contract' @@ -11,7 +10,7 @@ import { compareDottedVersions } from '../orcad/native-host-abi' import { DaemonClient } from './client' import { DaemonEndpointOwnershipError, holdDaemonAdoptionLease } from './daemon-endpoint-adoption' import { buildDaemonScriptArgs, type DaemonChildSpawnOptions } from './daemon-launched-child-spawn' -import { PROTOCOL_VERSION } from './daemon-protocol-version' +import { macDaemonJobLabel } from './macos-daemon-job-label' import type { DaemonProcessHandle } from './daemon-spawner' import { DAEMON_SHUTDOWN_TIMEOUT_MS } from './daemon-shutdown-deadline' import { @@ -43,11 +42,7 @@ const BOOTSTRAP_TIMEOUT_MS = 10_000 const STARTUP_TIMEOUT_MS = 10_000 const JOB_CHECK_INTERVAL_MS = 250 -/** One job per profile and protocol, matching the one daemon each of them may run. */ -export function macDaemonJobLabel(userDataPath: string): string { - const profileKey = createHash('sha256').update(resolve(userDataPath)).digest('hex').slice(0, 16) - return `com.stablyai.orca.terminal.${profileKey}.v${PROTOCOL_VERSION}` -} +export { macDaemonJobLabel } from './macos-daemon-job-label' function buildMacDaemonLaunchJob( options: MacDaemonLaunchOptions, diff --git a/src/main/daemon/macos-retired-daemon-jobs.test.ts b/src/main/daemon/macos-retired-daemon-jobs.test.ts new file mode 100644 index 00000000000..faed86d9e09 --- /dev/null +++ b/src/main/daemon/macos-retired-daemon-jobs.test.ts @@ -0,0 +1,119 @@ +import { beforeEach, expect, it, vi } from 'vitest' +import type { ProcessResult, ProcessSpec } from '@orca/process-host/process-spec' +import { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from './daemon-spawner' +import { PROTOCOL_VERSION } from './daemon-protocol-version' +import { macDaemonJobLabel } from './macos-daemon-job-label' + +const { run } = vi.hoisted(() => ({ run: vi.fn<(spec: ProcessSpec) => Promise>() })) +vi.mock('@orca/process-host', () => ({ runProcess: run })) +import { listMacDaemonJobVersions, readMacDaemonRecoveryJob } from './macos-retired-daemon-jobs' + +const ok: ProcessResult = { code: 0, signal: null, stdout: '', stderr: '', timedOut: false } +const profile = '/profiles/a' +const runtime = '/profiles/a/daemon' +const version = PROTOCOL_VERSION - 1 +const args = [ + '/copy/host', + '/copy/daemon-entry.js', + '--socket', + getDaemonSocketPath(runtime, version), + '--token', + getDaemonTokenPath(runtime, version), + '--pid-record', + getDaemonPidPath(runtime, version), + '--launch-nonce', + 'old-nonce', + '--login-session-watch' +] +const job = (argv = args): string => + `service = {\n\tstate = running\n\tpid = 12345\n\targuments = {\n${argv.map((v) => `\t\t${v}\n`).join('')}\t}\n}\n` + +beforeEach(() => run.mockReset()) + +it('discovers only exact same-profile supported labels in one bounded command', async () => { + run.mockResolvedValue({ + ...ok, + stdout: [ + `12345\t0\t${macDaemonJobLabel(profile, version)}`, + `12346\t0\t${macDaemonJobLabel(profile)}`, + `12347\t0\t${macDaemonJobLabel(profile, PROTOCOL_VERSION + 1)}`, + `12348\t0\t${macDaemonJobLabel('/profiles/other', version)}`, + `12349\t0\t${macDaemonJobLabel(profile, version)}.extra` + ].join('\n') + }) + expect(await listMacDaemonJobVersions(profile)).toEqual([version, PROTOCOL_VERSION]) + expect(run).toHaveBeenCalledOnce() + expect(run).toHaveBeenCalledWith( + expect.objectContaining({ args: ['list'], timeoutMs: 3_000, maxOutputBytes: 128 * 1024 }) + ) +}) + +it('requires exact endpoint arguments from the top-level launchd job', async () => { + run.mockResolvedValue({ ...ok, stdout: job() }) + expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toEqual({ + service: `gui/${process.getuid?.()}/${macDaemonJobLabel(profile, version)}`, + protocolVersion: version, + pid: 12345, + launchNonce: 'old-nonce', + socketPath: getDaemonSocketPath(runtime, version), + pidPath: getDaemonPidPath(runtime, version) + }) +}) + +it.each( + [ + args.filter((a) => a !== '--login-session-watch'), + [...args, '--socket', getDaemonSocketPath(runtime, version)], + args.map((a) => (a === getDaemonSocketPath(runtime, version) ? `${a}.other` : a)), + args.map((a) => (a === getDaemonTokenPath(runtime, version) ? '/another/token' : a)), + args.map((a) => (a === getDaemonPidPath(runtime, version) ? '/another/pid' : a)), + args.filter((a) => a !== 'old-nonce') + ].map((argv) => ({ argv })) +)('rejects ambiguous or foreign ownership: $argv', async ({ argv }) => { + run.mockResolvedValue({ ...ok, stdout: job(argv) }) + expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull() +}) + +it('ignores a job that exited between list and print', async () => { + run.mockResolvedValue({ ...ok, code: 113, stderr: 'Could not find service "old"' }) + expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull() +}) + +it.each([ + { timedOut: true }, + { outputTruncated: true }, + { code: 1 }, + { code: 113, stderr: 'Could not find service; Permission denied' } +])('fails closed on unreadable or incomplete job output: %j', async (override) => { + run.mockResolvedValue({ ...ok, stdout: job(), ...override }) + await expect(readMacDaemonRecoveryJob(profile, runtime, version)).rejects.toThrow( + 'Could not inspect' + ) +}) + +it('never treats a nested state or pid as the running job', async () => { + run.mockResolvedValue({ + ...ok, + stdout: job().replace('\tstate = running', '\t\tstate = running') + }) + expect(await readMacDaemonRecoveryJob(profile, runtime, version)).toBeNull() +}) + +it('bounds job discovery and rejects an oversized profile inventory', async () => { + run.mockResolvedValue({ + ...ok, + stdout: Array.from( + { length: 33 }, + (_, index) => `12345\t0\t${macDaemonJobLabel(profile, index + 1)}` + ).join('\n') + }) + await expect(listMacDaemonJobVersions(profile)).rejects.toThrow('Too many') + expect(run).toHaveBeenCalledOnce() +}) + +it('never inspects newer or invalid protocol jobs', async () => { + for (const unsupported of [0, -1, 1.5, PROTOCOL_VERSION + 1]) { + expect(await readMacDaemonRecoveryJob(profile, runtime, unsupported)).toBeNull() + } + expect(run).not.toHaveBeenCalled() +}) diff --git a/src/main/daemon/macos-retired-daemon-jobs.ts b/src/main/daemon/macos-retired-daemon-jobs.ts new file mode 100644 index 00000000000..4df190cb78c --- /dev/null +++ b/src/main/daemon/macos-retired-daemon-jobs.ts @@ -0,0 +1,102 @@ +import { runProcess } from '@orca/process-host' +import { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from './daemon-spawner' +import { PROTOCOL_VERSION } from './daemon-protocol-version' +import { macDaemonJobLabel } from './macos-daemon-job-label' + +export type RetiredMacDaemonJob = { + service: string + protocolVersion: number + pid: number + launchNonce: string + socketPath: string + pidPath: string +} + +async function launchctl(args: string[]): Promise { + const result = await runProcess({ + program: '/bin/launchctl', + args, + timeoutMs: 3_000, + maxOutputBytes: 128 * 1024 + }) + if ( + args[0] === 'print' && + result.code === 113 && + !result.timedOut && + !result.outputTruncated && + result.stderr.includes('Could not find service') && + !/Operation not permitted|Permission denied/i.test(result.stderr) + ) { + return null + } + if (result.code !== 0 || result.timedOut || result.outputTruncated) { + throw new Error('Could not inspect macOS terminal services for restart recovery') + } + return result.stdout +} + +/** Reads only exact profile labels; newer protocols remain owned by their newer app. */ +export async function listMacDaemonJobVersions(userDataPath: string): Promise { + const prefix = macDaemonJobLabel(userDataPath, 0).slice(0, -1) + const versions = new Set() + for (const line of (await launchctl(['list']))?.split('\n') ?? []) { + const label = line.trim().split(/\s+/)[2] + if (!label?.startsWith(prefix)) { + continue + } + const suffix = label.slice(prefix.length) + const version = Number(suffix) + if (/^[1-9]\d*$/.test(suffix) && Number.isSafeInteger(version) && version <= PROTOCOL_VERSION) { + versions.add(version) + } + } + if (versions.size > 32) { + throw new Error('Too many macOS terminal services to recover safely') + } + return [...versions] +} + +/** launchctl's structured argument block provides exact endpoint ownership, without parsing argv text. */ +export async function readMacDaemonRecoveryJob( + userDataPath: string, + runtimeDir: string, + protocolVersion: number +): Promise { + if ( + !Number.isSafeInteger(protocolVersion) || + protocolVersion < 1 || + protocolVersion > PROTOCOL_VERSION + ) { + return null + } + const service = `gui/${process.getuid?.()}/${macDaemonJobLabel(userDataPath, protocolVersion)}` + const output = await launchctl(['print', service]) + if (!output || !/^\tstate = running$/m.test(output)) { + return null + } + const pid = Number(/^\tpid = (\d+)$/m.exec(output)?.[1]) + const block = /^\targuments = \{\n((?:\t\t[^\n]*\n)+)\t\}$/m.exec(output)?.[1] + const args = + block + ?.split('\n') + .filter(Boolean) + .map((line) => line.slice(2)) ?? [] + const argument = (flag: string): string | undefined => + args.filter((value) => value === flag).length === 1 ? args[args.indexOf(flag) + 1] : undefined + const socketPath = getDaemonSocketPath(runtimeDir, protocolVersion) + const pidPath = getDaemonPidPath(runtimeDir, protocolVersion) + const launchNonce = argument('--launch-nonce') + if ( + !Number.isSafeInteger(pid) || + pid <= 1 || + !launchNonce || + launchNonce.startsWith('--') || + !args.includes('--login-session-watch') || + argument('--socket') !== socketPath || + argument('--token') !== getDaemonTokenPath(runtimeDir, protocolVersion) || + argument('--pid-record') !== pidPath + ) { + return null + } + return { service, protocolVersion, pid, launchNonce, socketPath, pidPath } +} diff --git a/src/main/daemon/macos-retired-daemon-recovery.test.ts b/src/main/daemon/macos-retired-daemon-recovery.test.ts new file mode 100644 index 00000000000..fb4853e7b18 --- /dev/null +++ b/src/main/daemon/macos-retired-daemon-recovery.test.ts @@ -0,0 +1,225 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { DescendantSnapshot } from '../pty-descendant-termination' +import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification' +import type { RetiredMacDaemonJob } from './macos-retired-daemon-jobs' +import type { ProcessIdentity } from './daemon-process-identity-query' +import type { SocketProbeOutcome } from './daemon-endpoint-probe' + +const mocks = vi.hoisted(() => ({ + packaged: true, + stat: vi.fn(), + versions: vi.fn<() => Promise>(), + job: vi.fn<() => Promise>(), + probe: vi.fn<() => Promise>(), + identity: vi.fn<() => Promise>(), + capture: vi.fn<() => Promise>(), + terminate: vi.fn<() => Promise>(), + release: vi.fn<() => Promise>() +})) +vi.mock('node:fs/promises', () => ({ lstat: mocks.stat })) +vi.mock('../../shared/app-environment', () => ({ + getAppEnvironment: () => ({ isPackaged: () => mocks.packaged, getPath: () => '/profile' }) +})) +vi.mock('../pty-descendant-termination', () => ({ captureDescendantSnapshot: mocks.capture })) +vi.mock('./macos-retired-daemon-jobs', () => ({ + listMacDaemonJobVersions: mocks.versions, + readMacDaemonRecoveryJob: mocks.job +})) +vi.mock('./daemon-endpoint-probe', () => ({ + endpointIsProvenDead: (outcome: string) => outcome === 'missing' || outcome === 'refused', + probeSocketConnect: mocks.probe +})) +vi.mock('./daemon-process-identity-query', () => ({ getPsProcessIdentityAsync: mocks.identity })) +vi.mock('./terminal-descendant-shutdown', () => ({ terminateShutdownDescendants: mocks.terminate })) +vi.mock('./macos-daemon-job-state', () => ({ releaseMacDaemonJob: mocks.release })) +import { recoverRetiredMacDaemonJobs } from './macos-retired-daemon-recovery' + +const startedAt = 'Mon Oct 12 16:28:32 2026' +const identity = { commandLine: 'daemon', startedAtMs: Date.parse(startedAt) } +const job: RetiredMacDaemonJob = { + service: 'gui/501/profile.v45', + protocolVersion: 45, + pid: 7621, + launchNonce: 'old', + socketPath: '/profile/daemon/v45.sock', + pidPath: '/profile/daemon/v45.pid' +} +const snapshot: DescendantSnapshot = { + root: { pid: job.pid, startedAt }, + rootPgid: job.pid, + descendants: [{ pid: 88912, ppid: job.pid, pgid: 88912, startedAt }], + capturedAtMs: Date.parse(startedAt) + 60_000 +} +const typeDescriptor = Object.getOwnPropertyDescriptor(process, 'type') +const kill = vi.fn() + +beforeEach(() => { + vi.resetAllMocks() + vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin') + Object.defineProperty(process, 'type', { configurable: true, value: 'browser' }) + vi.spyOn(Date, 'now').mockReturnValue(identity.startedAtMs + 60_000) + vi.spyOn(console, 'info').mockImplementation(() => {}) + kill.mockReturnValue(true) + vi.spyOn(process, 'kill').mockImplementation(kill) + mocks.packaged = true + mocks.stat.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })) + mocks.versions.mockResolvedValue([45]) + mocks.job.mockResolvedValue(job) + mocks.probe.mockResolvedValue('refused') + mocks.identity.mockResolvedValue(identity) + mocks.capture.mockResolvedValue(snapshot) + mocks.terminate.mockResolvedValue('exited') + mocks.release.mockResolvedValue(true) +}) + +afterEach(() => { + vi.restoreAllMocks() + if (typeDescriptor) { + Object.defineProperty(process, 'type', typeDescriptor) + } else { + Reflect.deleteProperty(process, 'type') + } +}) + +it('releases retired old-protocol conversation owners before returning from Restart', async () => { + mocks.identity + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(null) + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(mocks.terminate).toHaveBeenCalledExactlyOnceWith(snapshot) + expect(kill.mock.calls).toEqual([[job.pid, 'SIGKILL']]) + expect(mocks.terminate.mock.invocationCallOrder[0]).toBeLessThan(kill.mock.invocationCallOrder[0]) + expect(mocks.release).toHaveBeenCalledExactlyOnceWith(job.service, Date.now() + 3_000) + expect(mocks.versions).toHaveBeenCalledOnce() + expect(mocks.capture).toHaveBeenCalledOnce() +}) + +it.each(['connected', 'unknown'] as const)('preserves endpoints that read %s', async (outcome) => { + mocks.probe.mockResolvedValue(outcome) + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(kill).not.toHaveBeenCalled() + expect(mocks.capture).not.toHaveBeenCalled() +}) + +it.each(['present', 'permission-denied'] as const)( + 'preserves non-retired PID records: %s', + async (state) => { + if (state === 'present') { + mocks.stat.mockResolvedValue({}) + } else { + mocks.stat.mockRejectedValue(Object.assign(new Error('denied'), { code: 'EACCES' })) + } + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(kill).not.toHaveBeenCalled() + expect(mocks.probe).not.toHaveBeenCalled() + } +) + +it.each([null, { ...identity, startedAtMs: identity.startedAtMs + 59_000 }])( + 'preserves uninspectable or newly starting processes: %j', + async (value) => { + mocks.identity.mockResolvedValue(value) + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(kill).not.toHaveBeenCalled() + } +) + +it('never signals a reused root PID or a replacement service', async () => { + mocks.job.mockResolvedValueOnce(job).mockResolvedValueOnce({ ...job, launchNonce: 'new' }) + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(kill).not.toHaveBeenCalled() + mocks.job.mockResolvedValue(job) + mocks.identity + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce({ ...identity, startedAtMs: 1 }) + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(kill).not.toHaveBeenCalled() +}) + +it('signals nothing and refuses success when process-table capture fails', async () => { + mocks.capture.mockResolvedValue(null) + await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify') + expect(kill).not.toHaveBeenCalled() + expect(mocks.terminate).not.toHaveBeenCalled() +}) + +it('rechecks retirement after capturing the descendant tree', async () => { + mocks.probe.mockResolvedValueOnce('refused').mockResolvedValueOnce('connected') + await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify') + expect(mocks.terminate).not.toHaveBeenCalled() + expect(kill).not.toHaveBeenCalled() +}) + +it('never sweeps a snapshot naming another root incarnation', async () => { + mocks.capture.mockResolvedValue({ + ...snapshot, + root: { pid: job.pid, startedAt: 'Tue Oct 13 16:28:32 2026' } + }) + await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify') + expect(mocks.terminate).not.toHaveBeenCalled() + expect(kill).not.toHaveBeenCalled() +}) + +it('preserves a service that republishes its PID record during inspection', async () => { + mocks.stat + .mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' })) + .mockResolvedValue({}) + await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not verify') + expect(mocks.terminate).not.toHaveBeenCalled() + expect(kill).not.toHaveBeenCalled() +}) + +it.each(['live', 'unverifiable'] as const)( + 'fails closed when descendants are %s', + async (verdict) => { + mocks.terminate.mockResolvedValue(verdict) + await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow(verdict) + expect(kill).not.toHaveBeenCalled() + expect(mocks.release).not.toHaveBeenCalled() + } +) + +it('never signals a root recycled during descendant teardown', async () => { + mocks.identity + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + .mockResolvedValue({ ...identity, startedAtMs: 1 }) + await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('changed') + expect(kill.mock.calls).toEqual([[job.pid, 0]]) + expect(mocks.release).not.toHaveBeenCalled() +}) + +it('does not report success while a service registration remains', async () => { + mocks.release.mockResolvedValue(false) + await expect(recoverRetiredMacDaemonJobs('/profile/daemon')).rejects.toThrow('Could not release') +}) + +it('accepts a root that exits naturally after its descendants', async () => { + mocks.identity + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(null) + kill.mockImplementation(() => { + throw Object.assign(new Error('gone'), { code: 'ESRCH' }) + }) + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(kill.mock.calls).toEqual([[job.pid, 0]]) + expect(mocks.release).toHaveBeenCalledOnce() +}) + +it.each(['linux', 'win32'] as const)('does no discovery on %s', async (platform) => { + vi.spyOn(process, 'platform', 'get').mockReturnValue(platform) + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(mocks.versions).not.toHaveBeenCalled() +}) + +it('does no discovery for Node/SSH execution hosts or unpackaged builds', async () => { + Object.defineProperty(process, 'type', { configurable: true, value: undefined }) + await recoverRetiredMacDaemonJobs('/profile/daemon') + Object.defineProperty(process, 'type', { configurable: true, value: 'browser' }) + mocks.packaged = false + await recoverRetiredMacDaemonJobs('/profile/daemon') + expect(mocks.versions).not.toHaveBeenCalled() +}) diff --git a/src/main/daemon/macos-retired-daemon-recovery.ts b/src/main/daemon/macos-retired-daemon-recovery.ts new file mode 100644 index 00000000000..6c7143731f1 --- /dev/null +++ b/src/main/daemon/macos-retired-daemon-recovery.ts @@ -0,0 +1,110 @@ +import { lstat } from 'node:fs/promises' +import { getAppEnvironment } from '../../shared/app-environment' +import { captureDescendantSnapshot } from '../pty-descendant-termination' +import { endpointIsProvenDead, probeSocketConnect } from './daemon-endpoint-probe' +import { getPsProcessIdentityAsync } from './daemon-process-identity-query' +import { hasErrorCode, inspectProcessLiveness } from './daemon-process-inspection' +import { releaseMacDaemonJob } from './macos-daemon-job-state' +import { + listMacDaemonJobVersions, + readMacDaemonRecoveryJob, + type RetiredMacDaemonJob +} from './macos-retired-daemon-jobs' +import { terminateShutdownDescendants } from './terminal-descendant-shutdown' + +async function pidRecordWasRetired(path: string): Promise { + try { + await lstat(path) + return false + } catch (error) { + return hasErrorCode(error, 'ENOENT') + } +} + +async function endpointWasRetired(job: RetiredMacDaemonJob): Promise { + return ( + (await pidRecordWasRetired(job.pidPath)) && + endpointIsProvenDead(await probeSocketConnect(job.socketPath)) + ) +} + +async function sameJob( + job: RetiredMacDaemonJob, + userDataPath: string, + runtimeDir: string +): Promise { + const current = await readMacDaemonRecoveryJob(userDataPath, runtimeDir, job.protocolVersion) + return current?.pid === job.pid && current.launchNonce === job.launchNonce +} + +async function recoverJob( + job: RetiredMacDaemonJob, + userDataPath: string, + runtimeDir: string +): Promise { + if (!(await endpointWasRetired(job))) { + return + } + const identity = await getPsProcessIdentityAsync(job.pid) + // A newly bootstrapping service may not have published yet; it is never a recovery target. + if (!identity?.startedAtMs || Date.now() - identity.startedAtMs < 10_000) { + return + } + if (!(await sameJob(job, userDataPath, runtimeDir))) { + return + } + const beforeCapture = await getPsProcessIdentityAsync(job.pid) + if (beforeCapture?.startedAtMs !== identity.startedAtMs) { + return + } + // Capture before signalling; let the daemon reap children while their exits are verified. + const snapshot = await captureDescendantSnapshot(job.pid) + if ( + !snapshot?.root || + Date.parse(snapshot.root.startedAt) !== identity.startedAtMs || + !(await sameJob(job, userDataPath, runtimeDir)) || + !(await endpointWasRetired(job)) + ) { + throw new Error('Could not verify the retired macOS terminal service for recovery') + } + const verdict = await terminateShutdownDescendants(snapshot) + if (verdict !== 'exited') { + throw new Error(`Retired macOS terminal service descendants are ${verdict}`) + } + const remaining = await getPsProcessIdentityAsync(job.pid) + if (remaining?.startedAtMs === identity.startedAtMs) { + try { + process.kill(job.pid, 'SIGKILL') + } catch (error) { + if (!hasErrorCode(error, 'ESRCH')) { + throw error + } + } + } else if (inspectProcessLiveness(job.pid).status !== 'exited') { + throw new Error('Retired macOS terminal service changed during recovery') + } + if (!(await releaseMacDaemonJob(job.service, Date.now() + 3_000))) { + throw new Error('Could not release the retired macOS terminal service registration') + } + console.info( + `[daemon] Recovered retired macOS terminal service protocol=${job.protocolVersion} pid=${job.pid} descendants=${snapshot.descendants.length}` + ) +} + +/** Explicit Restart only: healthy legacy daemons and newer app protocols retain their sessions. */ +export async function recoverRetiredMacDaemonJobs(runtimeDir: string): Promise { + if ( + process.platform !== 'darwin' || + process.type !== 'browser' || + !getAppEnvironment().isPackaged() + ) { + return + } + const userDataPath = getAppEnvironment().getPath('userData') + for (const version of await listMacDaemonJobVersions(userDataPath)) { + const job = await readMacDaemonRecoveryJob(userDataPath, runtimeDir, version) + if (job) { + await recoverJob(job, userDataPath, runtimeDir) + } + } +}