From fde0fe35a79ede7231aa5b032754cfd83b69bf06 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Mon, 12 Oct 2026 08:50:25 -0700 Subject: [PATCH] fix(daemon): stop giving new terminals to a daemon whose AppImage files are gone (#28011) * fix(daemon): stop giving new terminals to a daemon whose AppImage files are gone On Linux AppImage, a terminal daemon can outlive the mount it runs from and keep answering health checks until it SIGBUSes. Orca now reads the daemon's /proc exe link and mount table at adoption: an idle severed daemon is replaced; a busy one is kept for its sessions but quarantined, so fresh terminals use the local provider and a healthy RPC never re-admits them. Status and a verified restart are available over RPC and as `orca terminal-daemon status|restart`. Co-Authored-By: Claude * fix(daemon): lift the severed quarantine once a new daemon owns the endpoint Co-Authored-By: Claude * fix(daemon): a severed-daemon restart counts fallback terminals and never stops a replacement The restart re-checks the authenticated endpoint owner before any teardown and again at shutdown, and never falls back to a pid kill when targeted. Status counts the local fallback terminals the restart also ends. Co-Authored-By: Claude * chore(daemon): merge duplicate import Co-Authored-By: Claude * fix(runtime): classify the terminalDaemon RPC namespace as host-owned Co-Authored-By: Claude --------- Co-authored-by: m4air Co-authored-by: Claude --- src/cli/handler-group-manifest.ts | 2 +- src/cli/handlers/diagnostics.ts | 4 +- src/cli/handlers/terminal-daemon.test.ts | 19 ++ src/cli/handlers/terminal-daemon.ts | 72 +++++ src/cli/specs/index.ts | 2 + src/cli/specs/terminal-daemon.ts | 27 ++ src/main/daemon/daemon-appimage-launch.ts | 2 +- src/main/daemon/daemon-host-image-state.ts | 105 ++++++++ src/main/daemon/daemon-host-image.test.ts | 121 +++++++++ src/main/daemon/daemon-host-image.ts | 125 +++++++++ .../daemon/daemon-init-dependency-mocks.ts | 3 + src/main/daemon/daemon-init-mock-types.ts | 2 + .../daemon-init-restart-sequence.test.ts | 22 ++ .../daemon-init-severed-host-image.test.ts | 245 ++++++++++++++++++ .../daemon/daemon-out-of-process-launcher.ts | 20 +- .../daemon/daemon-protocol-cleanup.test.ts | 66 +++++ src/main/daemon/daemon-protocol-cleanup.ts | 19 +- src/main/daemon/daemon-provider-init.ts | 14 + src/main/daemon/daemon-provider-restart.ts | 46 +++- .../daemon/daemon-replacement-preflight.ts | 28 +- .../daemon/daemon-severed-restart.test.ts | 127 +++++++++ src/main/daemon/daemon-severed-restart.ts | 116 +++++++++ src/main/daemon/daemon-spawner.ts | 2 + .../degraded-daemon-pty-provider.test.ts | 4 +- .../daemon/degraded-daemon-pty-provider.ts | 8 +- src/main/runtime/rpc/methods/index.ts | 2 + .../runtime/rpc/methods/terminal-daemon.ts | 21 ++ src/shared/daemon-lifecycle-telemetry.ts | 3 +- src/shared/host-operation-ownership.ts | 1 + .../rpc-params-catalog.generated.ts | 3 + .../rpc-contract/terminal-daemon-params.ts | 7 + 31 files changed, 1217 insertions(+), 21 deletions(-) create mode 100644 src/cli/handlers/terminal-daemon.test.ts create mode 100644 src/cli/handlers/terminal-daemon.ts create mode 100644 src/cli/specs/terminal-daemon.ts create mode 100644 src/main/daemon/daemon-host-image-state.ts create mode 100644 src/main/daemon/daemon-host-image.test.ts create mode 100644 src/main/daemon/daemon-host-image.ts create mode 100644 src/main/daemon/daemon-init-severed-host-image.test.ts create mode 100644 src/main/daemon/daemon-protocol-cleanup.test.ts create mode 100644 src/main/daemon/daemon-severed-restart.test.ts create mode 100644 src/main/daemon/daemon-severed-restart.ts create mode 100644 src/main/runtime/rpc/methods/terminal-daemon.ts create mode 100644 src/shared/rpc-contract/terminal-daemon-params.ts diff --git a/src/cli/handler-group-manifest.ts b/src/cli/handler-group-manifest.ts index 40dbc30dd37..3c9706876c3 100644 --- a/src/cli/handler-group-manifest.ts +++ b/src/cli/handler-group-manifest.ts @@ -217,7 +217,7 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [ }, { name: 'diagnostics', - keys: ['diagnostics memory'], + keys: ['diagnostics memory', 'terminal-daemon status', 'terminal-daemon restart'], load: async () => (await import('./handlers/diagnostics.js')).DIAGNOSTICS_HANDLERS }, { diff --git a/src/cli/handlers/diagnostics.ts b/src/cli/handlers/diagnostics.ts index 340bd6afa16..eb894b73696 100644 --- a/src/cli/handlers/diagnostics.ts +++ b/src/cli/handlers/diagnostics.ts @@ -1,10 +1,12 @@ import type { MemorySnapshot } from '../../shared/process-stats-types' import type { CommandHandler } from '../dispatch' import { formatMemorySnapshot, printResult } from '../format' +import { TERMINAL_DAEMON_HANDLERS } from './terminal-daemon' export const DIAGNOSTICS_HANDLERS: Record = { 'diagnostics memory': async ({ client, json }) => { const result = await client.call('diagnostics.memory') printResult(result, json, formatMemorySnapshot) - } + }, + ...TERMINAL_DAEMON_HANDLERS } diff --git a/src/cli/handlers/terminal-daemon.test.ts b/src/cli/handlers/terminal-daemon.test.ts new file mode 100644 index 00000000000..4d8961a9ed4 --- /dev/null +++ b/src/cli/handlers/terminal-daemon.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' +import { formatSeveredDaemonStatus } from './terminal-daemon' + +describe('terminal-daemon status', () => { + it('says OK when nothing is severed', () => { + expect(formatSeveredDaemonStatus({ severed: null })).toBe('Terminal daemon: OK.') + }) + + it('says a restart ends running processes and only resumable agents come back', () => { + const text = formatSeveredDaemonStatus({ + severed: { pid: 4288, linuxStartTicks: '777', bootId: 'b', liveSessionCount: 3 } + }) + expect(text).toContain('PID 4288') + expect(text).toContain('3 running terminals') + expect(text).toContain('ENDS') + expect(text).toContain('Agents that support resume can be resumed') + expect(text).toContain('shells and other jobs cannot') + }) +}) diff --git a/src/cli/handlers/terminal-daemon.ts b/src/cli/handlers/terminal-daemon.ts new file mode 100644 index 00000000000..99a277eab97 --- /dev/null +++ b/src/cli/handlers/terminal-daemon.ts @@ -0,0 +1,72 @@ +import type { CommandHandler } from '../dispatch' +import { printResult } from '../format' +import { RuntimeClientError } from '../runtime-client' + +type SeveredDaemonStatus = { + pid: number + linuxStartTicks: string + bootId: string + liveSessionCount: number | null +} + +type SeveredStatusResult = { severed: SeveredDaemonStatus | null } + +type RestartResult = + | { status: 'restarted'; killedCount: number } + | { status: 'refused'; reason: 'not-severed' | 'replaced' | 'unverifiable' } + +const REFUSAL_MESSAGES: Record['reason'], string> = { + 'not-severed': 'The terminal daemon has its program files; nothing to restart.', + replaced: 'The terminal daemon was already replaced; nothing was stopped.', + unverifiable: 'The host could not verify the daemon process, so nothing was stopped.' +} + +function describeSessions(count: number | null): string { + return count === null + ? 'an unknown number of running terminals' + : `${count} running terminal${count === 1 ? '' : 's'}` +} + +export function formatSeveredDaemonStatus(result: SeveredStatusResult): string { + const severed = result.severed + if (!severed) { + return 'Terminal daemon: OK.' + } + return [ + `Terminal daemon PID ${severed.pid} is running without its program files (the AppImage it started from is gone).`, + `It still hosts ${describeSessions(severed.liveSessionCount)}. New terminals are not kept across restarts until it is restarted.`, + 'Restarting ENDS those processes. Agents that support resume can be resumed afterwards; shells and other jobs cannot.', + 'Run `orca terminal-daemon restart --yes` when your agents are idle.' + ].join('\n') +} + +export const TERMINAL_DAEMON_HANDLERS: Record = { + 'terminal-daemon status': async ({ client, json }) => { + const result = await client.call('terminalDaemon.severedStatus') + printResult(result, json, formatSeveredDaemonStatus) + }, + 'terminal-daemon restart': async ({ client, json, flags }) => { + const status = await client.call('terminalDaemon.severedStatus') + const severed = status.result.severed + if (!severed) { + printResult(status, json, () => REFUSAL_MESSAGES['not-severed']) + return + } + if (flags.get('yes') !== true) { + throw new RuntimeClientError( + 'confirmation_required', + `Restarting the terminal daemon ends ${describeSessions(severed.liveSessionCount)}. Re-run with --yes to confirm.` + ) + } + const result = await client.call('terminalDaemon.restartSevered', { + pid: severed.pid, + linuxStartTicks: severed.linuxStartTicks, + bootId: severed.bootId + }) + printResult(result, json, (value) => + value.status === 'restarted' + ? `Restarted the terminal daemon; ended ${describeSessions(value.killedCount)}.` + : REFUSAL_MESSAGES[value.reason] + ) + } +} diff --git a/src/cli/specs/index.ts b/src/cli/specs/index.ts index fbafcb8100d..0909d8e354b 100644 --- a/src/cli/specs/index.ts +++ b/src/cli/specs/index.ts @@ -13,6 +13,7 @@ import { MANAGED_SERVER_COMMAND_SPECS } from './managed-server' import { AGENT_HOOK_COMMAND_SPECS } from './agent-hooks' import { AGENT_LAUNCH_COMMAND_SPECS } from './agent-launch' import { DIAGNOSTICS_COMMAND_SPECS } from './diagnostics' +import { TERMINAL_DAEMON_COMMAND_SPECS } from './terminal-daemon' import { EMULATOR_COMMAND_SPECS } from './emulator' import { INTROSPECTION_COMMAND_SPECS } from './introspection' import { LINEAR_COMMAND_SPECS } from './linear' @@ -38,6 +39,7 @@ export const COMMAND_SPECS: CommandSpec[] = [ ...AGENT_HOOK_COMMAND_SPECS, ...AGENT_LAUNCH_COMMAND_SPECS, ...DIAGNOSTICS_COMMAND_SPECS, + ...TERMINAL_DAEMON_COMMAND_SPECS, ...INTROSPECTION_COMMAND_SPECS, ...ENVIRONMENT_COMMAND_SPECS, ...MANAGED_SERVER_COMMAND_SPECS, diff --git a/src/cli/specs/terminal-daemon.ts b/src/cli/specs/terminal-daemon.ts new file mode 100644 index 00000000000..0ab468ecbba --- /dev/null +++ b/src/cli/specs/terminal-daemon.ts @@ -0,0 +1,27 @@ +import type { CommandSpec } from '../args' +import { GLOBAL_FLAGS } from '../args' + +export const TERMINAL_DAEMON_COMMAND_SPECS: CommandSpec[] = [ + { + path: ['terminal-daemon', 'status'], + summary: 'Report whether the terminal daemon still has its program files', + usage: 'orca terminal-daemon status [--json]', + allowedFlags: [...GLOBAL_FLAGS], + notes: [ + 'On Linux AppImage, a daemon started by an earlier Orca can outlive the files it runs from. Orca keeps it so its terminals keep running, but new terminals are not kept across restarts until it is restarted.' + ], + examples: ['orca terminal-daemon status --json'] + }, + { + path: ['terminal-daemon', 'restart'], + destructive: true, + summary: 'Restart a terminal daemon whose program files are gone', + usage: 'orca terminal-daemon restart --yes [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'yes'], + notes: [ + 'Restarting ENDS every process running in its terminals. Agents that support resume can be resumed afterwards; shells and other jobs must be started again.', + 'The host re-checks that the daemon is still the one reported by status before stopping it, and refuses otherwise.' + ], + examples: ['orca terminal-daemon restart --yes'] + } +] diff --git a/src/main/daemon/daemon-appimage-launch.ts b/src/main/daemon/daemon-appimage-launch.ts index 0e8b6ab1638..47c55ba1c81 100644 --- a/src/main/daemon/daemon-appimage-launch.ts +++ b/src/main/daemon/daemon-appimage-launch.ts @@ -37,7 +37,7 @@ function decodeMountInfoPath(value: string): string { ) } -function isFuseMountPoint(mountInfo: string, mountPoint: string): boolean { +export function isFuseMountPoint(mountInfo: string, mountPoint: string): boolean { return mountInfo.split('\n').some((line) => { const [mountFields, fsFields] = line.split(' - ') const point = mountFields?.split(' ')[4] diff --git a/src/main/daemon/daemon-host-image-state.ts b/src/main/daemon/daemon-host-image-state.ts new file mode 100644 index 00000000000..587e2e6cb58 --- /dev/null +++ b/src/main/daemon/daemon-host-image-state.ts @@ -0,0 +1,105 @@ +/** + * The terminal daemon this app kept although its AppImage files are gone. New terminals avoid it + * until the user restarts it; this record is what the notice and the restart action read. + */ +import type { DaemonHostImageInspection } from './daemon-host-image' + +export type SeveredDaemonIncarnation = { + pid: number + linuxStartTicks: string + bootId: string +} + +/** `liveSessionCount`: every running terminal a restart would end, daemon and local fallback. */ +export type SeveredDaemonStatus = SeveredDaemonIncarnation & { liveSessionCount: number | null } + +export type SeveredDaemonRestartResult = + | { status: 'restarted'; killedCount: number } + /** `replaced`: another incarnation owns the daemon now; `unverifiable`: the host could not + * prove the process is the one the caller saw, so nothing was killed. */ + | { status: 'refused'; reason: 'not-severed' | 'replaced' | 'unverifiable' } + +/** Supplied by the desktop daemon provider; hosts without one (orcad) report nothing severed. */ +export type SeveredDaemonController = { + countLiveSessions(): Promise + restart(expected: SeveredDaemonIncarnation): Promise +} + +let severedDaemon: SeveredDaemonIncarnation | null = null +let controller: SeveredDaemonController | null = null +const listeners = new Set<() => void>() + +export function installSeveredDaemonController(next: SeveredDaemonController): void { + controller = next +} + +export async function readSeveredDaemonStatus(): Promise { + const severed = severedDaemon + if (!severed || !controller) { + return null + } + return { ...severed, liveSessionCount: await controller.countLiveSessions() } +} + +export async function restartSeveredDaemon( + expected: SeveredDaemonIncarnation +): Promise { + const severed = severedDaemon + if (!severed || !controller) { + return { status: 'refused', reason: 'not-severed' } + } + if (!isSameIncarnation(severed, expected)) { + return { status: 'refused', reason: 'replaced' } + } + return controller.restart(expected) +} + +export function isSameIncarnation( + left: SeveredDaemonIncarnation, + right: SeveredDaemonIncarnation +): boolean { + return ( + left.pid === right.pid && + left.linuxStartTicks === right.linuxStartTicks && + left.bootId === right.bootId + ) +} + +export function noteSeveredDaemon(inspection: DaemonHostImageInspection): void { + if ( + inspection.verdict !== 'severed' || + inspection.pid === null || + !inspection.linuxStartTicks || + !inspection.bootId + ) { + return + } + severedDaemon = { + pid: inspection.pid, + linuxStartTicks: inspection.linuxStartTicks, + bootId: inspection.bootId + } + notify() +} + +export function clearSeveredDaemon(): void { + if (severedDaemon) { + severedDaemon = null + notify() + } +} + +export function getSeveredDaemon(): SeveredDaemonIncarnation | null { + return severedDaemon +} + +export function onSeveredDaemonChanged(listener: () => void): () => void { + listeners.add(listener) + return () => listeners.delete(listener) +} + +function notify(): void { + for (const listener of listeners) { + listener() + } +} diff --git a/src/main/daemon/daemon-host-image.test.ts b/src/main/daemon/daemon-host-image.test.ts new file mode 100644 index 00000000000..6b0a2b5a351 --- /dev/null +++ b/src/main/daemon/daemon-host-image.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest' +import { inspectDaemonHostImage, type DaemonHostImageProc } from './daemon-host-image' +import type { ParsedDaemonPid } from './daemon-pid-file-parse' + +const PID = 4288 +const STAT = `${PID} (orca-ide) S 1 4288 4288 0 -1 4194560 0 0 0 0 0 0 0 0 20 0 1 0 777 0 0` +const BOOT = 'boot-1' +const OLD_MOUNT = '/tmp/.mount_orcaAb12' +const OWN_MOUNT = '/tmp/.mount_orcaZz99' + +function fuseLine(point: string): string { + return `1 2 0:50 / ${point} ro,nosuid,nodev relatime - fuse.orca-linux.AppImage orca-linux.AppImage ro` +} + +function record(overrides: Partial = {}): ParsedDaemonPid { + return { + pid: PID, + startedAtMs: null, + entryPath: `${OLD_MOUNT}/resources/app.asar.unpacked/out/main/daemon-entry.js`, + appVersion: '1.4.212', + launchNonce: 'nonce', + linuxStartTicks: '777', + bootId: BOOT, + spawnerExecPath: `${OLD_MOUNT}/orca-ide`, + cgroupUnit: null, + ...overrides + } +} + +function proc(options: { + exe: string | Error + mountinfo?: string + existing?: string[] + stat?: string +}): DaemonHostImageProc { + return { + readlink: (path) => { + expect(path).toBe(`/proc/${PID}/exe`) + if (options.exe instanceof Error) { + throw options.exe + } + return options.exe + }, + readFile: (path) => { + if (path === `/proc/${PID}/stat`) { + return options.stat ?? STAT + } + if (path === '/proc/sys/kernel/random/boot_id') { + return `${BOOT}\n` + } + if (path === `/proc/${PID}/mountinfo`) { + return options.mountinfo ?? '' + } + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + }, + exists: (path) => (options.existing ?? []).includes(path) + } +} + +function verdict(r: ParsedDaemonPid | null, p: DaemonHostImageProc) { + return inspectDaemonHostImage(r, { proc: p, platform: 'linux' }).verdict +} + +describe('inspectDaemonHostImage', () => { + it('is intact while the daemon mount is still in its mount table', () => { + expect( + verdict(record(), proc({ exe: `${OLD_MOUNT}/orca-ide`, mountinfo: fuseLine(OLD_MOUNT) })) + ).toBe('intact') + }) + + it('is intact for an own-mount daemon recorded with the window mount as its spawner', () => { + expect( + verdict(record(), proc({ exe: `${OWN_MOUNT}/orca-ide`, mountinfo: fuseLine(OWN_MOUNT) })) + ).toBe('intact') + }) + + it('is severed once the mount is gone from the daemon mount table', () => { + expect(verdict(record(), proc({ exe: `${OLD_MOUNT}/orca-ide`, mountinfo: '' }))).toBe('severed') + }) + + it('is severed when a lazy unmount leaves a path relative to the detached root', () => { + // #18200: readlink /proc//exe printed "/orca-ide" after the mount went away. + expect(verdict(record(), proc({ exe: '/orca-ide' }))).toBe('severed') + }) + + it('is severed when the runtime extract dir was deleted', () => { + const extracted = '/tmp/appimage_extracted_0123abcd/orca-ide' + const r = record({ spawnerExecPath: extracted, entryPath: null }) + expect(verdict(r, proc({ exe: `${extracted} (deleted)` }))).toBe('severed') + expect(verdict(r, proc({ exe: extracted, existing: [`/proc/${PID}/root${extracted}`] }))).toBe( + 'intact' + ) + }) + + it('leaves deb, rpm and extracted-cache daemons alone', () => { + const installed = record({ + spawnerExecPath: '/opt/Orca/orca-ide', + entryPath: '/opt/Orca/resources/app.asar.unpacked/out/main/daemon-entry.js' + }) + expect(verdict(installed, proc({ exe: '/opt/Orca/orca-ide' }))).toBe('not-ephemeral') + // An in-place package upgrade is not an ephemeral image; stale-bundle policy owns it. + expect(verdict(installed, proc({ exe: '/opt/Orca/orca-ide (deleted)' }))).toBe('not-ephemeral') + }) + + it('says nothing when the pid is not the recorded incarnation or cannot be read', () => { + const recycled = `${PID} (bash) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 999 0 0` + expect(verdict(record(), proc({ exe: '/orca-ide', stat: recycled }))).toBe('unknown') + expect( + verdict(record(), proc({ exe: Object.assign(new Error('EACCES'), { code: 'EACCES' }) })) + ).toBe('unknown') + expect(verdict(record({ linuxStartTicks: null }), proc({ exe: '/orca-ide' }))).toBe('unknown') + expect(verdict(null, proc({ exe: '/orca-ide' }))).toBe('unknown') + }) + + it('never reports severance off Linux', () => { + expect( + inspectDaemonHostImage(record(), { proc: proc({ exe: '/orca-ide' }), platform: 'darwin' }) + .verdict + ).toBe('not-ephemeral') + }) +}) diff --git a/src/main/daemon/daemon-host-image.ts b/src/main/daemon/daemon-host-image.ts new file mode 100644 index 00000000000..a6075ae7fa6 --- /dev/null +++ b/src/main/daemon/daemon-host-image.ts @@ -0,0 +1,125 @@ +/** + * Whether a Linux daemon's program files still exist (#18200). An AppImage daemon runs from a FUSE + * mount (`.mount_*`) or a runtime extract dir (`appimage_extracted_*`); once that goes, the next + * cold page faults with SIGBUS. The probe reads only `/proc` paths and the daemon's mount table, so + * a dead or wedged FUSE server cannot block it. + * + * `severed` is a health verdict, never liveness: the daemon may keep serving for hours. + */ +import { lstatSync, readFileSync, readlinkSync } from 'node:fs' +import type { ParsedDaemonPid } from './daemon-pid-file-parse' +import { parseLinuxStartTicks } from '../agent-hooks/managed-hook-owner-identity' +import { isFuseMountPoint } from './daemon-appimage-launch' + +export type DaemonHostImageVerdict = 'intact' | 'severed' | 'unknown' | 'not-ephemeral' + +export type DaemonHostImageInspection = { + verdict: DaemonHostImageVerdict + pid: number | null + linuxStartTicks: string | null + bootId: string | null +} + +export type DaemonHostImageProc = { + readlink(path: string): string + readFile(path: string): string + /** True when the path names an entry; throws on anything but ENOENT. */ + exists(path: string): boolean +} + +const linuxProc: DaemonHostImageProc = { + readlink: (path) => readlinkSync(path), + readFile: (path) => readFileSync(path, 'utf8'), + exists: (path) => { + try { + lstatSync(path) + return true + } catch (error) { + if (hasErrorCode(error, 'ENOENT') || hasErrorCode(error, 'ENOTDIR')) { + return false + } + throw error + } + } +} + +const DELETED_SUFFIX = ' (deleted)' + +export function inspectDaemonHostImage( + record: ParsedDaemonPid | null, + options: { proc?: DaemonHostImageProc; platform?: NodeJS.Platform } = {} +): DaemonHostImageInspection { + const unknown = (): DaemonHostImageInspection => ({ + verdict: 'unknown', + pid: record?.pid ?? null, + linuxStartTicks: record?.linuxStartTicks ?? null, + bootId: record?.bootId ?? null + }) + if ((options.platform ?? process.platform) !== 'linux') { + return { ...unknown(), verdict: 'not-ephemeral' } + } + if (!record || !record.linuxStartTicks || !record.bootId) { + return unknown() + } + const proc = options.proc ?? linuxProc + const result = (verdict: DaemonHostImageVerdict): DaemonHostImageInspection => ({ + ...unknown(), + verdict + }) + try { + // Verify the pid is still the recorded incarnation before reading anything else about it. + const startTicks = parseLinuxStartTicks(proc.readFile(`/proc/${record.pid}/stat`)) + const bootId = proc.readFile('/proc/sys/kernel/random/boot_id').trim() + if (startTicks !== record.linuxStartTicks || bootId !== record.bootId) { + return unknown() + } + const executable = proc.readlink(`/proc/${record.pid}/exe`) + const deleted = executable.endsWith(DELETED_SUFFIX) + const path = deleted ? executable.slice(0, -DELETED_SUFFIX.length) : executable + const ephemeralOrigin = [record.spawnerExecPath, record.entryPath, path].some( + (candidate) => candidate !== null && ephemeralRootOf(candidate) !== null + ) + if (!ephemeralOrigin) { + return result('not-ephemeral') + } + const root = ephemeralRootOf(path) + if (root?.kind === 'mount') { + const mounted = isFuseMountPoint(proc.readFile(`/proc/${record.pid}/mountinfo`), root.path) + return result(mounted && !deleted ? 'intact' : 'severed') + } + if (deleted) { + return result(root ? 'severed' : 'not-ephemeral') + } + // After a lazy unmount the kernel renders the path relative to the detached mount + // ("/orca-ide"), which resolves to nothing in the daemon's own root. + const resolves = proc.exists(`/proc/${record.pid}/root${path}`) + if (root) { + return result(resolves ? 'intact' : 'severed') + } + return result(resolves ? 'not-ephemeral' : 'severed') + } catch { + // EACCES (hidepid), an exited pid, or an unreadable mount table: say nothing. + return unknown() + } +} + +type EphemeralRoot = { kind: 'mount' | 'extract'; path: string } + +/** The AppImage mount point or runtime extract dir a path lives under, if any. */ +export function ephemeralRootOf(path: string): EphemeralRoot | null { + const segments = path.split('/') + for (let index = 1; index < segments.length - 1; index += 1) { + const segment = segments[index]! + if (segment.startsWith('.mount_')) { + return { kind: 'mount', path: segments.slice(0, index + 1).join('/') } + } + if (segment.startsWith('appimage_extracted_')) { + return { kind: 'extract', path: segments.slice(0, index + 1).join('/') } + } + } + return null +} + +function hasErrorCode(error: unknown, code: string): boolean { + return error instanceof Error && 'code' in error && error.code === code +} diff --git a/src/main/daemon/daemon-init-dependency-mocks.ts b/src/main/daemon/daemon-init-dependency-mocks.ts index d70bc4a29e5..bbb3e770a1b 100644 --- a/src/main/daemon/daemon-init-dependency-mocks.ts +++ b/src/main/daemon/daemon-init-dependency-mocks.ts @@ -84,6 +84,9 @@ export function createDaemonInitModuleFactories(state: DaemonInitMockState) { if (result.mode) { this.handle.mode = result.mode } + if (result.quarantine) { + this.handle.quarantine = result.quarantine + } if (result.adopted) { this.handle.adopted = true } diff --git a/src/main/daemon/daemon-init-mock-types.ts b/src/main/daemon/daemon-init-mock-types.ts index 341fcd26df7..cdc5a5ae9b2 100644 --- a/src/main/daemon/daemon-init-mock-types.ts +++ b/src/main/daemon/daemon-init-mock-types.ts @@ -47,6 +47,7 @@ export type MockAdapterConstructor = new (opts: MockAdapter['options']) => MockA /** Handle the fake spawner hands back from ensureRunning/getHandle. */ export type MockSpawnerHandle = { mode?: 'degraded-new-pty-fallback' + quarantine?: 'host-image-severed' adopted?: true releaseAdoptionLease?: () => void shutdown: () => Promise @@ -96,6 +97,7 @@ export type EnsureRunningOverride = () => Promise<{ socketPath: string tokenPath: string mode?: 'degraded-new-pty-fallback' + quarantine?: 'host-image-severed' adopted?: true }> diff --git a/src/main/daemon/daemon-init-restart-sequence.test.ts b/src/main/daemon/daemon-init-restart-sequence.test.ts index 55f226b4bf2..df3b48417cb 100644 --- a/src/main/daemon/daemon-init-restart-sequence.test.ts +++ b/src/main/daemon/daemon-init-restart-sequence.test.ts @@ -79,6 +79,28 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => { expect(order).toEqual(['fanout', 'unbind']) }) + it('refuses a targeted restart before ending anything when another daemon owns the endpoint', async () => { + const mod = await importFresh() + await mod.initDaemonPtyProvider() + const originalAdapter = adapterInstances[0] + daemonClientMock.mockImplementation(function MockDaemonClient() { + return { + ensureConnected: vi.fn(async () => {}), + ensureConnectedWithin: vi.fn(async () => {}), + getDaemonIdentity: vi.fn(() => ({ pid: 9000, startedAtMs: 1, launchNonce: 'other' })), + request: vi.fn(async () => ({ sessions: [] })), + disconnect: vi.fn() + } + }) + + await expect(mod.restartDaemon({ expectedDaemonPid: 4288 })).rejects.toThrow( + 'owned by a different daemon' + ) + + expect(originalAdapter.fanoutSyntheticExits).not.toHaveBeenCalled() + expect(unbindLocalProviderListenersMock).not.toHaveBeenCalled() + }) + it('uses daemon-owned idle retirement after a failed manual-restart adoption', async () => { const mod = await importFresh() await mod.initDaemonPtyProvider() diff --git a/src/main/daemon/daemon-init-severed-host-image.test.ts b/src/main/daemon/daemon-init-severed-host-image.test.ts new file mode 100644 index 00000000000..acb81c1d72e --- /dev/null +++ b/src/main/daemon/daemon-init-severed-host-image.test.ts @@ -0,0 +1,245 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { + forkMock, + checkDaemonHealthMock, + getDaemonLaunchIdentityMock, + killStaleDaemonMock, + daemonClientMock, + spawnerInstances, + ensureRunningOverrides, + trackDaemonReplacedMock, + localFallbackProvider, + importFresh, + installDefaultNetConnectStub, + moduleFactories +} = await vi.hoisted(async () => + (await import('./daemon-init-test-harness')).createDaemonInitMocks() +) + +const hostImage = vi.hoisted(() => { + const state: { verdict: 'intact' | 'severed' | 'unknown' | 'not-ephemeral'; pid: number } = { + verdict: 'not-ephemeral', + pid: 4288 + } + return state +}) + +vi.mock('fs', () => moduleFactories.fs()) +vi.mock('child_process', async (importOriginal) => + moduleFactories.childProcess(await importOriginal>()) +) +vi.mock('net', () => moduleFactories.net()) +vi.mock('./daemon-health', () => moduleFactories.daemonHealth()) +vi.mock('./daemon-pid-identity', () => moduleFactories.daemonPidIdentity()) +vi.mock('./daemon-tcc-attribution', () => moduleFactories.daemonTccAttribution()) +vi.mock('./daemon-bundle-staleness', () => moduleFactories.daemonBundleStaleness()) +vi.mock('./daemon-stale-kill', () => moduleFactories.daemonStaleKill()) +vi.mock('./daemon-process-start-time', () => moduleFactories.daemonProcessStartTime()) +vi.mock('./daemon-pid-file-parse', () => moduleFactories.daemonPidFileParse()) +vi.mock('./client', () => moduleFactories.client()) +vi.mock('./daemon-lifecycle-event', () => moduleFactories.daemonLifecycleEvent()) +vi.mock('./daemon-spawner', () => moduleFactories.daemonSpawner()) +vi.mock('./daemon-pty-adapter', () => moduleFactories.daemonPtyAdapter()) +vi.mock('../ipc/pty/provider/registry', () => moduleFactories.ptyProviderRegistry()) +vi.mock('../ipc/pty/provider/listener-lifecycle', () => moduleFactories.ptyListenerLifecycle()) +vi.mock('./daemon-host-image', () => ({ + inspectDaemonHostImage: () => ({ + verdict: hostImage.verdict, + pid: hostImage.pid, + linuxStartTicks: '777', + bootId: 'boot-1' + }) +})) + +type Launcher = ( + socketPath: string, + tokenPath: string, + pidPath?: string, + launchNonce?: string +) => Promise<{ mode?: string; quarantine?: string; releaseAdoptionLease?(): void }> + +function isLauncher(value: unknown): value is Launcher { + return typeof value === 'function' +} + +function launcher(): Launcher { + const candidate = spawnerInstances[0].launcher + if (!isLauncher(candidate)) { + throw new Error('launcher not captured') + } + return candidate +} + +function mockDaemonWithLiveSessions(count: number): void { + daemonClientMock.mockImplementation(function MockDaemonClient() { + return { + ensureConnected: vi.fn(async () => {}), + ensureConnectedWithin: vi.fn(async () => {}), + request: vi.fn(async () => ({ + sessions: Array.from({ length: count }, (_, index) => ({ + sessionId: `s${index}`, + isAlive: true + })) + })), + disconnect: vi.fn() + } + }) +} + +function mockReadyFork(): void { + forkMock.mockImplementationOnce(() => ({ + pid: 12345, + on(event: string, cb: (arg?: unknown) => void) { + if (event === 'message') { + queueMicrotask(() => cb({ type: 'ready', pid: 12345, startedAtMs: 1_000_000 })) + } + return this + }, + off() { + return this + }, + disconnect: vi.fn(), + unref: vi.fn() + })) +} + +describe('daemon-init: a daemon whose AppImage files are gone', () => { + let warn: ReturnType + + beforeEach(() => { + installDefaultNetConnectStub() + hostImage.verdict = 'severed' + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + }) + + afterEach(async () => { + hostImage.verdict = 'not-ephemeral' + hostImage.pid = 4288 + warn.mockRestore() + vi.clearAllMocks() + ;(await import('./daemon-host-image-state')).clearSeveredDaemon() + }) + + it('keeps a busy daemon from another mount but quarantines it from fresh terminals', async () => { + const mod = await importFresh() + await mod.initDaemonPtyProvider() + // A previous mount always reads as a different app path; that branch would adopt it normally. + getDaemonLaunchIdentityMock.mockReturnValueOnce('mismatch') + mockDaemonWithLiveSessions(2) + + const handle = await launcher()('/fake/socket', '/fake/token') + + expect(handle.mode).toBe('degraded-new-pty-fallback') + expect(handle.quarantine).toBe('host-image-severed') + expect(forkMock).not.toHaveBeenCalled() + expect(killStaleDaemonMock).not.toHaveBeenCalled() + const state = await import('./daemon-host-image-state') + expect(state.getSeveredDaemon()).toEqual({ + pid: 4288, + linuxStartTicks: '777', + bootId: 'boot-1' + }) + handle.releaseAdoptionLease?.() + }) + + it('replaces an idle daemon whose AppImage files are gone', async () => { + const mod = await importFresh() + await mod.initDaemonPtyProvider() + mockReadyFork() + + await launcher()('/fake/socket', '/fake/token') + + expect(forkMock).toHaveBeenCalledTimes(1) + expect(trackDaemonReplacedMock).toHaveBeenCalledWith('severed_host_image', 0) + }) + + it('quarantines a daemon kept by the failed-health branch too', async () => { + const mod = await importFresh() + await mod.initDaemonPtyProvider() + checkDaemonHealthMock.mockResolvedValueOnce('unresponsive') + mockDaemonWithLiveSessions(1) + + const handle = await launcher()('/fake/socket', '/fake/token') + + expect(handle.quarantine).toBe('host-image-severed') + expect(handle.mode).toBe('degraded-new-pty-fallback') + expect(forkMock).not.toHaveBeenCalled() + handle.releaseAdoptionLease?.() + }) + + it('leaves intact and non-AppImage daemons on the existing adoption policy', async () => { + hostImage.verdict = 'intact' + const mod = await importFresh() + await mod.initDaemonPtyProvider() + mockDaemonWithLiveSessions(1) + + const handle = await launcher()('/fake/socket', '/fake/token') + + expect(handle.mode).toBeUndefined() + expect(handle.quarantine).toBeUndefined() + handle.releaseAdoptionLease?.() + }) + + it('never re-admits fresh terminals to a quarantined daemon on a healthy check', async () => { + const mod = await importFresh() + ;(await import('./daemon-host-image-state')).noteSeveredDaemon({ + verdict: 'severed', + pid: 4288, + linuxStartTicks: '777', + bootId: 'boot-1' + }) + ensureRunningOverrides.push(async () => ({ + socketPath: '/fake/severed-socket', + tokenPath: '/fake/severed-token', + mode: 'degraded-new-pty-fallback', + quarantine: 'host-image-severed' + })) + await mod.initDaemonPtyProvider() + checkDaemonHealthMock.mockResolvedValue('healthy') + + const { DegradedDaemonPtyProvider } = await import('./degraded-daemon-pty-provider') + const provider = mod.getDaemonProvider() + expect(provider).toBeInstanceOf(DegradedDaemonPtyProvider) + if (!(provider instanceof DegradedDaemonPtyProvider)) { + throw new Error('expected degraded routing') + } + const degraded = provider + + await expect(degraded.recoverFreshSpawnRouting()).resolves.toBe(false) + const result = await degraded.spawn({ cols: 80, rows: 24 }) + expect(result.id).toBe('local-fallback-pty') + expect(localFallbackProvider.spawn).toHaveBeenCalled() + }) + + it('lets fresh terminals back once a new daemon incarnation owns the endpoint', async () => { + const mod = await importFresh() + const state = await import('./daemon-host-image-state') + state.noteSeveredDaemon({ + verdict: 'severed', + pid: 4288, + linuxStartTicks: '777', + bootId: 'boot-1' + }) + ensureRunningOverrides.push(async () => ({ + socketPath: '/fake/severed-socket', + tokenPath: '/fake/severed-token', + mode: 'degraded-new-pty-fallback', + quarantine: 'host-image-severed' + })) + await mod.initDaemonPtyProvider() + checkDaemonHealthMock.mockResolvedValue('healthy') + const { DegradedDaemonPtyProvider } = await import('./degraded-daemon-pty-provider') + const provider = mod.getDaemonProvider() + if (!(provider instanceof DegradedDaemonPtyProvider)) { + throw new Error('expected degraded routing') + } + + // The severed daemon crashed and its adapter respawned a new one at the endpoint. + hostImage.verdict = 'intact' + hostImage.pid = 9000 + + await expect(provider.recoverFreshSpawnRouting()).resolves.toBe(true) + expect(state.getSeveredDaemon()).toBeNull() + }) +}) diff --git a/src/main/daemon/daemon-out-of-process-launcher.ts b/src/main/daemon/daemon-out-of-process-launcher.ts index 8e47635fa7b..25b5356a087 100644 --- a/src/main/daemon/daemon-out-of-process-launcher.ts +++ b/src/main/daemon/daemon-out-of-process-launcher.ts @@ -30,6 +30,9 @@ import { import { PROTOCOL_VERSION } from './types' import { prepareDaemonReplacement } from './daemon-replacement-preflight' import { ensureDaemonSocketDir } from './daemon-socket-endpoint-path' +import { inspectDaemonHostImage } from './daemon-host-image' +import { noteSeveredDaemon } from './daemon-host-image-state' +import { readDaemonPidRecord } from './daemon-endpoint-incarnation' // Why: the adapter decides a runtime resolver replacement, but the launcher completes it — and by // then the daemon has usually self-retired (dropping its last authenticated client is enough), so @@ -45,8 +48,11 @@ export function attributeNextDaemonReplacement(reason: DaemonReplaceReason): voi attributedReplaceReason = reason } +// Why every adoption path calls this: a daemon whose AppImage files are gone answers RPCs for +// hours, so whichever branch decided to keep it, fresh terminals must not land on it. function createPreservedDaemonHandle( runtimeDir: string, + pidPath: string, mode?: 'degraded-new-pty-fallback' ): DaemonProcessHandle { const handle: DaemonProcessHandle = { @@ -55,6 +61,16 @@ function createPreservedDaemonHandle( await cleanupDaemonForProtocol(runtimeDir, PROTOCOL_VERSION) } } + const hostImage = inspectDaemonHostImage(readDaemonPidRecord(pidPath)) + if (hostImage.verdict === 'severed') { + console.warn( + `[daemon] DEGRADED MODE: keeping daemon PID ${hostImage.pid} whose AppImage files are gone. Existing sessions keep working; fresh terminals run on the local provider WITHOUT daemon persistence until you restart the terminal daemon.` + ) + noteSeveredDaemon(hostImage) + handle.mode = 'degraded-new-pty-fallback' + handle.quarantine = 'host-image-severed' + return handle + } if (mode) { handle.mode = mode } @@ -110,7 +126,7 @@ export function createOutOfProcessLauncher( ): Promise => { const connectedClient = adoptionClient ?? undefined adoptionClient = null - return holdDaemonAdoptionLease(createPreservedDaemonHandle(runtimeDir, mode), { + return holdDaemonAdoptionLease(createPreservedDaemonHandle(runtimeDir, pidPath, mode), { socketPath, tokenPath, pidPath, @@ -189,7 +205,7 @@ export function createOutOfProcessLauncher( '[daemon] Endpoint was taken by another daemon during startup — adopting it instead' ) // Why pidPath: adopting repairs a PID record that names the wrong incarnation. - return await holdDaemonAdoptionLease(createPreservedDaemonHandle(runtimeDir), { + return await holdDaemonAdoptionLease(createPreservedDaemonHandle(runtimeDir, pidPath), { socketPath, tokenPath, pidPath diff --git a/src/main/daemon/daemon-protocol-cleanup.test.ts b/src/main/daemon/daemon-protocol-cleanup.test.ts new file mode 100644 index 00000000000..6f61ba4db45 --- /dev/null +++ b/src/main/daemon/daemon-protocol-cleanup.test.ts @@ -0,0 +1,66 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => { + const state: { + identityPid: number + connectError: Error | null + request: ReturnType + killStaleDaemon: ReturnType + } = { + identityPid: 4288, + connectError: null, + request: vi.fn(async () => ({ sessions: [] })), + killStaleDaemon: vi.fn(async () => ({ killed: true, liveOwnerSurvived: false })) + } + return state +}) + +vi.mock('./daemon-launch-paths', () => ({ probeDaemonSocket: vi.fn(async () => true) })) +vi.mock('./daemon-stale-kill', () => ({ killStaleDaemon: mocks.killStaleDaemon })) +vi.mock('./client', () => ({ + DaemonClient: class { + async ensureConnected(): Promise { + if (mocks.connectError) { + throw mocks.connectError + } + } + getDaemonIdentity() { + return { pid: mocks.identityPid, startedAtMs: 1, launchNonce: 'n' } + } + request = mocks.request + disconnect(): void {} + } +})) + +import { cleanupDaemonForProtocol } from './daemon-protocol-cleanup' + +afterEach(() => { + mocks.identityPid = 4288 + mocks.connectError = null + vi.clearAllMocks() +}) + +describe('cleanupDaemonForProtocol with an expected owner', () => { + it('never shuts down a different endpoint owner', async () => { + mocks.identityPid = 9000 + await expect(cleanupDaemonForProtocol('/runtime', 1, 4288)).rejects.toThrow( + 'owned by a different daemon' + ) + expect(mocks.request).not.toHaveBeenCalledWith('shutdown', expect.anything()) + expect(mocks.killStaleDaemon).not.toHaveBeenCalled() + }) + + it('never falls back to a pid kill when the owner cannot be verified', async () => { + mocks.connectError = new Error('connect failed') + await expect(cleanupDaemonForProtocol('/runtime', 1, 4288)).rejects.toThrow( + 'could not be verified' + ) + expect(mocks.killStaleDaemon).not.toHaveBeenCalled() + }) + + it('keeps the untargeted pid-kill fallback for unreachable daemons', async () => { + mocks.connectError = new Error('connect failed') + await expect(cleanupDaemonForProtocol('/runtime', 1)).resolves.toMatchObject({ cleaned: true }) + expect(mocks.killStaleDaemon).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/daemon/daemon-protocol-cleanup.ts b/src/main/daemon/daemon-protocol-cleanup.ts index 69808ac5716..16bea112f15 100644 --- a/src/main/daemon/daemon-protocol-cleanup.ts +++ b/src/main/daemon/daemon-protocol-cleanup.ts @@ -17,7 +17,9 @@ export type OrphanedDaemonCleanupResult = { export async function cleanupDaemonForProtocol( runtimeDir: string, - protocolVersion: number + protocolVersion: number, + /** When set, stop only if the authenticated endpoint owner is this pid; never fall back to a pid kill. */ + expectedOwnerPid?: number ): Promise { const socketPath = getDaemonSocketPath(runtimeDir, protocolVersion) const tokenPath = getDaemonTokenPath(runtimeDir, protocolVersion) @@ -43,6 +45,11 @@ export async function cleanupDaemonForProtocol( let didKillStaleDaemon = false try { await client.ensureConnected() + if (expectedOwnerPid !== undefined && client.getDaemonIdentity()?.pid !== expectedOwnerPid) { + throw new DaemonEndpointOwnershipError( + 'Daemon cleanup aborted: the endpoint is owned by a different daemon than expected' + ) + } const sessions = await client .request('listSessions', undefined) .catch(() => ({ sessions: [] })) @@ -53,7 +60,15 @@ export async function cleanupDaemonForProtocol( // Daemon exits immediately after the RPC, so the socket may close before the reply arrives; treat as success. }) didRequestShutdown = true - } catch { + } catch (error) { + if (expectedOwnerPid !== undefined) { + // A targeted stop never kills by pid: the owner it would find may be a replacement. + throw error instanceof DaemonEndpointOwnershipError + ? error + : new DaemonEndpointOwnershipError( + 'Daemon cleanup aborted: the expected daemon could not be verified' + ) + } // Previous-protocol daemons may be wedged or too old for the RPC path; fall back to PID cleanup (only unlinks a live socket after proving the process is killed). const killOutcome = await killStaleDaemon(runtimeDir, socketPath, tokenPath, protocolVersion) didKillStaleDaemon = killOutcome.killed diff --git a/src/main/daemon/daemon-provider-init.ts b/src/main/daemon/daemon-provider-init.ts index dbfb91bd301..40645f40040 100644 --- a/src/main/daemon/daemon-provider-init.ts +++ b/src/main/daemon/daemon-provider-init.ts @@ -35,6 +35,11 @@ import type { DaemonRespawnReason } from './daemon-pty-runtime-state' import { DaemonPtyRouter } from './daemon-pty-router' import { isDaemonRestartInFlight } from './daemon-restart-state' import { DaemonSpawner, getDaemonPidPath } from './daemon-spawner' +import { clearSeveredDaemon, installSeveredDaemonController } from './daemon-host-image-state' +import { + createSeveredDaemonController, + hasSeveredDaemonBeenReplaced +} from './daemon-severed-restart' // Why: daemon init runs concurrent with window load, so an in-process t timestamp (not harness stderr timing) measures cold-start. function logDaemonMilestone(event: string, details: Record = {}): void { @@ -68,6 +73,7 @@ export async function initDaemonPtyProvider( // Why: reclaim superseded daemon-host copies on EVERY launch (spawns are rare), keeping current + live-daemon-pinned versions. pruneOldDaemonHosts(collectPinnedDaemonVersions(runtimeDir)) const launchMode = newSpawner.getHandle()?.mode + const quarantine = newSpawner.getHandle()?.quarantine logDaemonMilestone('daemon-current-ready') if (signal?.aborted) { // Why: fail-open may already have spawned fallback PTYs; don't install late, but retire an empty daemon (live sessions reject it and survive). @@ -132,7 +138,11 @@ export async function initDaemonPtyProvider( current: newAdapter, legacy: legacyAdapters, fallback: getLocalPtyProvider(), + // Why the incarnation gate: a severed daemon passes health checks for hours, so only a + // new daemon at the endpoint (after a crash respawn or restart) may take fresh terminals. probeCurrentDaemonSpawn: async () => + (quarantine !== 'host-image-severed' || + hasSeveredDaemonBeenReplaced(getDaemonPidPath(runtimeDir))) && (await checkDaemonHealth(info.socketPath, info.tokenPath)) === 'healthy' }) : legacyAdapters.length > 0 || isHoldingForNewerProtocolDaemon() @@ -161,6 +171,10 @@ export async function initDaemonPtyProvider( throw error } installDaemonProvider(newSpawner, routedAdapter) + installSeveredDaemonController(createSeveredDaemonController()) + if (quarantine !== 'host-image-severed') { + clearSeveredDaemon() + } // Why: the first window may register PTY listeners before daemon init finishes; rebind so daemon PTYs still fan out events. rebindLocalProviderListeners() logDaemonMilestone('daemon-init-done', { diff --git a/src/main/daemon/daemon-provider-restart.ts b/src/main/daemon/daemon-provider-restart.ts index 203fcef7f15..f6d11d868f8 100644 --- a/src/main/daemon/daemon-provider-restart.ts +++ b/src/main/daemon/daemon-provider-restart.ts @@ -4,6 +4,7 @@ import { } from '../ipc/pty/provider/listener-lifecycle' import { DegradedDaemonPtyProvider } from './degraded-daemon-pty-provider' import { + DaemonEndpointOwnershipError, cleanupFailedDaemonAdoption, releaseDaemonAdoptionLease, takeDaemonAdoptionLeaseRelease @@ -32,17 +33,26 @@ import { runCoalescedDaemonRestart, type RestartDaemonResult } from './daemon-restart-state' -import { getDaemonPidPath, type DaemonSpawner } from './daemon-spawner' +import { + getDaemonPidPath, + getDaemonSocketPath, + getDaemonTokenPath, + type DaemonSpawner +} from './daemon-spawner' +import { DaemonClient } from './client' +import { clearSeveredDaemon } from './daemon-host-image-state' import { PROTOCOL_VERSION } from './types' export type { RestartDaemonResult } from './daemon-restart-state' // Why: the 7-step restart sequence from docs/daemon-staleness-ux.md §Phase 1; current-protocol only (legacy adapters preserved). -export async function restartDaemon(): Promise { - return runCoalescedDaemonRestart(runRestartDaemon) +export async function restartDaemon( + options: { expectedDaemonPid?: number } = {} +): Promise { + return runCoalescedDaemonRestart(() => runRestartDaemon(options.expectedDaemonPid)) } -async function runRestartDaemon(): Promise { +async function runRestartDaemon(expectedDaemonPid?: number): Promise { const currentSpawner = getDaemonSpawner() const currentAdapter = getDaemonProvider() if (!currentSpawner || !currentAdapter) { @@ -50,6 +60,11 @@ async function runRestartDaemon(): Promise { } const runtimeDir = getRuntimeDir() + if (expectedDaemonPid !== undefined) { + // Why before any teardown: a targeted restart must not end a single terminal when the + // endpoint now belongs to a replacement daemon. + await assertDaemonEndpointOwner(runtimeDir, expectedDaemonPid) + } // An operator asking for a restart is the deliberate "try again" that clears crash-loop // containment; without this a wedged host could never be recovered from the UI. currentSpawner.resetRespawnWindow() @@ -63,7 +78,7 @@ async function runRestartDaemon(): Promise { : 0 const currentDaemonSessionIds = currentAdapter instanceof DegradedDaemonPtyProvider - ? currentAdapter.getCurrentDaemonSessionIds() + ? currentAdapter.getSessionIdsRoutedTo('current-daemon') : [] const killedCount = new Set([...currentOnly.getActiveSessionIds(), ...currentDaemonSessionIds]).size + @@ -79,7 +94,7 @@ async function runRestartDaemon(): Promise { // Step 3: kill the current-protocol daemon process; legacy adapters untouched. let info: Awaited> try { - await cleanupDaemonForProtocol(runtimeDir, PROTOCOL_VERSION) + await cleanupDaemonForProtocol(runtimeDir, PROTOCOL_VERSION, expectedDaemonPid) // Step 4: reuse the existing spawner so the respawn closure baked into long-lived adapters stays valid (do NOT new one). currentSpawner.resetHandle() @@ -163,6 +178,25 @@ async function runRestartDaemon(): Promise { // Step 7: rebind renderer listeners against the new provider. rebindLocalProviderListeners() + // A new incarnation replaced any daemon whose AppImage files were gone. + clearSeveredDaemon() return { killedCount } } + +async function assertDaemonEndpointOwner(runtimeDir: string, expectedPid: number): Promise { + const client = new DaemonClient({ + socketPath: getDaemonSocketPath(runtimeDir, PROTOCOL_VERSION), + tokenPath: getDaemonTokenPath(runtimeDir, PROTOCOL_VERSION) + }) + try { + await client.ensureConnected() + if (client.getDaemonIdentity()?.pid !== expectedPid) { + throw new DaemonEndpointOwnershipError( + 'Daemon restart aborted: the endpoint is owned by a different daemon than expected' + ) + } + } finally { + client.disconnect() + } +} diff --git a/src/main/daemon/daemon-replacement-preflight.ts b/src/main/daemon/daemon-replacement-preflight.ts index 1573755efef..de91a88b8b9 100644 --- a/src/main/daemon/daemon-replacement-preflight.ts +++ b/src/main/daemon/daemon-replacement-preflight.ts @@ -16,6 +16,7 @@ import { cleanupDaemonForProtocol } from './daemon-protocol-cleanup' import { getDaemonPidPath, type DaemonProcessHandle } from './daemon-spawner' import { killStaleDaemon } from './daemon-stale-kill' import { getMacDaemonTccAttributionHealth } from './daemon-tcc-attribution' +import { inspectDaemonHostImage } from './daemon-host-image' import { PROTOCOL_VERSION } from './types' // Why a count on top of the wall clock: DAEMON_RECOVERY_BUDGET_MS ends the grace, but a socket @@ -67,8 +68,31 @@ export async function prepareDaemonReplacement( `[daemon] Migrated adopted daemon PID ${pidRecord.pid} out of legacy app scope into a durable scope` ) } - const resolverHealth = await getMacDaemonSystemResolverHealth(socketPath, tokenPath) - if (resolverHealth === 'unhealthy') { + // Why before identity: a daemon from a previous mount always reads as a different app path, + // and that branch would keep it as a normal target for fresh terminals. + const hostImage = inspectDaemonHostImage(pidRecord) + const resolverHealth = + hostImage.verdict === 'severed' + ? null + : await getMacDaemonSystemResolverHealth(socketPath, tokenPath) + if (hostImage.verdict === 'severed') { + const liveSessionCount = await getAliveDaemonSessionCount( + socketPath, + tokenPath, + recoveryDeadlineMs + ) + if (liveSessionCount !== 0) { + console.warn( + liveSessionCount === null + ? '[daemon] Keeping daemon whose AppImage files are gone because live session state could not be verified' + : `[daemon] Keeping daemon whose AppImage files are gone because it owns ${liveSessionCount} live session${liveSessionCount === 1 ? '' : 's'}` + ) + return preserveDaemon('degraded-new-pty-fallback') + } + console.warn('[daemon] Replacing idle daemon whose AppImage files are gone') + pendingReplacement = { reason: 'severed_host_image', liveSessionCount } + confirmedReplacement = (await cleanupDaemonForProtocol(runtimeDir, PROTOCOL_VERSION)).cleaned + } else if (resolverHealth === 'unhealthy') { const liveSessionCount = await getAliveDaemonSessionCount( socketPath, tokenPath, diff --git a/src/main/daemon/daemon-severed-restart.test.ts b/src/main/daemon/daemon-severed-restart.test.ts new file mode 100644 index 00000000000..da2b021964e --- /dev/null +++ b/src/main/daemon/daemon-severed-restart.test.ts @@ -0,0 +1,127 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./daemon-provider-restart', () => ({ restartDaemon: vi.fn() })) + +import { + clearSeveredDaemon, + getSeveredDaemon, + installSeveredDaemonController, + noteSeveredDaemon, + readSeveredDaemonStatus, + restartSeveredDaemon +} from './daemon-host-image-state' +import type { DaemonHostImageInspection } from './daemon-host-image' +import { + createSeveredDaemonController, + hasSeveredDaemonBeenReplaced +} from './daemon-severed-restart' + +const SEVERED: DaemonHostImageInspection = { + verdict: 'severed', + pid: 4288, + linuxStartTicks: '777', + bootId: 'boot-1' +} +const EXPECTED = { pid: 4288, linuxStartTicks: '777', bootId: 'boot-1' } + +function setUp( + current: DaemonHostImageInspection, + sessions = [{ isAlive: true }], + fallbackSessions = 0 +) { + const restart = vi.fn(async (_expectedPid: number) => ({ killedCount: sessions.length })) + installSeveredDaemonController( + createSeveredDaemonController({ + inspect: () => current, + restart, + listSessions: async () => sessions, + countFallbackSessions: () => fallbackSessions + }) + ) + noteSeveredDaemon(SEVERED) + return restart +} + +afterEach(() => { + clearSeveredDaemon() +}) + +describe('restarting a daemon whose AppImage files are gone', () => { + it('counts every running terminal a restart would end, local fallback included', async () => { + setUp(SEVERED, [{ isAlive: true }, { isAlive: false }, { isAlive: true }], 2) + await expect(readSeveredDaemonStatus()).resolves.toEqual({ ...EXPECTED, liveSessionCount: 4 }) + }) + + it('restarts only the incarnation the caller saw, after re-reading it', async () => { + const restart = setUp(SEVERED) + await expect(restartSeveredDaemon(EXPECTED)).resolves.toEqual({ + status: 'restarted', + killedCount: 1 + }) + // The pid travels into the restart so its teardown re-checks the endpoint owner. + expect(restart).toHaveBeenCalledWith(4288) + expect(getSeveredDaemon()).toBeNull() + }) + + it('refuses when the caller saw another incarnation', async () => { + const restart = setUp(SEVERED) + await expect(restartSeveredDaemon({ ...EXPECTED, linuxStartTicks: '1' })).resolves.toEqual({ + status: 'refused', + reason: 'replaced' + }) + expect(restart).not.toHaveBeenCalled() + }) + + it('refuses when the host now names a different daemon', async () => { + const restart = setUp({ ...SEVERED, verdict: 'intact', pid: 9000, linuxStartTicks: '5' }) + await expect(restartSeveredDaemon(EXPECTED)).resolves.toEqual({ + status: 'refused', + reason: 'replaced' + }) + expect(restart).not.toHaveBeenCalled() + }) + + it('refuses without killing when the process cannot be verified', async () => { + const restart = setUp({ ...SEVERED, verdict: 'unknown' }) + await expect(restartSeveredDaemon(EXPECTED)).resolves.toEqual({ + status: 'refused', + reason: 'unverifiable' + }) + expect(restart).not.toHaveBeenCalled() + expect(getSeveredDaemon()).toEqual(EXPECTED) + }) + + it('reports nothing severed until a daemon was kept severed', async () => { + setUp(SEVERED) + clearSeveredDaemon() + await expect(readSeveredDaemonStatus()).resolves.toBeNull() + await expect(restartSeveredDaemon(EXPECTED)).resolves.toEqual({ + status: 'refused', + reason: 'not-severed' + }) + }) +}) + +describe('hasSeveredDaemonBeenReplaced', () => { + it('keeps the quarantine while the severed incarnation still owns the endpoint', () => { + setUp(SEVERED) + expect(hasSeveredDaemonBeenReplaced('/pid', () => SEVERED)).toBe(false) + expect(hasSeveredDaemonBeenReplaced('/pid', () => ({ ...SEVERED, verdict: 'unknown' }))).toBe( + false + ) + expect(getSeveredDaemon()).toEqual(EXPECTED) + }) + + it('lifts it once a verified, intact daemon replaced the severed one', () => { + setUp(SEVERED) + expect( + hasSeveredDaemonBeenReplaced('/pid', () => ({ + verdict: 'intact', + pid: 9000, + linuxStartTicks: '5', + bootId: 'boot-1' + })) + ).toBe(true) + expect(getSeveredDaemon()).toBeNull() + }) +}) diff --git a/src/main/daemon/daemon-severed-restart.ts b/src/main/daemon/daemon-severed-restart.ts new file mode 100644 index 00000000000..cd486aabb51 --- /dev/null +++ b/src/main/daemon/daemon-severed-restart.ts @@ -0,0 +1,116 @@ +/** + * Restarting a daemon whose AppImage files are gone ends every process it hosts, so the action + * re-reads the host's own evidence right before acting: it proceeds only when the pid record and + * `/proc` still name the exact incarnation the user was shown. + */ +import { getDaemonRuntimeDir } from './daemon-launch-paths' +import { readDaemonPidRecord } from './daemon-endpoint-incarnation' +import { inspectDaemonHostImage } from './daemon-host-image' +import { + clearSeveredDaemon, + getSeveredDaemon, + isSameIncarnation, + type SeveredDaemonController, + type SeveredDaemonIncarnation, + type SeveredDaemonRestartResult +} from './daemon-host-image-state' +import { getCurrentDaemonAdapter } from './daemon-provider-routing' +import { restartDaemon } from './daemon-provider-restart' +import { getDaemonProvider } from './daemon-provider-state' +import { DegradedDaemonPtyProvider } from './degraded-daemon-pty-provider' +import { getDaemonPidPath } from './daemon-spawner' + +type Dependencies = { + inspect: () => ReturnType + restart: (expectedDaemonPid: number) => Promise<{ killedCount: number }> + listSessions: () => Promise<{ isAlive: boolean }[]> + /** Local terminals opened while quarantined; a restart ends them as well. */ + countFallbackSessions: () => number +} + +const defaultDependencies: Dependencies = { + inspect: () => + inspectDaemonHostImage(readDaemonPidRecord(getDaemonPidPath(getDaemonRuntimeDir()))), + restart: (expectedDaemonPid) => restartDaemon({ expectedDaemonPid }), + listSessions: async () => { + const provider = getDaemonProvider() + return provider ? getCurrentDaemonAdapter(provider).listSessions() : [] + }, + countFallbackSessions: () => { + const provider = getDaemonProvider() + return provider instanceof DegradedDaemonPtyProvider + ? provider.getSessionIdsRoutedTo('fallback').length + : 0 + } +} + +export function createSeveredDaemonController( + dependencies: Dependencies = defaultDependencies +): SeveredDaemonController { + return { + countLiveSessions: async () => { + try { + const daemonSessions = (await dependencies.listSessions()).filter( + (session) => session.isAlive + ).length + return daemonSessions + dependencies.countFallbackSessions() + } catch { + return null + } + }, + restart: async (expected) => restartVerifiedSeveredDaemon(expected, dependencies) + } +} + +async function restartVerifiedSeveredDaemon( + expected: SeveredDaemonIncarnation, + dependencies: Dependencies +): Promise { + const current = dependencies.inspect() + if (current.verdict === 'unknown') { + return { status: 'refused', reason: 'unverifiable' } + } + if ( + current.pid === null || + !current.linuxStartTicks || + !current.bootId || + !isSameIncarnation( + { pid: current.pid, linuxStartTicks: current.linuxStartTicks, bootId: current.bootId }, + expected + ) + ) { + clearSeveredDaemon() + return { status: 'refused', reason: 'replaced' } + } + const { killedCount } = await dependencies.restart(expected.pid) + clearSeveredDaemon() + return { status: 'restarted', killedCount } +} + +/** True once the endpoint's pid record names a verified incarnation other than the severed one. */ +export function hasSeveredDaemonBeenReplaced( + pidPath: string, + inspect: (pidPath: string) => ReturnType = (path) => + inspectDaemonHostImage(readDaemonPidRecord(path)) +): boolean { + const severed = getSeveredDaemon() + if (!severed) { + return true + } + const current = inspect(pidPath) + if ( + current.verdict === 'unknown' || + current.verdict === 'severed' || + current.pid === null || + !current.linuxStartTicks || + !current.bootId || + isSameIncarnation( + { pid: current.pid, linuxStartTicks: current.linuxStartTicks, bootId: current.bootId }, + severed + ) + ) { + return false + } + clearSeveredDaemon() + return true +} diff --git a/src/main/daemon/daemon-spawner.ts b/src/main/daemon/daemon-spawner.ts index 4e490d4c6ea..fe96bb56e65 100644 --- a/src/main/daemon/daemon-spawner.ts +++ b/src/main/daemon/daemon-spawner.ts @@ -39,6 +39,8 @@ export type DaemonPidFile = { export type DaemonProcessHandle = { mode?: 'degraded-new-pty-fallback' + /** Fresh terminals stay off this daemon until a new incarnation replaces it. */ + quarantine?: 'host-image-severed' /** Set when the launcher kept a daemon some earlier app launch forked, rather than forking one. */ adopted?: true releaseAdoptionLease?(): void diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index a19d91ba524..eed79501220 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -479,7 +479,7 @@ describe('DegradedDaemonPtyProvider', () => { exitedBeforeSpawnReply: true }) - expect(provider.getCurrentDaemonSessionIds()).toEqual([]) + expect(provider.getSessionIdsRoutedTo('current-daemon')).toEqual([]) }) it('routes a previously daemon-backed id to fallback after daemon exit removes the mapping', async () => { @@ -670,7 +670,7 @@ describe('DegradedDaemonPtyProvider', () => { expect(exitSpy).toHaveBeenCalledOnce() expect(exitSpy).toHaveBeenCalledWith({ id: 'current-session', code: -1 }) - expect(provider.getCurrentDaemonSessionIds()).toEqual([]) + expect(provider.getSessionIdsRoutedTo('current-daemon')).toEqual([]) expect(provider.hasPty('legacy-session')).toBe(true) }) diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index ee5c4c31366..f1db303ddfd 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -311,12 +311,14 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { return shutdownDegradedFallbackSessions(this.sessionProviders, this.fallback) } - getCurrentDaemonSessionIds(): string[] { - return listProviderSessionIds(this.sessionProviders, this.current) + /** Sessions routed to the preserved daemon, or to the local fallback a restart also ends. */ + getSessionIdsRoutedTo(route: 'current-daemon' | 'fallback'): string[] { + const target = route === 'fallback' ? this.fallback : this.current + return listProviderSessionIds(this.sessionProviders, target) } fanoutCurrentDaemonSyntheticExits(code: number): void { - for (const id of this.getCurrentDaemonSessionIds()) { + for (const id of this.getSessionIdsRoutedTo('current-daemon')) { this.sessionProviders.delete(id) // Why: restart kills listed sessions even when the adapter did not track them active. // oxlint-disable-next-line unicorn/no-useless-spread -- copy-safe: listeners may unsubscribe during iteration diff --git a/src/main/runtime/rpc/methods/index.ts b/src/main/runtime/rpc/methods/index.ts index a4049c49d3c..db3786c3e2f 100644 --- a/src/main/runtime/rpc/methods/index.ts +++ b/src/main/runtime/rpc/methods/index.ts @@ -19,6 +19,7 @@ import { ORCHESTRATION_METHODS } from './orchestration' import { NOTIFICATION_METHODS } from './notifications' import { STATS_METHODS } from './stats' import { DIAGNOSTICS_METHODS } from './diagnostics' +import { TERMINAL_DAEMON_METHODS } from './terminal-daemon' import { ACCOUNT_METHODS } from './accounts' import { ANTIGRAVITY_ACCOUNT_METHODS } from './antigravity-accounts' import { PREFLIGHT_METHODS } from './preflight' @@ -93,6 +94,7 @@ export const ALL_RPC_METHODS = [ ...NOTIFICATION_METHODS, ...STATS_METHODS, ...DIAGNOSTICS_METHODS, + ...TERMINAL_DAEMON_METHODS, ...ACCOUNT_METHODS, ...ANTIGRAVITY_ACCOUNT_METHODS, ...PREFLIGHT_METHODS, diff --git a/src/main/runtime/rpc/methods/terminal-daemon.ts b/src/main/runtime/rpc/methods/terminal-daemon.ts new file mode 100644 index 00000000000..98cf2777419 --- /dev/null +++ b/src/main/runtime/rpc/methods/terminal-daemon.ts @@ -0,0 +1,21 @@ +import { TerminalDaemonRestartSeveredParams } from '../../../../shared/rpc-contract/terminal-daemon-params' +import { + readSeveredDaemonStatus, + restartSeveredDaemon +} from '../../../daemon/daemon-host-image-state' +import { defineMethod } from '../core' + +export const TERMINAL_DAEMON_METHODS = [ + defineMethod({ + name: 'terminalDaemon.severedStatus', + permission: 'workspace', + params: null, + handler: async () => ({ severed: await readSeveredDaemonStatus() }) + }), + defineMethod({ + name: 'terminalDaemon.restartSevered', + permission: 'host-admin', + params: TerminalDaemonRestartSeveredParams, + handler: async (params) => restartSeveredDaemon(params) + }) +] diff --git a/src/shared/daemon-lifecycle-telemetry.ts b/src/shared/daemon-lifecycle-telemetry.ts index afa9c71ec7c..e755aed4f34 100644 --- a/src/shared/daemon-lifecycle-telemetry.ts +++ b/src/shared/daemon-lifecycle-telemetry.ts @@ -9,7 +9,8 @@ export const DAEMON_REPLACE_REASONS = [ 'stale_bundle', 'different_app_path', 'failed_health_check', - 'severed_tcc_attribution' + 'severed_tcc_attribution', + 'severed_host_image' ] as const export type DaemonReplaceReason = (typeof DAEMON_REPLACE_REASONS)[number] diff --git a/src/shared/host-operation-ownership.ts b/src/shared/host-operation-ownership.ts index 0b40e5d539d..3716efcac35 100644 --- a/src/shared/host-operation-ownership.ts +++ b/src/shared/host-operation-ownership.ts @@ -62,6 +62,7 @@ export const HOST_OPERATION_OWNERSHIP = { stats: 'host', status: 'host', terminal: 'host', + terminalDaemon: 'host', ui: 'host', updater: 'host', workspacePorts: 'host', diff --git a/src/shared/rpc-contract/rpc-params-catalog.generated.ts b/src/shared/rpc-contract/rpc-params-catalog.generated.ts index 6adf73e2683..1898c288e85 100644 --- a/src/shared/rpc-contract/rpc-params-catalog.generated.ts +++ b/src/shared/rpc-contract/rpc-params-catalog.generated.ts @@ -528,6 +528,7 @@ import { QueuedMessageEditHoldParams, QueuedMessageUpdateParams } from './structured-agent-session-queued-edit-params' +import { TerminalDaemonRestartSeveredParams } from './terminal-daemon-params' import { TerminalAdoptOrphans } from './terminal-orphan-params' import { TerminalQuickCommandsUpdate } from './terminal-quick-command-params' import { @@ -1268,6 +1269,8 @@ export const RPC_PARAMS_BY_METHOD = { 'terminal.unsubscribe': TerminalUnsubscribe, 'terminal.updateViewport': TerminalUpdateViewport, 'terminal.wait': TerminalWait, + 'terminalDaemon.restartSevered': TerminalDaemonRestartSeveredParams, + 'terminalDaemon.severedStatus': null, 'ui.get': null, 'ui.recordFeatureInteraction': FeatureInteractionIdParam, 'ui.set': UiUpdate, diff --git a/src/shared/rpc-contract/terminal-daemon-params.ts b/src/shared/rpc-contract/terminal-daemon-params.ts new file mode 100644 index 00000000000..5d20d6985ea --- /dev/null +++ b/src/shared/rpc-contract/terminal-daemon-params.ts @@ -0,0 +1,7 @@ +import { z } from 'zod' + +export const TerminalDaemonRestartSeveredParams = z.object({ + pid: z.number().int().positive(), + linuxStartTicks: z.string().min(1).max(32), + bootId: z.string().min(1).max(64) +})