diff --git a/config/docker/daemon-scope-lifetime/Dockerfile b/config/docker/daemon-scope-lifetime/Dockerfile index 128e491bc61..918aacc18cc 100644 --- a/config/docker/daemon-scope-lifetime/Dockerfile +++ b/config/docker/daemon-scope-lifetime/Dockerfile @@ -1,4 +1,6 @@ -FROM node:22-bookworm-slim +ARG BASE_IMAGE=debian:bookworm-slim +FROM node:22-bookworm-slim AS node +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive ENV container=docker @@ -9,7 +11,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && useradd --uid 1100 --create-home --shell /bin/sh orca-repro -COPY process-fixture.cjs run-cases.cjs /opt/daemon-scope-lifetime/ +COPY --from=node /usr/local/bin/node /usr/local/bin/node +COPY process-fixture.cjs run-cases.cjs terminal-scope-cases.cjs /opt/daemon-scope-lifetime/ STOPSIGNAL SIGRTMIN+3 CMD ["/lib/systemd/systemd"] diff --git a/config/docker/daemon-scope-lifetime/README.md b/config/docker/daemon-scope-lifetime/README.md index 27113aa8113..234ce6897d6 100644 --- a/config/docker/daemon-scope-lifetime/README.md +++ b/config/docker/daemon-scope-lifetime/README.md @@ -12,6 +12,8 @@ The baseline uses the production scope launcher from the selected Git revision. Four cycles cover graceful SIGTERM, SIGKILL, ordinary exit, and SIGKILL of the daemon's entire process group. The baseline must retain one additional worker each cycle. The candidate must return to zero workers and remove each old cgroup after every cycle. The report includes retained worker RSS and cgroup memory. Both must preserve the daemon and worker when only the runtime exits, and an unrelated canary in its own scope. The candidate must also preserve a scope whose daemon lacks exclusive ownership, and keep live work running if the lifetime pipe closes prematurely. +Each daemon also starts terminals through the production terminal-scope wrap, on Ubuntu 22.04 (systemd 249, which rejects `OOMPolicy` on scopes), Debian 12 (252) and Debian 13 (257). The baseline reports what one kernel OOM kill in today's shared scope does to the daemon and the other terminals. The candidate must put each terminal in its own scope under a slice beside the daemon's, keep the daemon and terminals 1 and 3 running when terminal 2 is OOM-killed, keep terminal 2's shell when `OOMPolicy=continue` was accepted, enforce `ORCA_TERMINAL_PROCESS_CEILING` per terminal, and reap the slice, a SIGTERM-ignoring job and every terminal when the daemon scope is stopped or the daemon is SIGKILLed. `ORCA_DAEMON_SCOPE_BASE_IMAGES` overrides the comma-separated image list. + This proves the lifetime cleanup mechanism against real Linux process and systemd behavior. It does not reproduce a user's complete workload or assert that every reported OOM has this cause. `ORCA_DOCKER` overrides the Docker executable. `ORCA_DOCKER_PLATFORM` overrides the Linux architecture. diff --git a/config/docker/daemon-scope-lifetime/bundle-entry.ts b/config/docker/daemon-scope-lifetime/bundle-entry.ts index ea8ba04d1fb..f2358976b7d 100644 --- a/config/docker/daemon-scope-lifetime/bundle-entry.ts +++ b/config/docker/daemon-scope-lifetime/bundle-entry.ts @@ -1,7 +1,24 @@ +import { ownFreshDaemonScope } from '../../../src/main/daemon/daemon-cgroup-scope' import { startDaemonScopeDeathWatch } from '../../../src/main/daemon/daemon-scope-death-watch' +import { initializeTerminalScopes } from '../../../src/main/daemon/terminal-scopes-runtime' export { buildDurableDaemonScopeCommand } from '../../../src/main/daemon/daemon-cgroup-scope' +export { wrapPtySpawnInTerminalScope } from '../../../src/main/daemon/terminal-scopes-runtime' export function startScopeReaper(launchNonce: string, freshScope: boolean) { - return startDaemonScopeDeathWatch({ freshScope, launchNonce, log: () => {} }) + return startDaemonScopeDeathWatch({ + daemonScope: ownFreshDaemonScope(freshScope, launchNonce), + log: () => {} + }) +} + +/** What daemon-entry does at start: terminal scopes, then the death watch. */ +export function startTerminalScopes(launchNonce: string) { + const daemonScope = ownFreshDaemonScope(true, launchNonce) + const events: unknown[] = [] + const ceiling = initializeTerminalScopes(daemonScope, (event, details) => + events.push({ event, ...details }) + ) + startDaemonScopeDeathWatch({ daemonScope, log: () => {} }) + return { ceiling, events } } diff --git a/config/docker/daemon-scope-lifetime/process-fixture.cjs b/config/docker/daemon-scope-lifetime/process-fixture.cjs index 9c8161fcdda..8b0d83349c3 100644 --- a/config/docker/daemon-scope-lifetime/process-fixture.cjs +++ b/config/docker/daemon-scope-lifetime/process-fixture.cjs @@ -28,6 +28,23 @@ if (role === 'worker') { env: command.env }) daemon.unref() +} else if (role === 'daemon' && ownership === 'terminals') { + // The baseline bundle predates terminal scopes, so its terminals stay in the daemon's scope. + const { startTerminalScopes = () => ({}), wrapPtySpawnInTerminalScope } = require(bundlePath) + const started = startTerminalScopes(nonce) + // Same argv and env a PTY spawn gets; a pipe-less child stands in for node-pty. + const terminals = JSON.parse(process.env.ORCA_FIXTURE_TERMINALS).map((script) => { + const wrapped = wrapPtySpawnInTerminalScope + ? wrapPtySpawnInTerminalScope('/bin/sh', ['-c', script], process.env) + : { file: '/bin/sh', args: ['-c', script], env: process.env } + return spawn(wrapped.file, wrapped.args, { env: wrapped.env, stdio: 'ignore' }).pid + }) + fs.writeFileSync( + `${resultPath}.daemon`, + JSON.stringify({ pid: process.pid, terminals, ...started }) + ) + process.on('SIGTERM', () => process.exit(0)) + setInterval(() => {}, 1000) } else if (role === 'daemon') { const { startScopeReaper } = require(bundlePath) const watcher = startScopeReaper?.(nonce, ownership !== 'shared') diff --git a/config/docker/daemon-scope-lifetime/run-cases.cjs b/config/docker/daemon-scope-lifetime/run-cases.cjs index 3b7cf0b8a15..da6be2a3431 100644 --- a/config/docker/daemon-scope-lifetime/run-cases.cjs +++ b/config/docker/daemon-scope-lifetime/run-cases.cjs @@ -2,6 +2,7 @@ const assert = require('node:assert/strict') const fs = require('node:fs') const { join } = require('node:path') const { spawn, execFileSync } = require('node:child_process') +const { runTerminalScopeCases } = require('./terminal-scope-cases.cjs') const [bundlePath, mode] = process.argv.slice(2) assert.ok(['baseline', 'candidate'].includes(mode)) @@ -184,8 +185,10 @@ async function main() { }) ) } + await runTerminalScopeCases(bundlePath, prefix, mode) + assert.ok(stat(canary.pid), 'unrelated canary was killed') } finally { - for (const unit of scopes) { + for (const unit of [...scopes, `orca-daemon-${prefix}-*`, 'orca_ptys_*']) { try { execFileSync('systemctl', ['--user', '--signal=SIGKILL', 'kill', unit], { stdio: 'ignore' }) } catch {} diff --git a/config/docker/daemon-scope-lifetime/terminal-scope-cases.cjs b/config/docker/daemon-scope-lifetime/terminal-scope-cases.cjs new file mode 100644 index 00000000000..0cf2d7288b8 --- /dev/null +++ b/config/docker/daemon-scope-lifetime/terminal-scope-cases.cjs @@ -0,0 +1,193 @@ +// Per-terminal scopes against the real user manager: an OOM kill or a fork limit in one +// terminal leaves the daemon and the other terminals running, and the daemon scope's end +// takes the terminal slice with it. The baseline only reports what one OOM kill does today. +const assert = require('node:assert/strict') +const fs = require('node:fs') +const { join } = require('node:path') +const { spawn, execFileSync } = require('node:child_process') + +const fixture = join(__dirname, 'process-fixture.cjs') +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +const alive = (pid) => { + try { + return !fs.readFileSync(`/proc/${pid}/stat`, 'utf8').includes(') Z ') + } catch { + return false + } +} +const cgroupOf = (pid) => + fs.readFileSync(`/proc/${pid}/cgroup`, 'utf8').trim().split('\n').at(-1).replace(/^0::/, '') +const cgroupFile = (cgroup, name) => fs.readFileSync(join('/sys/fs/cgroup', cgroup, name), 'utf8') +const pidFrom = (path) => (fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : null) +const systemctl = (...args) => + execFileSync('systemctl', ['--user', ...args], { encoding: 'utf8' }).trim() + +async function until(predicate, message, timeoutMs = 15000) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (predicate()) { + return + } + await delay(50) + } + throw new Error(message) +} + +async function startDaemon(bundlePath, nonce, terminals, env = {}, mode = 'candidate') { + const resultPath = `/tmp/${nonce}` + const runtime = spawn( + process.execPath, + [fixture, 'runtime', resultPath, bundlePath, nonce, 'terminals'], + { + stdio: 'inherit', + env: { ...process.env, ...env, ORCA_FIXTURE_TERMINALS: JSON.stringify(terminals) } + } + ) + await new Promise((resolve) => runtime.on('exit', resolve)) + await until(() => fs.existsSync(`${resultPath}.daemon`), 'terminal daemon did not start') + const daemon = JSON.parse(fs.readFileSync(`${resultPath}.daemon`, 'utf8')) + const unit = `orca-daemon-${nonce}.scope` + assert.ok(cgroupOf(daemon.pid).endsWith(`/${unit}`), 'daemon is not in its own scope') + if (mode === 'baseline') { + return { ...daemon, unit } + } + // systemd-run moves itself into the scope before it execs the shell. + await until( + () => daemon.terminals.every((pid) => /\/orca_pty_[^/]+\.scope$/.test(cgroupOf(pid))), + `terminals never entered their scopes: ${JSON.stringify(daemon.events)}` + ) + const scopes = daemon.terminals.map(cgroupOf) + const slice = scopes[0].split('/').slice(0, -1).join('/') + assert.equal(new Set(scopes).size, scopes.length, 'two terminals share a scope') + for (const scope of scopes) { + assert.match(scope, /\/orca_ptys_[^/]+\.slice\/orca_pty_[^/]+\.scope$/) + assert.ok(scope.startsWith(`${slice}/`), 'terminal scopes are not siblings in one slice') + } + assert.ok(!slice.includes(unit), 'terminal slice is nested in the daemon scope') + // Without delegated controllers the OOM and fork cases prove nothing. + assert.match(cgroupFile(slice, 'cgroup.controllers'), /\bmemory\b.*\bpids\b|\bpids\b.*\bmemory\b/) + return { ...daemon, resultPath, unit, slice, sliceUnit: slice.split('/').at(-1), scopes } +} + +async function expectReaped(daemon, extraPids) { + await until( + () => [daemon.pid, ...daemon.terminals, ...extraPids].every((pid) => !alive(pid)), + 'terminal processes outlived the daemon scope' + ) + await until( + () => !fs.existsSync(join('/sys/fs/cgroup', daemon.slice)), + 'terminal slice outlived the daemon scope' + ) +} + +async function oomCase(bundlePath, prefix, systemdVersion, mode) { + const r = `/tmp/${prefix}-oom` + const hog = 'const a = []; for (;;) a.push(Buffer.alloc(1 << 20, 1))' + const daemon = await startDaemon( + bundlePath, + `${prefix}-oom`, + [ + // Ignores SIGTERM and SIGHUP, so only the scope stop's SIGKILL can end it. + `(trap '' TERM HUP; exec sleep 600) & echo $! > ${r}.stubborn; exec sleep 600`, + `sleep 600 & echo $! > ${r}.sleeper; while [ ! -f ${r}.go ]; do sleep 0.05; done; ` + + `${process.execPath} -e '${hog}'; echo $? > ${r}.hog-exit; wait`, + 'exec sleep 600' + ], + {}, + mode + ) + const properties = daemon.events?.find((entry) => entry.event === 'terminal-scopes').properties + const continuePolicy = properties?.includes('OOMPolicy=continue') === true + const [term1, term2, term3] = daemon.terminals + await until(() => pidFrom(`${r}.sleeper`) && pidFrom(`${r}.stubborn`), 'terminals not ready') + const sleeper = pidFrom(`${r}.sleeper`) + // Today every terminal shares the daemon's scope, so its limit is the only one to hit. + const [limited, memoryMax] = + mode === 'baseline' ? [daemon.unit, '192M'] : [daemon.scopes[1].split('/').at(-1), '64M'] + systemctl('set-property', '--runtime', limited, `MemoryMax=${memoryMax}`, 'MemorySwapMax=0') + fs.writeFileSync(`${r}.go`, '') + await until(() => fs.existsSync(`${r}.hog-exit`) || !alive(term2), 'hog was not OOM-killed') + await delay(500) + const terminal2 = alive(term2) && alive(sleeper) ? 'hog-only' : 'whole-terminal' + if (mode === 'baseline') { + const daemonState = alive(daemon.pid) ? 'live' : 'killed' + const neighbours = alive(term1) && alive(term3) ? 'live' : 'killed' + console.log(JSON.stringify({ mode, systemdVersion, daemon: daemonState, neighbours })) + try { + systemctl('stop', daemon.unit) + } catch { + // The OOM stop already ended the scope, and --collect may have unloaded it. + } + return + } + // Without OOMPolicy, systemd's own policy decides terminal 2's fate; neighbours never share it. + if (continuePolicy) { + assert.equal(terminal2, 'hog-only', 'OOMPolicy=continue still stopped terminal 2') + } + assert.ok(alive(daemon.pid), 'an OOM kill in terminal 2 killed the daemon') + assert.ok(alive(term1) && alive(term3), 'an OOM kill in terminal 2 killed its neighbours') + console.log( + JSON.stringify({ + systemdVersion, + oomPolicyContinue: continuePolicy, + oomInTerminal2: terminal2, + daemon: 'live', + terminals1And3: 'live' + }) + ) + const stubborn = pidFrom(`${r}.stubborn`) + systemctl('stop', daemon.unit) + await expectReaped(daemon, [sleeper, stubborn]) + console.log(JSON.stringify({ daemonScopeStopped: 'slice-and-terminals-reaped' })) +} + +async function ceilingCase(bundlePath, prefix) { + const r = `/tmp/${prefix}-ceiling` + const daemon = await startDaemon( + bundlePath, + `${prefix}-ceiling`, + [ + // The shell exits at the first refused fork; its children keep the scope at the limit. + `while [ ! -f ${r}.bomb ]; do sleep 0.05; done; ` + + 'i=0; while [ $i -lt 100 ]; do sleep 600 & i=$((i + 1)); done; wait', + `while [ ! -f ${r}.go ]; do sleep 0.05; done; for i in 1 2 3 4 5; do sleep 600 & done; ` + + `echo ok > ${r}.forked; wait` + ], + { ORCA_TERMINAL_PROCESS_CEILING: '64' } + ) + assert.equal(daemon.ceiling.state, 'on', JSON.stringify(daemon.ceiling)) + const [forkBomb, neighbour] = daemon.scopes + for (const scope of daemon.scopes) { + assert.equal(cgroupFile(scope, 'pids.max').trim(), '64') + } + fs.writeFileSync(`${r}.bomb`, '') + await until( + () => /^max [1-9]/m.test(cgroupFile(forkBomb, 'pids.events')), + 'terminal 1 never reached its own limit' + ) + fs.writeFileSync(`${r}.go`, '') + await until(() => fs.existsSync(`${r}.forked`), 'terminal 2 could not fork beside terminal 1') + assert.match(cgroupFile(neighbour, 'pids.events'), /^max 0$/m) + assert.ok(alive(daemon.pid), 'the fork limit in terminal 1 starved the daemon') + console.log(JSON.stringify({ ceilingPerTerminal: 64, terminal1: 'at-limit', terminal2: 'forks' })) + const descendants = fs + .readFileSync(join('/sys/fs/cgroup', neighbour, 'cgroup.procs'), 'utf8') + .split('\n') + .filter(Boolean) + .map(Number) + process.kill(daemon.pid, 'SIGKILL') + await expectReaped(daemon, descendants) + console.log(JSON.stringify({ daemonSigkill: 'slice-and-terminals-reaped' })) +} + +async function runTerminalScopeCases(bundlePath, prefix, mode) { + const systemdVersion = Number( + execFileSync('systemctl', ['--version'], { encoding: 'utf8' }).match(/^systemd (\d+)/)[1] + ) + await oomCase(bundlePath, prefix, systemdVersion, mode) + if (mode === 'candidate') { + await ceilingCase(bundlePath, prefix) + } +} + +module.exports = { runTerminalScopeCases } diff --git a/config/main-sync-spawn-baseline.txt b/config/main-sync-spawn-baseline.txt index 59644c5f559..99d02ed756b 100644 --- a/config/main-sync-spawn-baseline.txt +++ b/config/main-sync-spawn-baseline.txt @@ -22,9 +22,8 @@ 1 src/main/daemon/daemon-process-identity-query.ts 1 src/main/daemon/daemon-process-start-time.ts 1 src/main/daemon/daemon-user-manager-lifetime.ts -1 src/main/daemon/pty-process-ceiling-runtime.ts -1 src/main/daemon/pty-process-ceiling.ts 1 src/main/daemon/pty-subprocess/shell-launch-plan.ts +1 src/main/daemon/terminal-scopes.ts 1 src/main/emulator/serve-sim-runtime-materializer.ts 1 src/main/git/command-runner/git-exec-file.ts 1 src/main/git/repo-detection-command.ts diff --git a/config/scripts/run-daemon-scope-lifetime-docker.mjs b/config/scripts/run-daemon-scope-lifetime-docker.mjs index 3075c78665d..eba8a69c0d4 100644 --- a/config/scripts/run-daemon-scope-lifetime-docker.mjs +++ b/config/scripts/run-daemon-scope-lifetime-docker.mjs @@ -94,10 +94,24 @@ async function bundle(mode) { }) } -try { - await bundle('baseline') - await bundle('candidate') - runDocker(['build', '--platform', platform, '-t', image, dockerDir]) +// Why three releases: OOMPolicy on scopes is rejected by systemd 249 (Ubuntu 22.04), accepted by +// Debian's 252 and native from 253 (Debian 13's 257). +const baseImages = ( + process.env.ORCA_DAEMON_SCOPE_BASE_IMAGES ?? + 'ubuntu:22.04,debian:bookworm-slim,debian:trixie-slim' +).split(',') + +function runCases(baseImage) { + runDocker([ + 'build', + '--platform', + platform, + '--build-arg', + `BASE_IMAGE=${baseImage}`, + '-t', + image, + dockerDir + ]) runDocker([ 'run', '--detach', @@ -139,7 +153,7 @@ try { container, 'timeout', '--kill-after=5s', - '90s', + '240s', 'runuser', '-u', 'orca-repro', @@ -157,11 +171,24 @@ try { process.stdout.write(result.stdout ?? '') process.stderr.write(result.stderr ?? '') if (result.status !== 0) { - throw new Error(`${mode} scope lifetime oracle failed with ${result.status}`) + throw new Error(`${mode} scope lifetime oracle failed on ${baseImage} with ${result.status}`) + } + } +} + +try { + await bundle('baseline') + await bundle('candidate') + for (const baseImage of baseImages) { + console.log(`== ${baseImage}`) + try { + runCases(baseImage) + } finally { + runDocker(['rm', '--force', container], true) } } console.log( - 'Linux systemd scope lifetime oracle passed: baseline accumulates orphans; candidate reaps them; live and unrelated work survives.' + 'Linux systemd scope lifetime oracle passed: baseline accumulates orphans; candidate reaps them; live and unrelated work survives; each terminal is its own scope.' ) } finally { runDocker(['rm', '--force', container], true) diff --git a/src/cli/specs/serve.ts b/src/cli/specs/serve.ts index d107985ed00..afff43749ad 100644 --- a/src/cli/specs/serve.ts +++ b/src/cli/specs/serve.ts @@ -29,7 +29,7 @@ export const SERVE_COMMAND_SPECS: CommandSpec[] = [ 'Exits with code 3, without starting anything, when another Orca already serves this profile or cannot be shown to have stopped.', 'On macOS, after Orca serve aborts at startup (it could not register with Launch Services), later runs from the same environment refuse for 30 minutes with exit code 78. Use --force to try anyway.', '--supervise is for service managers (launchd, systemd). Orca serve then restarts the server itself with growing delays instead of exiting when it stops, waits while an update replaces the app, and waits while another Orca serves this profile. After 6 failures in 15 minutes it gives up and exits with code 0 so the manager stops too, and later supervised runs refuse until a new build is installed or --force is passed. Not available on Windows yet.', - 'Experimental and off by default: set ORCA_TERMINAL_PROCESS_CEILING= (64 or more) before the terminal host starts to have the OS cap how many processes all terminals together may run (Linux, through a systemd user slice; needs a user manager that outlives logout) or each terminal may run (Windows). `orca serve` reports the ceiling as on or off, with the reason.' + 'Experimental and off by default: set ORCA_TERMINAL_PROCESS_CEILING= (64 or more) before the terminal host starts to have the OS cap how many processes each terminal may run (Linux, through a systemd user scope per terminal, which needs a user manager that outlives logout; Windows, through a job per terminal). `orca serve` reports the ceiling as on or off, with the reason.' ], examples: [ 'orca serve', diff --git a/src/main/daemon/daemon-cgroup-scope.ts b/src/main/daemon/daemon-cgroup-scope.ts index 4ef52fb9249..f0d3cf50616 100644 --- a/src/main/daemon/daemon-cgroup-scope.ts +++ b/src/main/daemon/daemon-cgroup-scope.ts @@ -355,6 +355,22 @@ export function buildDurableDaemonScopeCommand( } } +/** + * The daemon's scope unit when the launcher created it for this daemon and the daemon is still in + * it. Migrated legacy scopes may also hold the GUI, so a matching name alone never qualifies. + */ +export function ownFreshDaemonScope( + freshScope: boolean, + launchNonce: string | undefined, + detectScope: () => string | null = detectOwnCgroupScopeUnit +): string | null { + if (!freshScope || !launchNonce) { + return null + } + const unit = daemonScopeUnitName(launchNonce) + return detectScope() === unit ? unit : null +} + /** * The daemon's own cgroup membership, read from /proc rather than trusted from the launcher's * intent, so a daemon that fell back (or was adopted from before this fix) cannot misreport diff --git a/src/main/daemon/daemon-entry.ts b/src/main/daemon/daemon-entry.ts index fb9371f33e2..7ef5053f840 100644 --- a/src/main/daemon/daemon-entry.ts +++ b/src/main/daemon/daemon-entry.ts @@ -13,7 +13,7 @@ import { warmWindowsConptyOnce } from './windows-conpty-warmup' import { warmPwshAvailabilityCache } from '../pwsh' import { createDaemonFileLog, createNoopDaemonFileLog } from './daemon-file-log' import { PROTOCOL_VERSION } from './types' -import { detectOwnCgroupScopeUnit } from './daemon-cgroup-scope' +import { detectOwnCgroupScopeUnit, ownFreshDaemonScope } from './daemon-cgroup-scope' import { DAEMON_EXIT_ENDPOINT_OCCUPIED, DaemonEndpointUnavailableError @@ -26,7 +26,7 @@ import { MacosLoginSessionDeathWatch } from './macos-login-session-death-watch' import { readCurrentProcessMacSystemResolverHealth } from '../network/macos-system-resolver-health' import { readCurrentDaemonReadyIdentity } from './daemon-ready-identity' import { publishDaemonPidFile } from './daemon-spawner' -import { initializePtyProcessCeiling } from './pty-process-ceiling-runtime' +import { initializeTerminalScopes } from './terminal-scopes-runtime' import { isNativePtyException } from './daemon-native-pty-exception' import { startDaemonScopeDeathWatch } from './daemon-scope-death-watch' import { isWindowsProcessTableAvailable } from '../windows/windows-process-table' @@ -139,13 +139,11 @@ async function main(): Promise { // Fail-open: a broken log path must never block daemon startup. const daemonLog = logFilePath ? createDaemonFileLog(logFilePath) : createNoopDaemonFileLog() daemonLog.log('startup', { protocolVersion: PROTOCOL_VERSION, socketPath }) - const processCeiling = initializePtyProcessCeiling(launchNonce ?? '') - startDaemonScopeDeathWatch({ - freshScope: freshDaemonScope === true, - launchNonce, - ceilingSlice: processCeiling.state === 'on' ? processCeiling.unit : null, - log: (event, details) => daemonLog.log(event, details) - }) + const daemonScope = ownFreshDaemonScope(freshDaemonScope === true, launchNonce) + const log = (event: string, details?: Record): void => + daemonLog.log(event, details) + const processCeiling = initializeTerminalScopes(daemonScope, log) + startDaemonScopeDeathWatch({ daemonScope, log }) void warmPwshAvailabilityCache() // Why: detached daemons destroy stderr, so the preflight's console.warn is lost; diff --git a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts index 22e42b543d7..516065b03e9 100644 --- a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts +++ b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts @@ -3,7 +3,7 @@ import { PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION } from './types' describe('foreground-confirmation daemon protocol', () => { it('rejects daemons from before the fresh-confirmation RPC', () => { - expect(PROTOCOL_VERSION).toBe(45) + expect(PROTOCOL_VERSION).toBe(46) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(19) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(22) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(23) diff --git a/src/main/daemon/daemon-protocol-version.ts b/src/main/daemon/daemon-protocol-version.ts index de8e76e35ad..9a332f0c273 100644 --- a/src/main/daemon/daemon-protocol-version.ts +++ b/src/main/daemon/daemon-protocol-version.ts @@ -1,10 +1,11 @@ // Why: daemons survive app updates, so wire behavior must be version-gated. +// v46 starts each new Linux terminal in its own systemd scope beside the daemon's. // v45 starts new macOS tabs in the update-safe terminal host helper beside an existing daemon. // v44 gives every pane the `claude` function that follows the selected Claude account (v42 had it, // v43 dropped it with the revert); v41 stages long startup commands as sourced scripts; v40 rolled // the #25130/#24636 shell-wrapper changes and the wider agent list into a fresh daemon; older owners // stay attachable. -export const PROTOCOL_VERSION = 45 +export const PROTOCOL_VERSION = 46 export const CLAUDE_ACCOUNT_FUNCTION_DAEMON_PROTOCOL_VERSION = 42 export const CLAUDE_ACCOUNT_FUNCTION_REVERTED_DAEMON_PROTOCOL_VERSION = 43 // v39 gives plain fish panes Orca's codex function through XDG_DATA_DIRS. @@ -38,7 +39,7 @@ export const CLEAN_DISCONNECT_PROTOCOL_VERSION = 24 export const MODE_2031_UNSUBSCRIBE_FACT_PROTOCOL_VERSION = 29 export const PREVIOUS_DAEMON_PROTOCOL_VERSIONS = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44 + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45 ] as const export function supportsColorQueryReplyColors(protocolVersion: number): boolean { diff --git a/src/main/daemon/daemon-scope-death-watch.test.ts b/src/main/daemon/daemon-scope-death-watch.test.ts index ca1c2919519..b02cee1631e 100644 --- a/src/main/daemon/daemon-scope-death-watch.test.ts +++ b/src/main/daemon/daemon-scope-death-watch.test.ts @@ -3,6 +3,7 @@ import { PassThrough } from 'node:stream' import { afterEach, describe, expect, it, vi } from 'vitest' import type { ProcessSpec, SpawnedProcess } from '@orca/process-host/process-spec' +import { ownFreshDaemonScope } from './daemon-cgroup-scope' import { startDaemonScopeDeathWatch } from './daemon-scope-death-watch' const spawnProcess = vi.fn<(spec: ProcessSpec) => SpawnedProcess>() @@ -14,36 +15,28 @@ afterEach(() => { function options() { return { - freshScope: true, - launchNonce: 'launch-a', - platform: 'linux' as const, + daemonScope: 'orca-daemon-launch-a.scope', spawn: spawnProcess, - detectScope: () => 'orca-daemon-launch-a.scope', log: vi.fn() } } describe('daemon scope death watch ownership', () => { - it.each(['darwin', 'win32'] as const)('never spawns on %s', (platform) => { - expect(startDaemonScopeDeathWatch({ ...options(), platform })).toBeNull() - expect(spawnProcess).not.toHaveBeenCalled() - }) - - it('does not arm an adopted scope even when its nonce matches', () => { - expect(startDaemonScopeDeathWatch({ ...options(), freshScope: false })).toBeNull() - expect(spawnProcess).not.toHaveBeenCalled() - }) - - it.each([null, 'app-orca-123.scope', 'orca-daemon-other.scope'])( - 'rejects unverified ownership of %s', - (unit) => { - expect(startDaemonScopeDeathWatch({ ...options(), detectScope: () => unit })).toBeNull() - expect(spawnProcess).not.toHaveBeenCalled() + it('only a fresh scope the daemon is still in qualifies', () => { + const own = (unit: string | null) => () => unit + expect(ownFreshDaemonScope(true, 'launch-a', own('orca-daemon-launch-a.scope'))).toBe( + 'orca-daemon-launch-a.scope' + ) + // An adopted scope, a migrated legacy one, another daemon's, or none (macOS, Windows). + expect(ownFreshDaemonScope(false, 'launch-a', own('orca-daemon-launch-a.scope'))).toBeNull() + expect(ownFreshDaemonScope(true, undefined, own('orca-daemon-launch-a.scope'))).toBeNull() + for (const unit of [null, 'app-orca-123.scope', 'orca-daemon-other.scope']) { + expect(ownFreshDaemonScope(true, 'launch-a', own(unit))).toBeNull() } - ) + }) - it('does not arm without the launcher nonce', () => { - expect(startDaemonScopeDeathWatch({ ...options(), launchNonce: undefined })).toBeNull() + it('does not arm without an owned scope', () => { + expect(startDaemonScopeDeathWatch({ ...options(), daemonScope: null })).toBeNull() expect(spawnProcess).not.toHaveBeenCalled() }) diff --git a/src/main/daemon/daemon-scope-death-watch.ts b/src/main/daemon/daemon-scope-death-watch.ts index be48e8a3ea7..9cecd837729 100644 --- a/src/main/daemon/daemon-scope-death-watch.ts +++ b/src/main/daemon/daemon-scope-death-watch.ts @@ -1,6 +1,5 @@ import { spawnProcess } from '@orca/process-host' import type { ProcessSpec, SpawnedProcess } from '@orca/process-host/process-spec' -import { daemonScopeUnitName, detectOwnCgroupScopeUnit } from './daemon-cgroup-scope' // The pipe closes even on SIGKILL; a live/reused owner PID always vetoes cleanup. const SCOPE_DEATH_WATCH_SCRIPT = ` @@ -18,29 +17,19 @@ while IFS= read -r cgroup; do esac done < /proc/self/cgroup [ "$owned" = 1 ] || exit 0 -# Slice first: stopping the scope kills this script before a second StopUnit is sent. -exec systemctl --user --no-block stop \${3:+"$3"} "$2" +exec systemctl --user --no-block stop "$2" ` type ScopeDeathWatchOptions = { - freshScope: boolean - launchNonce?: string - /** The terminal ceiling slice; its scopes sit outside the daemon's and must be reaped too. */ - ceilingSlice?: string | null + /** From `ownFreshDaemonScope`; the terminal slice is bound to it, so stopping it reaps both. */ + daemonScope: string | null log: (event: string, details?: Record) => void - platform?: NodeJS.Platform - detectScope?: typeof detectOwnCgroupScopeUnit spawn?: (spec: ProcessSpec) => SpawnedProcess } export function startDaemonScopeDeathWatch(options: ScopeDeathWatchOptions): SpawnedProcess | null { - const { freshScope, launchNonce, log } = options - if ((options.platform ?? process.platform) !== 'linux' || !freshScope || !launchNonce) { - return null - } - const unit = daemonScopeUnitName(launchNonce) - // Migrated legacy scopes may also hold the GUI, so a name alone never authorizes cleanup. - if ((options.detectScope ?? detectOwnCgroupScopeUnit)() !== unit) { + const { daemonScope: unit, log } = options + if (!unit) { return null } const env = { ...process.env } @@ -48,14 +37,7 @@ export function startDaemonScopeDeathWatch(options: ScopeDeathWatchOptions): Spa try { const child = (options.spawn ?? spawnProcess)({ program: '/bin/sh', - args: [ - '-c', - SCOPE_DEATH_WATCH_SCRIPT, - 'orca-daemon-scope-watch', - String(process.pid), - unit, - ...(options.ceilingSlice ? [options.ceilingSlice] : []) - ], + args: ['-c', SCOPE_DEATH_WATCH_SCRIPT, 'orca-daemon-scope-watch', String(process.pid), unit], env, detached: true, stdio: ['pipe', 'ignore', 'ignore'] diff --git a/src/main/daemon/macos-daemon-launchd.test.ts b/src/main/daemon/macos-daemon-launchd.test.ts index bcef4116b5c..1912ec71612 100644 --- a/src/main/daemon/macos-daemon-launchd.test.ts +++ b/src/main/daemon/macos-daemon-launchd.test.ts @@ -220,7 +220,7 @@ it('uses one launchd label per profile and protocol', () => { expect(macDaemonJobLabel('/Users/a/Library/Application Support/orca')).not.toBe( macDaemonJobLabel('/Users/a/Library/Application Support/orca-dev') ) - expect(macDaemonJobLabel('/p')).toMatch(/^com\.stablyai\.orca\.terminal\.[0-9a-f]{16}\.v45$/) + expect(macDaemonJobLabel('/p')).toMatch(/^com\.stablyai\.orca\.terminal\.[0-9a-f]{16}\.v46$/) }) it.each(['13.4.1', '12.7.6'])( diff --git a/src/main/daemon/pty-process-ceiling-runtime.ts b/src/main/daemon/pty-process-ceiling-runtime.ts deleted file mode 100644 index 8eb53c0c840..00000000000 --- a/src/main/daemon/pty-process-ceiling-runtime.ts +++ /dev/null @@ -1,119 +0,0 @@ -/** The daemon's chosen terminal process ceiling, applied to every PTY spawn it makes. */ -import { existsSync } from 'node:fs' -import { runProcessSync } from '@orca/process-host' -import { userManagerEnv } from './daemon-cgroup-scope' -import { setPtyActiveProcessLimit } from '../windows/windows-pty-job' -import { resolvePtyProcessCeiling, type LinuxCeilingRuntime } from './pty-process-ceiling' -import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' - -let currentState: PtyProcessCeilingState = { state: 'off', reason: 'disabled' } -let currentLinux: LinuxCeilingRuntime | null = null -let scopeCounter = 0 - -/** Daemon start: decide, remember for every spawn, and return what the pid record reports. */ -export function initializePtyProcessCeiling(launchId: string): PtyProcessCeilingState { - const resolved = resolvePtyProcessCeiling({ - env: process.env, - platform: process.platform, - launchId: launchId || `${process.pid}`, - setWindowsLimit: setPtyActiveProcessLimit - }) - setPtyProcessCeilingForTest(resolved.state, resolved.linux) - if (resolved.linux) { - // Why on exit: every daemon exit path ends here, and its terminals are gone by then. - process.once('exit', releasePtyProcessCeiling) - } - return resolved.state -} - -export function setPtyProcessCeilingForTest( - state: PtyProcessCeilingState, - linux: LinuxCeilingRuntime | null -): void { - currentState = state - currentLinux = linux -} - -export function currentPtyProcessCeiling(): PtyProcessCeilingState { - return currentState -} - -const SESSION_KEYS = ['DBUS_SESSION_BUS_ADDRESS', 'XDG_RUNTIME_DIR'] as const - -/** - * Linux with the ceiling on: exec through `systemd-run --scope`, which keeps the pid and exit - * status, into a scope under the ceiling slice. `systemd-run` talks to the user manager the daemon - * verified; `env` then gives the shell its own session variables back. Off, it is the identity. - */ -export function wrapPtySpawnForCeiling( - file: string, - args: string[], - env: Record, - pidsMaxExists: (path: string) => boolean = existsSync -): { file: string; args: string[]; env: Record } { - const runtime = currentLinux - if (!runtime) { - return { file, args, env } - } - // Why: a restarted user manager drops the transient slice, and `systemd-run --slice` would - // silently create a plain one with no limit. - if (!pidsMaxExists(runtime.pidsMaxPath)) { - setPtyProcessCeilingForTest({ state: 'off', reason: 'slice_lost' }, null) - return { file, args, env } - } - scopeCounter += 1 - const managerEnv: Record = {} - for (const [key, value] of Object.entries(userManagerEnv(env))) { - if (typeof value === 'string' && key !== 'DBUS_SESSION_BUS_ADDRESS') { - managerEnv[key] = value - } - } - const unsets: string[] = [] - const sets: string[] = [] - for (const key of SESSION_KEYS) { - if (env[key] !== managerEnv[key]) { - if (env[key] === undefined) { - unsets.push('-u', key) - } else { - sets.push(`${key}=${env[key]}`) - } - } - } - const restore = runtime.envBinary && unsets.length + sets.length > 0 - return { - file: runtime.systemdRun, - args: [ - '--user', - '--scope', - '--quiet', - '--collect', - `--slice=${runtime.slice}`, - `--unit=orca_pty_${runtime.launchId}_${scopeCounter}.scope`, - '--', - ...(restore && runtime.envBinary ? [runtime.envBinary, ...unsets, ...sets] : []), - file, - ...args - ], - env: managerEnv - } -} - -/** Daemon exit: its terminals are gone, so the slice goes too. A SIGKILL is the death watch's job. */ -export function releasePtyProcessCeiling(): void { - const runtime = currentLinux - currentLinux = null - if (!runtime) { - return - } - try { - runProcessSync({ - program: 'systemctl', - args: ['--user', '--no-block', 'stop', runtime.slice], - env: userManagerEnv(process.env), - stdio: 'ignore', - timeoutMs: 2_000 - }) - } catch { - // The slice is transient and holds nothing once every terminal has exited. - } -} diff --git a/src/main/daemon/pty-process-ceiling.test.ts b/src/main/daemon/pty-process-ceiling.test.ts deleted file mode 100644 index 7040b3d8cb7..00000000000 --- a/src/main/daemon/pty-process-ceiling.test.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' -import { - parsePtyProcessCeiling, - PTY_PROCESS_CEILING_ENV, - resolvePtyProcessCeiling, - type LinuxCeilingRuntime -} from './pty-process-ceiling' -import { - currentPtyProcessCeiling, - setPtyProcessCeilingForTest, - wrapPtySpawnForCeiling -} from './pty-process-ceiling-runtime' -import { - confirmPtyProcessCeiling, - describePtyProcessCeiling, - parsePtyProcessCeilingState -} from '../../shared/pty-process-ceiling-state' -import { parseDaemonPidFile } from './daemon-pid-file-parse' - -const on = { [PTY_PROCESS_CEILING_ENV]: '2048' } - -function linuxInput(overrides: Partial[0]> = {}) { - const run = vi.fn((program: string, _args: string[]) => ({ - code: 0, - timedOut: false, - stdout: - program === 'systemctl' - ? '/user.slice/user-1000.slice/user@1000.service/orca_ptys_1f_2e.slice\n' - : '' - })) - return { - run, - input: { - env: on, - platform: 'linux' as const, - launchId: '1f-2e', - scopeSupported: () => true, - findExecutable: (name: string) => `/usr/bin/${name}`, - readPidsMax: () => '2048\n', - run, - ...overrides - } - } -} - -describe('resolvePtyProcessCeiling', () => { - it('is off by default and refuses limits too small to run a shell', () => { - expect(resolvePtyProcessCeiling({ env: {}, platform: 'linux', launchId: 'a' }).state).toEqual({ - state: 'off', - reason: 'disabled' - }) - expect(parsePtyProcessCeiling('0')).toEqual({ kind: 'off' }) - expect( - resolvePtyProcessCeiling({ - env: { [PTY_PROCESS_CEILING_ENV]: '10' }, - platform: 'linux', - launchId: 'a' - }).state - ).toEqual({ state: 'off', reason: 'invalid_limit' }) - }) - - it('degrades with a reason without a user manager, systemd-run, or on macOS', () => { - const { run, input } = linuxInput({ scopeSupported: () => false }) - expect(resolvePtyProcessCeiling(input).state).toEqual({ - state: 'off', - reason: 'no_user_manager' - }) - expect(run).not.toHaveBeenCalled() - expect( - resolvePtyProcessCeiling(linuxInput({ findExecutable: () => null }).input).state - ).toEqual({ state: 'off', reason: 'systemd_run_missing' }) - expect(resolvePtyProcessCeiling({ env: on, platform: 'darwin', launchId: 'a' }).state).toEqual({ - state: 'off', - reason: 'unsupported_platform' - }) - }) - - it('is on only when the kernel shows the limit on a top-level slice', () => { - const { run, input } = linuxInput() - const resolved = resolvePtyProcessCeiling(input) - expect(resolved.state).toEqual({ - state: 'on', - limit: 2048, - mechanism: 'systemd-slice', - unit: 'orca_ptys_1f_2e.slice', - controlGroup: '/user.slice/user-1000.slice/user@1000.service/orca_ptys_1f_2e.slice' - }) - expect(resolved.linux?.pidsMaxPath).toBe( - '/sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/orca_ptys_1f_2e.slice/pids.max' - ) - expect(run.mock.calls[0]?.[0]).toBe('busctl') - expect(run.mock.calls[0]?.[1]).toEqual( - expect.arrayContaining(['StartTransientUnit', 'orca_ptys_1f_2e.slice', 'TasksMax', '2048']) - ) - }) - - it('reports off when TasksMax is accepted but not enforced, and removes the slice', () => { - const { run, input } = linuxInput({ readPidsMax: () => null }) - expect(resolvePtyProcessCeiling(input).state).toEqual({ - state: 'off', - reason: 'controller_unavailable' - }) - expect(run.mock.calls.at(-1)?.[1]).toEqual([ - '--user', - '--no-block', - 'stop', - 'orca_ptys_1f_2e.slice' - ]) - const failed = linuxInput({ - run: vi.fn(() => ({ code: 1, timedOut: false, stdout: '' })) - }) - expect(resolvePtyProcessCeiling(failed.input).state).toEqual({ - state: 'off', - reason: 'slice_unavailable' - }) - }) - - it('uses the per-terminal job limit on Windows only when the native build has it', () => { - const windows = (setWindowsLimit: () => boolean) => - resolvePtyProcessCeiling({ env: on, platform: 'win32', launchId: 'a', setWindowsLimit }).state - expect(windows(() => true)).toEqual({ - state: 'on', - limit: 2048, - mechanism: 'windows-job', - unit: null - }) - expect(windows(() => false)).toEqual({ state: 'off', reason: 'native_unavailable' }) - }) -}) - -describe('wrapPtySpawnForCeiling', () => { - const runtime: LinuxCeilingRuntime = { - slice: 'orca_ptys_a.slice', - pidsMaxPath: '/sys/fs/cgroup/x/pids.max', - systemdRun: '/usr/bin/systemd-run', - envBinary: '/usr/bin/env', - launchId: 'a' - } - - afterEach(() => setPtyProcessCeilingForTest({ state: 'off', reason: 'disabled' }, null)) - - it('leaves the spawn untouched while off', () => { - const env = { TERM: 'xterm' } - expect(wrapPtySpawnForCeiling('/bin/bash', ['-l'], env)).toEqual({ - file: '/bin/bash', - args: ['-l'], - env - }) - }) - - it('execs through systemd-run into the slice and gives the shell its own session bus back', () => { - setPtyProcessCeilingForTest( - { state: 'on', limit: 2048, mechanism: 'systemd-slice', unit: runtime.slice }, - runtime - ) - const wrapped = wrapPtySpawnForCeiling( - '/bin/bash', - ['-l'], - { TERM: 'xterm', DBUS_SESSION_BUS_ADDRESS: 'unix:path=/tmp/dbus-x' }, - () => true - ) - expect(wrapped.file).toBe('/usr/bin/systemd-run') - expect(wrapped.args).toEqual( - expect.arrayContaining([ - '--user', - '--scope', - '--slice=orca_ptys_a.slice', - '--unit=orca_pty_a_1.scope' - ]) - ) - const command = wrapped.args.slice(wrapped.args.indexOf('--') + 1) - expect(command[0]).toBe('/usr/bin/env') - expect(command).toContain('DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/dbus-x') - expect(command.slice(-2)).toEqual(['/bin/bash', '-l']) - expect(wrapped.env.DBUS_SESSION_BUS_ADDRESS).toBeUndefined() - }) - - it('stops wrapping and reports the slice lost when the user manager dropped it', () => { - setPtyProcessCeilingForTest( - { state: 'on', limit: 2048, mechanism: 'systemd-slice', unit: runtime.slice }, - runtime - ) - expect(wrapPtySpawnForCeiling('/bin/bash', [], { TERM: 'x' }, () => false).file).toBe( - '/bin/bash' - ) - expect(currentPtyProcessCeiling()).toEqual({ state: 'off', reason: 'slice_lost' }) - }) -}) - -describe('ceiling report', () => { - it('reports a slice the user manager dropped as lost, not on', () => { - const on = { - state: 'on' as const, - limit: 2048, - mechanism: 'systemd-slice' as const, - unit: 'orca_ptys_a.slice', - controlGroup: '/u/orca_ptys_a.slice' - } - expect(confirmPtyProcessCeiling(on, () => true)).toBe(on) - expect(confirmPtyProcessCeiling(on, () => false)).toEqual({ - state: 'off', - reason: 'slice_lost' - }) - expect(confirmPtyProcessCeiling(null)).toBeNull() - }) - - it('round-trips through the daemon pid record and reads old records as not reported', () => { - const record = parseDaemonPidFile( - JSON.stringify({ pid: 7, processCeiling: { state: 'off', reason: 'no_user_manager' } }) - ) - expect(record?.processCeiling).toEqual({ state: 'off', reason: 'no_user_manager' }) - expect(parseDaemonPidFile(JSON.stringify({ pid: 7 }))?.processCeiling).toBeNull() - expect(parsePtyProcessCeilingState({ state: 'off', reason: 'made-up' })).toBeNull() - expect(describePtyProcessCeiling(null)).toBe('not reported') - expect(describePtyProcessCeiling({ state: 'off', reason: 'no_user_manager' })).toBe( - 'off (no user manager)' - ) - }) -}) diff --git a/src/main/daemon/pty-process-ceiling.ts b/src/main/daemon/pty-process-ceiling.ts deleted file mode 100644 index 73d32953b52..00000000000 --- a/src/main/daemon/pty-process-ceiling.ts +++ /dev/null @@ -1,191 +0,0 @@ -/** - * An OS-enforced ceiling on how many processes the terminals of one daemon may run. Off by - * default: it turns on only when `ORCA_TERMINAL_PROCESS_CEILING` names a limit, until saturation - * tests show the daemon keeps serving with it on. - * - * Linux puts each PTY in its own transient scope under one slice that carries `TasksMax`, so the - * kernel refuses forks past the limit with EAGAIN. The slice is a sibling of the daemon's own - * scope: `pids.max` counts threads too, and the daemon must keep serving sockets, attach and - * cancel while a build inside a terminal hits the ceiling. - */ -import { accessSync, constants, readFileSync } from 'node:fs' -import { delimiter, isAbsolute, join } from 'node:path' -import { runProcessSync } from '@orca/process-host' -import { isDurableDaemonScopeSupported, userManagerEnv } from './daemon-cgroup-scope' -import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' - -export type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' - -export const PTY_PROCESS_CEILING_ENV = 'ORCA_TERMINAL_PROCESS_CEILING' -/** Below this a shell, its prompt hooks and one build step already collide. */ -const MIN_CEILING = 64 -const CGROUP_ROOT = '/sys/fs/cgroup' - -type Runner = ( - program: string, - args: string[], - env: NodeJS.ProcessEnv -) => { code: number | null; timedOut: boolean; stdout: string } - -const defaultRunner: Runner = (program, args, env) => - runProcessSync({ program, args, env, timeoutMs: 5_000 }) - -/** What the Linux wrap needs at spawn time; never reported, only used. */ -export type LinuxCeilingRuntime = { - slice: string - pidsMaxPath: string - systemdRun: string - envBinary: string | null - launchId: string -} - -export function parsePtyProcessCeiling( - raw: string | undefined -): { kind: 'off' } | { kind: 'limit'; limit: number } | { kind: 'invalid' } { - if (raw === undefined || raw.trim() === '' || raw.trim() === '0') { - return { kind: 'off' } - } - const limit = Number(raw.trim()) - return Number.isSafeInteger(limit) && limit >= MIN_CEILING - ? { kind: 'limit', limit } - : { kind: 'invalid' } -} - -const sanitizeId = (id: string): string => id.replace(/[^A-Za-z0-9]/g, '_') - -/** No dashes: systemd reads `a-b.slice` as a child of `a.slice`, and this one must be top-level. */ -export function ptyCeilingSliceName(launchId: string): string { - return `orca_ptys_${sanitizeId(launchId)}.slice` -} - -export function findExecutableOnPath(name: string, path: string | undefined): string | null { - for (const dir of (path ?? '').split(delimiter).filter(isAbsolute)) { - try { - accessSync(join(dir, name), constants.X_OK) - return join(dir, name) - } catch { - // Not here. - } - } - return null -} - -type ResolveInput = { - env: NodeJS.ProcessEnv - platform: NodeJS.Platform - launchId: string - scopeSupported?: () => boolean - run?: Runner - findExecutable?: (name: string) => string | null - readPidsMax?: (path: string) => string | null - setWindowsLimit?: (limit: number) => boolean -} - -function readFileOrNull(path: string): string | null { - try { - return readFileSync(path, 'utf8') - } catch { - return null - } -} - -/** - * Decides once, at daemon start, whether this host gets a ceiling. Every failure is a named - * "off", never a guess: "on" requires the kernel's own pids.max to show the limit. - */ -export function resolvePtyProcessCeiling(input: ResolveInput): { - state: PtyProcessCeilingState - linux: LinuxCeilingRuntime | null -} { - const off = (reason: Extract['reason']) => ({ - state: { state: 'off' as const, reason }, - linux: null - }) - const parsed = parsePtyProcessCeiling(input.env[PTY_PROCESS_CEILING_ENV]) - if (parsed.kind === 'off') { - return off('disabled') - } - if (parsed.kind === 'invalid') { - return off('invalid_limit') - } - if (input.platform === 'win32') { - return input.setWindowsLimit?.(parsed.limit) - ? { - state: { state: 'on', limit: parsed.limit, mechanism: 'windows-job', unit: null }, - linux: null - } - : off('native_unavailable') - } - if (input.platform !== 'linux') { - return off('unsupported_platform') - } - if (!(input.scopeSupported ?? isDurableDaemonScopeSupported)()) { - return off('no_user_manager') - } - const find = - input.findExecutable ?? ((name: string) => findExecutableOnPath(name, input.env.PATH)) - const systemdRun = find('systemd-run') - if (!systemdRun) { - return off('systemd_run_missing') - } - const run = input.run ?? defaultRunner - const managerEnv = userManagerEnv(input.env) - const slice = ptyCeilingSliceName(input.launchId) - try { - const created = run( - 'busctl', - [ - '--user', - 'call', - 'org.freedesktop.systemd1', - '/org/freedesktop/systemd1', - 'org.freedesktop.systemd1.Manager', - 'StartTransientUnit', - 'ssa(sv)a(sa(sv))', - slice, - 'fail', - '1', - 'TasksMax', - 't', - String(parsed.limit), - '0' - ], - managerEnv - ) - if (created.code !== 0 || created.timedOut) { - return off('slice_unavailable') - } - // Why verify: cgroup v1/hybrid hosts and managers without pids delegation accept TasksMax - // and enforce nothing. - const shown = run( - 'systemctl', - ['--user', 'show', slice, '--property=ControlGroup', '--value'], - managerEnv - ) - const controlGroup = shown.code === 0 ? shown.stdout.trim() : '' - const pidsMaxPath = controlGroup ? join(CGROUP_ROOT, controlGroup, 'pids.max') : '' - const pidsMax = pidsMaxPath ? (input.readPidsMax ?? readFileOrNull)(pidsMaxPath) : null - if (pidsMax?.trim() !== String(parsed.limit)) { - run('systemctl', ['--user', '--no-block', 'stop', slice], managerEnv) - return off('controller_unavailable') - } - return { - state: { - state: 'on', - limit: parsed.limit, - mechanism: 'systemd-slice', - unit: slice, - controlGroup - }, - linux: { - slice, - pidsMaxPath, - systemdRun, - envBinary: find('env'), - launchId: sanitizeId(input.launchId) - } - } - } catch { - return off('slice_unavailable') - } -} diff --git a/src/main/daemon/pty-subprocess/foreground-fallback-process.test.ts b/src/main/daemon/pty-subprocess/foreground-fallback-process.test.ts new file mode 100644 index 00000000000..e3b72660b60 --- /dev/null +++ b/src/main/daemon/pty-subprocess/foreground-fallback-process.test.ts @@ -0,0 +1,33 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { resolveFallbackForegroundProcess } from './foreground-fallback-process' +import { setTerminalScopesForTest } from '../terminal-scopes-runtime' + +describe('resolveFallbackForegroundProcess', () => { + afterEach(() => setTerminalScopesForTest(null)) + + // Why not Windows: an empty name there falls back to the shell, and terminal scopes are Linux-only. + it.skipIf(process.platform === 'win32')( + 'reads the terminal scope wrappers as the shell, but a user-typed systemd-run as work', + () => { + setTerminalScopesForTest({ + slice: 'orca_ptys_a.slice', + sliceCgroupPath: '/sys/fs/cgroup/x/orca_ptys_a.slice', + systemdRun: '/usr/bin/systemd-run', + envBinary: '/usr/bin/env', + launchId: 'a', + properties: [] + }) + // node-pty reports the foreground leader's argv[0]: the wrapper while it execs the shell. + expect(resolveFallbackForegroundProcess('/usr/bin/systemd-run', '/bin/bash')).toBe('bash') + expect(resolveFallbackForegroundProcess('/usr/bin/env', '/bin/bash')).toBe('bash') + // Its spawn file once that leader exited (`true | claude`, `seq | less`), as main reports. + expect(resolveFallbackForegroundProcess('/usr/bin/systemd-run', '/usr/bin/zsh')).toBe('zsh') + expect(resolveFallbackForegroundProcess('systemd-run', '/bin/bash')).toBe('systemd-run') + expect(resolveFallbackForegroundProcess('/usr/bin/bash', '/bin/bash')).toBe('bash') + } + ) + + it('keeps every name while terminal scopes are off', () => { + expect(resolveFallbackForegroundProcess('/usr/bin/systemd-run', '/bin/zsh')).toBe('systemd-run') + }) +}) diff --git a/src/main/daemon/pty-subprocess/foreground-fallback-process.ts b/src/main/daemon/pty-subprocess/foreground-fallback-process.ts index b463ec9a136..475e6099c4a 100644 --- a/src/main/daemon/pty-subprocess/foreground-fallback-process.ts +++ b/src/main/daemon/pty-subprocess/foreground-fallback-process.ts @@ -1,4 +1,5 @@ import { win32 as pathWin32 } from 'node:path' +import { isTerminalScopeWrapper } from '../terminal-scopes-runtime' function normalizeForegroundProcessName(processName: string | null | undefined): string | null { const trimmed = processName?.trim().replace(/^["']|["']$/g, '') ?? '' @@ -16,7 +17,11 @@ export function resolveFallbackForegroundProcess( processName: string | null | undefined, shellPath: string ): string | null { - const normalized = normalizeForegroundProcessName(processName) + // Why: node-pty names Orca's exact wrapper path while it execs the shell, and as its spawn file + // once the foreground group's leader has exited (`true | claude`); both mean the shell. + const normalized = normalizeForegroundProcessName( + processName && isTerminalScopeWrapper(processName.trim()) ? shellPath : processName + ) if (normalized || process.platform !== 'win32') { return normalized } diff --git a/src/main/daemon/pty-subprocess/native-pty-spawn.ts b/src/main/daemon/pty-subprocess/native-pty-spawn.ts index 5b858d58baa..012b644e4ce 100644 --- a/src/main/daemon/pty-subprocess/native-pty-spawn.ts +++ b/src/main/daemon/pty-subprocess/native-pty-spawn.ts @@ -5,7 +5,7 @@ import { } from '../../providers/macos-tcc-login-shell' import type { WindowsShellSpawnAttempt } from '../../providers/windows-shell-fallback-chain' import { assignHostProcessToKillOnCloseJob } from '../../windows/windows-pty-job' -import { wrapPtySpawnForCeiling } from '../pty-process-ceiling-runtime' +import { wrapPtySpawnInTerminalScope } from '../terminal-scopes-runtime' async function loadNodePty(): Promise { return import('node-pty') @@ -39,8 +39,8 @@ export async function spawnNativeDaemonPty(args: { ): Promise => { args.signal?.throwIfAborted() const tcc = wrapShellSpawnForMacosTccAttribution(shellPath, shellArgs, args.env) - // Linux with the opt-in ceiling: exec through systemd-run into the ceiling slice. - const wrapped = wrapPtySpawnForCeiling(tcc.file, tcc.args, args.env) + // Linux: exec through systemd-run into this terminal's own scope. + const wrapped = wrapPtySpawnInTerminalScope(tcc.file, tcc.args, args.env) const nodePty = await loadNodePty() // Why: children inherit job membership, so the host job must exist before the first Windows PTY. if (process.platform === 'win32') { diff --git a/src/main/daemon/terminal-scopes-runtime.ts b/src/main/daemon/terminal-scopes-runtime.ts new file mode 100644 index 00000000000..8d8578c71e0 --- /dev/null +++ b/src/main/daemon/terminal-scopes-runtime.ts @@ -0,0 +1,128 @@ +/** The daemon's terminal scopes and process ceiling, applied to every PTY spawn it makes. */ +import { existsSync } from 'node:fs' +import { spawnProcess } from '@orca/process-host' +import { userManagerEnv } from './daemon-cgroup-scope' +import { setPtyActiveProcessLimit } from '../windows/windows-pty-job' +import { resolveTerminalScopes, type TerminalScopeRuntime } from './terminal-scopes' +import type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' + +type DaemonLog = (event: string, details?: Record) => void + +let active: { scopes: TerminalScopeRuntime; log: DaemonLog } | null = null +let scopeCounter = 0 + +/** Daemon start: decide, remember for every spawn, and return what the pid record reports. */ +export function initializeTerminalScopes( + daemonScope: string | null, + log: DaemonLog +): PtyProcessCeilingState { + const resolved = resolveTerminalScopes({ + env: process.env, + platform: process.platform, + daemonScope, + setWindowsLimit: setPtyActiveProcessLimit + }) + active = 'on' in resolved.scopes ? { scopes: resolved.scopes.on, log } : null + if (active && daemonScope) { + // Why best-effort: oomd should pick a terminal before the daemon that holds every + // terminal's scrollback; older systemd ignores or rejects it, which changes nothing. + try { + const child = spawnProcess({ + program: 'systemctl', + args: ['--user', 'set-property', '--runtime', daemonScope, 'ManagedOOMPreference=avoid'], + env: userManagerEnv(process.env), + stdio: 'ignore' + }) + child.on('error', () => {}) + child.unref() + } catch { + // Only a spawn refused outright lands here; terminal scopes work without it. + } + } + if (process.platform === 'linux') { + log( + 'terminal-scopes', + 'on' in resolved.scopes + ? { slice: resolved.scopes.on.slice, properties: resolved.scopes.on.properties } + : resolved.scopes + ) + } + return resolved.ceiling +} + +export function setTerminalScopesForTest( + scopes: TerminalScopeRuntime | null, + log: DaemonLog = () => {} +): void { + active = scopes ? { scopes, log } : null +} + +/** Orca's own wrappers, by the exact argv[0] it execs them with; a user's `systemd-run` differs. */ +export function isTerminalScopeWrapper(processName: string): boolean { + return ( + active !== null && + (processName === active.scopes.systemdRun || processName === active.scopes.envBinary) + ) +} + +const SESSION_KEYS = ['DBUS_SESSION_BUS_ADDRESS', 'XDG_RUNTIME_DIR'] as const + +/** + * Linux with terminal scopes: exec through `systemd-run --scope`, which keeps the pid and exit + * status, into this terminal's own scope. `systemd-run` talks to the user manager the daemon + * verified; `env` then gives the shell its own session variables back. Off, it is the identity. + */ +export function wrapPtySpawnInTerminalScope( + file: string, + args: string[], + env: Record, + sliceExists: (path: string) => boolean = existsSync +): { file: string; args: string[]; env: Record } { + if (!active) { + return { file, args, env } + } + const { scopes, log } = active + // Why: `systemd-run --slice` would silently recreate a stopped slice unbound and unlimited. + if (!sliceExists(scopes.sliceCgroupPath)) { + active = null + log('terminal-scopes-slice-lost', { + message: 'terminal scope slice lost; later terminals stay in the daemon scope' + }) + return { file, args, env } + } + scopeCounter += 1 + const managerEnv: Record = {} + for (const [key, value] of Object.entries(userManagerEnv(env))) { + if (typeof value === 'string' && key !== 'DBUS_SESSION_BUS_ADDRESS') { + managerEnv[key] = value + } + } + const unsets: string[] = [] + const sets: string[] = [] + for (const key of SESSION_KEYS) { + if (env[key] !== managerEnv[key]) { + if (env[key] === undefined) { + unsets.push('-u', key) + } else { + sets.push(`${key}=${env[key]}`) + } + } + } + return { + file: scopes.systemdRun, + args: [ + '--user', + '--scope', + '--quiet', + '--collect', + `--slice=${scopes.slice}`, + `--unit=orca_pty_${scopes.launchId}_${scopeCounter}.scope`, + ...scopes.properties.map((property) => `--property=${property}`), + '--', + ...(unsets.length + sets.length > 0 ? [scopes.envBinary, ...unsets, ...sets] : []), + file, + ...args + ], + env: managerEnv + } +} diff --git a/src/main/daemon/terminal-scopes.test.ts b/src/main/daemon/terminal-scopes.test.ts new file mode 100644 index 00000000000..363a70ef0b5 --- /dev/null +++ b/src/main/daemon/terminal-scopes.test.ts @@ -0,0 +1,373 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + parsePtyProcessCeiling, + PTY_PROCESS_CEILING_ENV, + resolveTerminalScopes, + type TerminalScopeRuntime +} from './terminal-scopes' +import { + isTerminalScopeWrapper, + setTerminalScopesForTest, + wrapPtySpawnInTerminalScope +} from './terminal-scopes-runtime' +import { + confirmPtyProcessCeiling, + describePtyProcessCeiling, + parsePtyProcessCeilingState +} from '../../shared/pty-process-ceiling-state' +import { parseDaemonPidFile } from './daemon-pid-file-parse' +import type * as DaemonCgroupScope from './daemon-cgroup-scope' + +const managerRuntimeDir = vi.hoisted((): { value?: string } => ({})) +vi.mock('./daemon-cgroup-scope', async (importOriginal) => { + const original = await importOriginal() + return { + ...original, + // The user manager's runtime dir, independent of the host running the test. + userManagerEnv: (env: NodeJS.ProcessEnv) => { + const next = { ...env } + delete next.XDG_RUNTIME_DIR + return managerRuntimeDir.value ? { ...next, XDG_RUNTIME_DIR: managerRuntimeDir.value } : next + } + } +}) + +const SLICE_CGROUP = + '/user.slice/user-1000.slice/user@1000.service/orca_ptys_orca_daemon_1f_2e.slice' +const SCOPE_ON = { on: expect.objectContaining({ slice: 'orca_ptys_orca_daemon_1f_2e.slice' }) } + +type Call = { program: string; args: string[] } + +/** A user manager that may reject OOMPolicy on scopes and enforces `pidsMax` on each one. */ +function fakeManager({ oomPolicy = true, pidsMax = 'max\n', sliceStarts = true } = {}) { + const calls: Call[] = [] + const run = vi.fn((program: string, args: string[]) => { + calls.push({ program, args }) + if (program === 'busctl') { + return { code: sliceStarts ? 0 : 1, timedOut: false, stdout: '' } + } + if (program === '/usr/bin/systemd-run') { + const rejected = !oomPolicy && args.includes('--property=OOMPolicy=continue') + return rejected + ? { code: 1, timedOut: false, stdout: '' } + : { code: 0, timedOut: false, stdout: `0::${SLICE_CGROUP}/run-r1.scope\n${pidsMax}` } + } + return { code: 0, timedOut: false, stdout: '' } + }) + return { calls, run } +} + +function linuxInput( + manager: ReturnType, + overrides: Partial[0]> = {} +): Parameters[0] { + return { + env: {}, + platform: 'linux', + daemonScope: 'orca-daemon-1f-2e.scope', + unifiedCgroup: () => true, + findExecutable: (name: string) => `/usr/bin/${name}`, + run: manager.run, + ...overrides + } +} + +describe('resolveTerminalScopes', () => { + it('is on by default in the daemon’s own scope, with the slice bound to that scope', () => { + const manager = fakeManager() + const resolved = resolveTerminalScopes(linuxInput(manager)) + expect(resolved.ceiling).toEqual({ state: 'off', reason: 'disabled' }) + expect(resolved.scopes).toEqual({ + on: { + slice: 'orca_ptys_orca_daemon_1f_2e.slice', + sliceCgroupPath: `/sys/fs/cgroup${SLICE_CGROUP}`, + systemdRun: '/usr/bin/systemd-run', + envBinary: '/usr/bin/env', + launchId: 'orca_daemon_1f_2e', + properties: ['OOMPolicy=continue', 'TimeoutStopSec=5s'] + } + }) + expect(manager.calls[0]).toEqual({ + program: 'busctl', + args: [ + '--user', + 'call', + 'org.freedesktop.systemd1', + '/org/freedesktop/systemd1', + 'org.freedesktop.systemd1.Manager', + 'StartTransientUnit', + 'ssa(sv)a(sa(sv))', + 'orca_ptys_orca_daemon_1f_2e.slice', + 'fail', + '2', + 'BindsTo', + 'as', + '1', + 'orca-daemon-1f-2e.scope', + 'After', + 'as', + '1', + 'orca-daemon-1f-2e.scope', + '0' + ] + }) + expect(manager.calls[1]?.args.slice(0, 7)).toEqual([ + '--user', + '--scope', + '--quiet', + '--collect', + '--slice=orca_ptys_orca_daemon_1f_2e.slice', + '--property=OOMPolicy=continue', + '--property=TimeoutStopSec=5s' + ]) + expect(manager.calls).toHaveLength(2) + }) + + it('drops only OOMPolicy where scopes reject it', () => { + const manager = fakeManager({ oomPolicy: false }) + const resolved = resolveTerminalScopes(linuxInput(manager)) + expect(resolved.scopes).toMatchObject({ on: { properties: ['TimeoutStopSec=5s'] } }) + expect(manager.calls.filter((call) => call.program === '/usr/bin/systemd-run')).toHaveLength(2) + }) + + it.each([ + ['no owned daemon scope', { daemonScope: null }, 'no_user_manager'], + ['cgroup v1 or hybrid', { unifiedCgroup: () => false }, 'no_user_manager'], + [ + 'no env binary', + { findExecutable: (name: string) => (name === 'env' ? null : `/usr/bin/${name}`) }, + 'systemd_run_missing' + ], + ['macOS', { platform: 'darwin' as const }, 'unsupported_platform'] + ])('stays off with %s, without touching the user manager', (_label, overrides, reason) => { + const manager = fakeManager() + const resolved = resolveTerminalScopes(linuxInput(manager, overrides)) + expect(resolved).toEqual({ + ceiling: { state: 'off', reason: 'disabled' }, + scopes: { off: reason } + }) + expect(manager.run).not.toHaveBeenCalled() + }) + + it('stays off when the slice cannot start, or removes it when no probe scope starts', () => { + expect(resolveTerminalScopes(linuxInput(fakeManager({ sliceStarts: false }))).scopes).toEqual({ + off: 'slice_unavailable' + }) + const run = vi.fn((program: string, _args: string[]) => ({ + code: program === 'busctl' || program === 'systemctl' ? 0 : 1, + timedOut: false, + stdout: '' + })) + const resolved = resolveTerminalScopes(linuxInput({ ...fakeManager(), run })) + expect(resolved.scopes).toEqual({ off: 'slice_unavailable' }) + expect(run.mock.calls.at(-1)?.[1]).toEqual([ + '--user', + '--no-block', + 'stop', + 'orca_ptys_orca_daemon_1f_2e.slice' + ]) + }) + + it('stays off, marking a timeout for the log, when a wedged manager does not answer', () => { + const run = vi.fn((program: string, _args: string[]) => ({ + code: 1, + timedOut: program !== 'systemctl', + stdout: '' + })) + expect(resolveTerminalScopes(linuxInput({ ...fakeManager(), run })).scopes).toEqual({ + off: 'slice_unavailable', + timedOut: true + }) + run.mockImplementation((program: string) => ({ + code: program === 'busctl' ? 0 : 1, + timedOut: program !== 'busctl', + stdout: '' + })) + expect(resolveTerminalScopes(linuxInput({ ...fakeManager(), run })).scopes).toEqual({ + off: 'slice_unavailable', + timedOut: true + }) + }) + + it('puts the process ceiling on each terminal scope once the kernel shows it', () => { + const env = { [PTY_PROCESS_CEILING_ENV]: '2048' } + const resolved = resolveTerminalScopes(linuxInput(fakeManager({ pidsMax: '2048\n' }), { env })) + expect(resolved.ceiling).toEqual({ + state: 'on', + limit: 2048, + mechanism: 'systemd-slice', + unit: 'orca_ptys_orca_daemon_1f_2e.slice', + controlGroup: SLICE_CGROUP + }) + expect(resolved.scopes).toMatchObject({ + on: { properties: ['OOMPolicy=continue', 'TimeoutStopSec=5s', 'TasksMax=2048'] } + }) + }) + + it.each([ + ['accepted but not enforced', 'max\n'], + ['without pids delegation', ''] + ])( + 'keeps terminal scopes, reporting only the ceiling off, with TasksMax %s', + (_label, pidsMax) => { + const env = { [PTY_PROCESS_CEILING_ENV]: '2048' } + const resolved = resolveTerminalScopes(linuxInput(fakeManager({ pidsMax }), { env })) + expect(resolved.ceiling).toEqual({ state: 'off', reason: 'controller_unavailable' }) + expect(resolved.scopes).toMatchObject({ + on: { properties: ['OOMPolicy=continue', 'TimeoutStopSec=5s'] } + }) + } + ) + + it('reports a ceiling that needs a missing daemon scope as no user manager', () => { + const env = { [PTY_PROCESS_CEILING_ENV]: '2048' } + const missing = resolveTerminalScopes( + linuxInput(fakeManager(), { env, daemonScope: null }) + ).ceiling + expect(missing).toEqual({ state: 'off', reason: 'no_user_manager' }) + }) + + it('refuses ceilings too small to run a shell, without turning terminal scopes off', () => { + const env = { [PTY_PROCESS_CEILING_ENV]: '10' } + const resolved = resolveTerminalScopes(linuxInput(fakeManager(), { env })) + expect(resolved.ceiling).toEqual({ state: 'off', reason: 'invalid_limit' }) + expect(resolved.scopes).toEqual(SCOPE_ON) + expect(JSON.stringify(resolved.scopes)).not.toContain('TasksMax') + expect(parsePtyProcessCeiling('0')).toEqual({ kind: 'off' }) + }) + + it('uses the per-terminal job limit on Windows only when the native build has it', () => { + const windows = ( + setWindowsLimit: () => boolean, + env: NodeJS.ProcessEnv = { [PTY_PROCESS_CEILING_ENV]: '2048' } + ) => + resolveTerminalScopes({ + env, + platform: 'win32', + daemonScope: null, + setWindowsLimit + }).ceiling + expect(windows(() => true)).toEqual({ + state: 'on', + limit: 2048, + mechanism: 'windows-job', + unit: null + }) + expect(windows(() => false)).toEqual({ state: 'off', reason: 'native_unavailable' }) + expect(windows(() => true, {})).toEqual({ state: 'off', reason: 'disabled' }) + }) +}) + +describe('wrapPtySpawnInTerminalScope', () => { + const runtime: TerminalScopeRuntime = { + slice: 'orca_ptys_a.slice', + sliceCgroupPath: '/sys/fs/cgroup/x/orca_ptys_a.slice', + systemdRun: '/usr/bin/systemd-run', + envBinary: '/usr/bin/env', + launchId: 'a', + properties: ['OOMPolicy=continue', 'TimeoutStopSec=5s', 'TasksMax=2048'] + } + afterEach(() => setTerminalScopesForTest(null)) + + it('leaves the spawn untouched while off', () => { + const env = { TERM: 'xterm' } + expect(wrapPtySpawnInTerminalScope('/bin/bash', ['-l'], env)).toEqual({ + file: '/bin/bash', + args: ['-l'], + env + }) + }) + + it('execs each terminal into its own scope and gives the shell its own session bus back', () => { + setTerminalScopesForTest(runtime) + const spawn = () => + wrapPtySpawnInTerminalScope( + '/bin/bash', + ['-l'], + { TERM: 'xterm', DBUS_SESSION_BUS_ADDRESS: 'unix:path=/tmp/dbus-x' }, + () => true + ) + const wrapped = spawn() + expect(wrapped.file).toBe('/usr/bin/systemd-run') + const unit = wrapped.args.find((arg) => arg.startsWith('--unit=')) + expect(wrapped.args).toEqual( + expect.arrayContaining([ + '--user', + '--scope', + '--slice=orca_ptys_a.slice', + '--property=OOMPolicy=continue', + '--property=TasksMax=2048' + ]) + ) + expect(spawn().args).not.toContain(unit) + const command = wrapped.args.slice(wrapped.args.indexOf('--') + 1) + expect(command[0]).toBe('/usr/bin/env') + expect(command).toContain('DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/dbus-x') + expect(command.slice(-2)).toEqual(['/bin/bash', '-l']) + expect(wrapped.env.DBUS_SESSION_BUS_ADDRESS).toBeUndefined() + }) + + it('restores only the session variables that differ, unsetting ones the shell lacked', () => { + setTerminalScopesForTest(runtime) + managerRuntimeDir.value = '/run/user/1000' + const command = (env: Record) => { + const args = wrapPtySpawnInTerminalScope('/bin/bash', [], env, () => true).args + return args.slice(args.indexOf('--') + 1) + } + expect(command({ TERM: 'x' })).toEqual(['/usr/bin/env', '-u', 'XDG_RUNTIME_DIR', '/bin/bash']) + expect(command({ TERM: 'x', XDG_RUNTIME_DIR: '/run/user/1000' })).toEqual(['/bin/bash']) + managerRuntimeDir.value = undefined + }) + + it('stops wrapping for good once the slice is gone, and logs it once', () => { + const log = vi.fn() + setTerminalScopesForTest(runtime, log) + expect(wrapPtySpawnInTerminalScope('/bin/bash', [], { TERM: 'x' }, () => false).file).toBe( + '/bin/bash' + ) + expect(wrapPtySpawnInTerminalScope('/bin/bash', [], { TERM: 'x' }, () => true).file).toBe( + '/bin/bash' + ) + expect(log).toHaveBeenCalledOnce() + }) + + it('recognizes only the exact wrapper paths it execs, not a user-typed systemd-run', () => { + setTerminalScopesForTest(runtime) + expect(isTerminalScopeWrapper('/usr/bin/systemd-run')).toBe(true) + expect(isTerminalScopeWrapper('/usr/bin/env')).toBe(true) + expect(isTerminalScopeWrapper('systemd-run')).toBe(false) + setTerminalScopesForTest(null) + expect(isTerminalScopeWrapper('/usr/bin/systemd-run')).toBe(false) + }) +}) + +describe('ceiling report', () => { + it('reports a slice the user manager dropped as lost, not on', () => { + const on = { + state: 'on' as const, + limit: 2048, + mechanism: 'systemd-slice' as const, + unit: 'orca_ptys_a.slice', + controlGroup: '/u/orca_ptys_a.slice' + } + expect(confirmPtyProcessCeiling(on, () => true)).toBe(on) + expect(confirmPtyProcessCeiling(on, () => false)).toEqual({ + state: 'off', + reason: 'slice_lost' + }) + expect(confirmPtyProcessCeiling(null)).toBeNull() + }) + + it('round-trips through the daemon pid record and reads old records as not reported', () => { + const record = parseDaemonPidFile( + JSON.stringify({ pid: 7, processCeiling: { state: 'off', reason: 'no_user_manager' } }) + ) + expect(record?.processCeiling).toEqual({ state: 'off', reason: 'no_user_manager' }) + expect(parseDaemonPidFile(JSON.stringify({ pid: 7 }))?.processCeiling).toBeNull() + expect(parsePtyProcessCeilingState({ state: 'off', reason: 'made-up' })).toBeNull() + expect(describePtyProcessCeiling(null)).toBe('not reported') + expect(describePtyProcessCeiling({ state: 'off', reason: 'no_user_manager' })).toBe( + 'off (no user manager)' + ) + }) +}) diff --git a/src/main/daemon/terminal-scopes.ts b/src/main/daemon/terminal-scopes.ts new file mode 100644 index 00000000000..7768eca873b --- /dev/null +++ b/src/main/daemon/terminal-scopes.ts @@ -0,0 +1,242 @@ +/** + * On Linux each terminal runs in its own transient systemd scope, so a kernel OOM kill, a + * systemd-oomd kill or a fork limit lands on that terminal alone, as on macOS and Windows. In the + * daemon's own scope, systemd stops the whole scope on any OOM kill, oomd kills the whole leaf + * cgroup, and every terminal shares one TasksMax. + * + * The scopes sit under one slice beside the daemon's scope, never inside it: systemd counts OOM + * kills in a scope's descendants too, and `pids.max` counts threads, so the daemon must keep + * serving while a terminal is at its limit. The slice is bound to the daemon's scope, so systemd + * stops it with that scope for any reason, including ones that kill the death watch. + * + * `ORCA_TERMINAL_PROCESS_CEILING` sets TasksMax on each terminal's scope (Linux) or job + * (Windows); it stays off by default. + */ +import { accessSync, constants, existsSync } from 'node:fs' +import { basename, delimiter, dirname, isAbsolute, join } from 'node:path' +import { runProcessSync } from '@orca/process-host' +import { userManagerEnv } from './daemon-cgroup-scope' +import type { + PtyProcessCeilingOffReason, + PtyProcessCeilingState +} from '../../shared/pty-process-ceiling-state' + +export type { PtyProcessCeilingState } from '../../shared/pty-process-ceiling-state' + +export const PTY_PROCESS_CEILING_ENV = 'ORCA_TERMINAL_PROCESS_CEILING' +/** Below this a shell, its prompt hooks and one build step already collide. */ +const MIN_CEILING = 64 +const CGROUP_ROOT = '/sys/fs/cgroup' +/** Prints the probe scope's cgroup and, with pids delegated, the limit the kernel enforces on it. */ +const PROBE_SCRIPT = + 'read -r line < /proc/self/cgroup; echo "$line"; cat "/sys/fs/cgroup${line#0::}/pids.max" 2>/dev/null || :' + +type Runner = ( + program: string, + args: string[], + env: NodeJS.ProcessEnv +) => { code: number | null; timedOut: boolean; stdout: string } + +// Why 1.5 s: at most four calls run before the daemon is ready, inside the launcher's 10 s wait; +// a timed-out call turns terminal scopes off and the daemon serves with terminals in its scope. +const defaultRunner: Runner = (program, args, env) => + runProcessSync({ program, args, env, timeoutMs: 1_500 }) + +/** What the Linux wrap needs at spawn time; never reported, only used. */ +export type TerminalScopeRuntime = { + slice: string + sliceCgroupPath: string + systemdRun: string + envBinary: string + /** The daemon scope's name, sanitized; shared by the slice and every scope name. */ + launchId: string + /** `--property=` values every terminal scope gets; each one was accepted by the probe. */ + properties: string[] +} + +export type TerminalScopeResolution = { + ceiling: PtyProcessCeilingState + /** Off says why terminals share the daemon's cgroup, for the daemon log. */ + /** `timedOut` is log-only: a busy user manager, not one that refused. */ + scopes: { on: TerminalScopeRuntime } | { off: PtyProcessCeilingOffReason; timedOut?: true } +} + +export function parsePtyProcessCeiling( + raw: string | undefined +): { kind: 'off' } | { kind: 'limit'; limit: number } | { kind: 'invalid' } { + if (raw === undefined || raw.trim() === '' || raw.trim() === '0') { + return { kind: 'off' } + } + const limit = Number(raw.trim()) + return Number.isSafeInteger(limit) && limit >= MIN_CEILING + ? { kind: 'limit', limit } + : { kind: 'invalid' } +} + +const sanitizeId = (id: string): string => id.replace(/[^A-Za-z0-9]/g, '_') + +/** No dashes: systemd reads `a-b.slice` as a child of `a.slice`, and this one must be top-level. */ +export function terminalSliceName(launchId: string): string { + return `orca_ptys_${launchId}.slice` +} + +function findExecutableOnPath(name: string, path: string | undefined): string | null { + for (const dir of (path ?? '').split(delimiter).filter(isAbsolute)) { + try { + accessSync(join(dir, name), constants.X_OK) + return join(dir, name) + } catch { + // Not here. + } + } + return null +} + +type ResolveInput = { + env: NodeJS.ProcessEnv + platform: NodeJS.Platform + /** From `ownFreshDaemonScope`: the daemon's own scope, only when its launcher created it. */ + daemonScope: string | null + run?: Runner + findExecutable?: (name: string) => string | null + unifiedCgroup?: () => boolean + setWindowsLimit?: (limit: number) => boolean +} + +/** + * Decides once, at daemon start. Terminal scopes are on only after a probe scope started in the + * slice with every property terminals will get; the ceiling is on only when that probe saw the + * kernel enforce it. + */ +export function resolveTerminalScopes(input: ResolveInput): TerminalScopeResolution { + const parsed = parsePtyProcessCeiling(input.env[PTY_PROCESS_CEILING_ENV]) + const limit = parsed.kind === 'limit' ? parsed.limit : null + const ceilingOff = (reason: PtyProcessCeilingOffReason): PtyProcessCeilingState => ({ + state: 'off', + reason: + parsed.kind === 'off' ? 'disabled' : parsed.kind === 'invalid' ? 'invalid_limit' : reason + }) + const off = (reason: PtyProcessCeilingOffReason, timedOut = false): TerminalScopeResolution => ({ + ceiling: ceilingOff(reason), + scopes: timedOut ? { off: reason, timedOut } : { off: reason } + }) + if (input.platform === 'win32') { + return { + ceiling: + limit !== null && input.setWindowsLimit?.(limit) + ? { state: 'on', limit, mechanism: 'windows-job', unit: null } + : ceilingOff('native_unavailable'), + scopes: { off: 'unsupported_platform' } + } + } + if (input.platform !== 'linux') { + return off('unsupported_platform') + } + // Why the daemon's own fresh scope: the slice binds to it, and a migrated legacy scope may + // also hold the GUI. Why unified cgroups: OOMPolicy, oomd and the probe's pids.max need v2. + const unifiedCgroup = + input.unifiedCgroup ?? (() => existsSync(join(CGROUP_ROOT, 'cgroup.controllers'))) + if (!input.daemonScope || !unifiedCgroup()) { + return off('no_user_manager') + } + const find = + input.findExecutable ?? ((name: string) => findExecutableOnPath(name, input.env.PATH)) + const systemdRun = find('systemd-run') + // Why env is required: without it the shell would inherit the manager's bus address, not its own. + const envBinary = find('env') + if (!systemdRun || !envBinary) { + // One reason for either wrapper keeps the reported enum closed for older readers. + return off('systemd_run_missing') + } + const run = input.run ?? defaultRunner + const managerEnv = userManagerEnv(input.env) + const launchId = sanitizeId(basename(input.daemonScope, '.scope')) + const slice = terminalSliceName(launchId) + try { + const created = run( + 'busctl', + [ + '--user', + 'call', + 'org.freedesktop.systemd1', + '/org/freedesktop/systemd1', + 'org.freedesktop.systemd1.Manager', + 'StartTransientUnit', + 'ssa(sv)a(sa(sv))', + slice, + 'fail', + '2', + 'BindsTo', + 'as', + '1', + input.daemonScope, + 'After', + 'as', + '1', + input.daemonScope, + '0' + ], + managerEnv + ) + if (created.code !== 0 || created.timedOut) { + return off('slice_unavailable', created.timedOut) + } + let probeTimedOut = false + // Why probe, not parse a version: some systemd releases reject OOMPolicy on scopes, and an + // unknown property fails the whole start, which would fail every terminal. + const base = ['TimeoutStopSec=5s', ...(limit !== null ? [`TasksMax=${limit}`] : [])] + for (const properties of [['OOMPolicy=continue', ...base], base]) { + const probe = run( + systemdRun, + [ + '--user', + '--scope', + '--quiet', + '--collect', + `--slice=${slice}`, + ...properties.map((property) => `--property=${property}`), + '--', + '/bin/sh', + '-c', + PROBE_SCRIPT + ], + managerEnv + ) + const [cgroupLine, pidsMax] = probe.stdout.trim().split('\n') + if (probe.code !== 0 || probe.timedOut || !cgroupLine?.startsWith('0::/')) { + probeTimedOut ||= probe.timedOut + continue + } + const sliceControlGroup = dirname(cgroupLine.slice('0::'.length)) + // Why verify: managers without pids delegation accept TasksMax and enforce nothing. + const enforced = limit !== null && pidsMax?.trim() === String(limit) + return { + ceiling: enforced + ? { + state: 'on', + limit, + mechanism: 'systemd-slice', + unit: slice, + controlGroup: sliceControlGroup + } + : ceilingOff('controller_unavailable'), + scopes: { + on: { + slice, + sliceCgroupPath: join(CGROUP_ROOT, sliceControlGroup), + systemdRun, + envBinary, + launchId, + properties: enforced + ? properties + : properties.filter((property) => !property.startsWith('TasksMax=')) + } + } + } + } + run('systemctl', ['--user', '--no-block', 'stop', slice], managerEnv) + return off('slice_unavailable', probeTimedOut) + } catch { + return off('slice_unavailable') + } +} diff --git a/src/shared/local-build-compatibility-contract.json b/src/shared/local-build-compatibility-contract.json index 1b097ab4f27..3a18795c4da 100644 --- a/src/shared/local-build-compatibility-contract.json +++ b/src/shared/local-build-compatibility-contract.json @@ -3,9 +3,9 @@ "appId": "com.stablyai.orca", "stateSchemaVersion": 1, "readableStateSchemaVersions": [1], - "daemonProtocolVersion": 45, + "daemonProtocolVersion": 46, "attachableDaemonProtocolVersions": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45 + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46 ] } diff --git a/src/shared/local-build-compatibility-contract.ts b/src/shared/local-build-compatibility-contract.ts index 7d71fa8030b..6bea8a44498 100644 --- a/src/shared/local-build-compatibility-contract.ts +++ b/src/shared/local-build-compatibility-contract.ts @@ -3,9 +3,9 @@ export const LOCAL_BUILD_COMPATIBILITY_CONTRACT = { appId: 'com.stablyai.orca', stateSchemaVersion: 1, readableStateSchemaVersions: [1], - daemonProtocolVersion: 45, + daemonProtocolVersion: 46, attachableDaemonProtocolVersions: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45 + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46 ] } as const diff --git a/src/shared/pty-process-ceiling-state.ts b/src/shared/pty-process-ceiling-state.ts index a96bfb9902c..b55832acfdf 100644 --- a/src/shared/pty-process-ceiling-state.ts +++ b/src/shared/pty-process-ceiling-state.ts @@ -71,7 +71,7 @@ export function describePtyProcessCeiling(state: PtyProcessCeilingState | null): return 'not reported' } return state.state === 'on' - ? `on (${state.limit} processes, ${state.mechanism === 'systemd-slice' ? `all terminals, systemd ${state.unit ?? 'slice'}` : 'requested per terminal job'})` + ? `on (${state.limit} processes, ${state.mechanism === 'systemd-slice' ? `per terminal, systemd ${state.unit ?? 'slice'}` : 'requested per terminal job'})` : `off (${state.reason.replace(/_/g, ' ')})` }