Files
orca/config/docker/daemon-scope-lifetime/run-cases.cjs
T
Jinwoo Hong 78bb358c72 fix(linux): run each terminal in its own systemd scope so one out-of-memory kill no longer closes every terminal (#28871)
* refactor(daemon): rename the PTY process ceiling modules to terminal scopes

Pure move ahead of making per-terminal scopes the Linux default.

* fix(linux): run each terminal in its own systemd scope

The daemon and every terminal shared one systemd scope, so one kernel OOM
kill stopped the whole scope, systemd-oomd killed every terminal at once,
and all terminals shared one TasksMax. When the daemon runs in the fresh
scope its launcher made, each terminal now gets its own transient scope
under a slice bound to the daemon's scope, as the opt-in process ceiling
already did. OOMPolicy=continue is used where a probe scope accepts it,
the ceiling becomes TasksMax per terminal, and oomd is asked to avoid the
daemon's scope. The slice's binding replaces the death watch's slice
argument and the exit-time slice stop.

* test(linux): cover per-terminal scopes in the daemon scope Docker harness

Runs on systemd 249, 252 and 257: an OOM kill in one terminal leaves the
daemon and the other terminals running, the process ceiling applies per
terminal, and stopping the daemon scope or SIGKILLing the daemon reaps the
slice and every terminal. The baseline reports today's whole-daemon kill.

* chore: prune the sync-spawn baseline after removing the exit-time slice stop

* fix(linux): read the terminal scope wrappers as no foreground process yet

systemd-run and env own a new terminal until the shell execs, which took up
to ~270 ms per terminal when 50 started at once on Ubuntu 24.04. Reported as
the foreground, they read as running work and could pass the four-poll
child-process readiness gates before the shell existed.

* fix(linux): simplify terminal scope state and keep scopes without pids delegation

One nullable runtime replaces the unread ceiling state; losing the slice
logs once and later terminals stay in the daemon scope. The probe no longer
fails when pids.max is absent, so only the ceiling turns off there. The
resolution is one on/off value, the launch id comes from the daemon scope
alone, and the ManagedOOMPreference request no longer blocks startup. The
foreground filter matches Orca's exact wrapper paths, so a user-run
systemd-run stays visible.

* refactor(linux): name the terminal slice from the daemon scope and start the oomd hint at init

The slice and scope names now come from the daemon scope's own unit name, so
nothing parses its prefix back out. The ManagedOOMPreference request moves
out of the resolver into initializeTerminalScopes, where a spawn error can
no longer land in the resolver's catch after the slice exists.

* test(linux): run the terminal scope wrapper foreground test off Windows only

* test(linux): tolerate an already unloaded scope after the baseline OOM stop

* fix(linux): read Orca's scope wrapper as the shell, and bound startup systemd calls

node-pty names the wrapper path both while it execs the shell and as its
spawn file once the foreground group's leader has exited (true | claude,
seq | less). Hiding it skipped the agent scan; reporting the shell instead
matches what the unwrapped spawn reports. Each startup systemd call now
times out after 1.5 s, so a wedged user manager turns terminal scopes off
inside the launcher's 10 s wait instead of failing the launch.

* fix(linux): mark a timed-out terminal scope setup in the daemon log

A busy user manager and one that refused both reported slice_unavailable.
The log-only off detail now carries timedOut: true for the first; the
reported ceiling reason and the wire are unchanged.
2026-10-15 03:34:22 -04:00

203 lines
6.7 KiB
JavaScript

const assert = require('node:assert/strict')
const fs = require('node:fs')
const { join } = require('node:path')
const { spawn, execFileSync } = require('node:child_process')
const { runTerminalScopeCases } = require('./terminal-scope-cases.cjs')
const [bundlePath, mode] = process.argv.slice(2)
assert.ok(['baseline', 'candidate'].includes(mode))
const fixture = join(__dirname, 'process-fixture.cjs')
const prefix = `retention-${mode}-${process.pid}-${Date.now()}`
const scopes = []
const workers = []
const cgroups = []
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
function stat(pid) {
try {
const contents = fs.readFileSync(`/proc/${pid}/stat`, 'utf8')
const fields = contents.slice(contents.lastIndexOf(') ') + 2).split(' ')
return fields[0] === 'Z'
? null
: { ppid: Number(fields[1]), pgid: Number(fields[2]), startTicks: fields[19] }
} catch {
return null
}
}
function rss(pid) {
try {
const status = fs.readFileSync(`/proc/${pid}/status`, 'utf8')
return Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? 0)
} catch {
return 0
}
}
async function until(predicate, message, timeoutMs = 10000) {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
if (predicate()) {
return
}
await delay(25)
}
throw new Error(message)
}
function readPid(path) {
return fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : null
}
function scopePath(pid) {
return fs.readFileSync(`/proc/${pid}/cgroup`, 'utf8').trim()
}
async function startCycle(cycle, ownership = 'exclusive') {
const nonce = `${prefix}-${cycle}`
const resultPath = `/tmp/${nonce}`
const unit = `orca-daemon-${nonce}.scope`
scopes.push(unit)
const runtime = spawn(
process.execPath,
[fixture, 'runtime', resultPath, bundlePath, nonce, ownership],
{ stdio: 'inherit' }
)
await new Promise((resolve, reject) => {
runtime.on('error', reject)
runtime.on('exit', (code) =>
code === 0 ? resolve() : reject(new Error(`runtime exited ${code}`))
)
})
let workerPid, daemonPid
await until(() => {
workerPid = readPid(resultPath)
daemonPid = readPid(`${resultPath}.daemon`)
return workerPid && daemonPid && stat(workerPid)?.ppid === 1
}, 'worker did not reparent before daemon stop')
assert.ok(stat(daemonPid), 'daemon did not survive its runtime exit')
assert.equal(scopePath(workerPid), scopePath(daemonPid), 'worker escaped its owning scope')
assert.ok(scopePath(workerPid).endsWith(unit), 'daemon did not enter requested scope')
const cgroupDirectory = join('/sys/fs/cgroup', scopePath(workerPid).split(':').slice(2).join(':'))
assert.ok(
fs.existsSync(join(cgroupDirectory, 'memory.current')),
'cgroup memory accounting missing'
)
await delay(300)
assert.ok(stat(daemonPid) && stat(workerPid), 'runtime exit killed terminal work')
return { workerPid, daemonPid, resultPath, unit, cgroupDirectory }
}
async function main() {
const canaryUnit = `orca-canary-${prefix}.scope`
scopes.push(canaryUnit)
const canary = spawn(
'systemd-run',
[
'--user',
'--scope',
'--collect',
'--quiet',
`--unit=${canaryUnit}`,
'--',
'/bin/sleep',
'120'
],
{ stdio: 'inherit' }
)
try {
await until(
() => stat(canary.pid) && scopePath(canary.pid).endsWith(canaryUnit),
'canary scope missing'
)
for (const [index, signal] of [
'SIGTERM',
'SIGKILL',
'natural-exit',
'process-group-SIGKILL'
].entries()) {
const cycle = index + 1
const { workerPid, daemonPid, resultPath, cgroupDirectory } = await startCycle(cycle)
workers.push(workerPid)
cgroups.push(cgroupDirectory)
if (signal === 'natural-exit') {
fs.writeFileSync(`${resultPath}.exit`, '')
} else if (signal === 'process-group-SIGKILL') {
assert.equal(stat(daemonPid)?.pgid, daemonPid, 'daemon does not own its process group')
process.kill(-daemonPid, 'SIGKILL')
} else {
process.kill(daemonPid, signal)
}
await until(() => !stat(daemonPid), 'daemon did not exit')
await (mode === 'candidate'
? until(() => !stat(workerPid), 'scope retained orphan after daemon exit', 12000)
: delay(700))
if (mode === 'candidate') {
await until(
() => !fs.existsSync(cgroupDirectory),
'scope retained a process after worker exit'
)
}
const live = workers.filter((pid) => stat(pid))
const retainedRssKiB = live.reduce((sum, pid) => sum + rss(pid), 0)
const retainedCgroupMemoryKiB = cgroups.reduce((sum, directory) => {
const memoryFile = join(directory, 'memory.current')
return (
sum + (fs.existsSync(memoryFile) ? Number(fs.readFileSync(memoryFile, 'utf8')) / 1024 : 0)
)
}, 0)
assert.equal(live.length, mode === 'baseline' ? cycle : 0)
assert.ok(stat(canary.pid), 'unrelated scope was killed')
console.log(
JSON.stringify({
mode,
cycle,
signal,
liveWorkers: live.length,
retainedRssKiB,
retainedCgroupMemoryKiB,
unrelatedCanary: 'live',
runtimeExitPreservedDaemon: true,
workerReparentedBeforeStop: true
})
)
}
if (mode === 'candidate') {
const { workerPid, daemonPid } = await startCycle('shared', 'shared')
process.kill(daemonPid, 'SIGKILL')
await until(() => !stat(daemonPid), 'shared-scope daemon did not exit')
await delay(700)
assert.ok(stat(workerPid), 'scope without exclusive ownership was stopped')
assert.ok(stat(canary.pid), 'unrelated canary was killed')
console.log(
JSON.stringify({ mode, sharedScopeWithoutOwnership: 'preserved', unrelatedCanary: 'live' })
)
const brokenPipe = await startCycle('broken-pipe', 'broken-pipe')
await delay(5700)
assert.ok(stat(brokenPipe.daemonPid), 'premature pipe closure killed the daemon')
assert.ok(stat(brokenPipe.workerPid), 'premature pipe closure killed live terminal work')
assert.ok(stat(canary.pid), 'unrelated canary was killed')
console.log(
JSON.stringify({
mode,
prematurePipeClosure: 'live-work-preserved',
unrelatedCanary: 'live'
})
)
}
await runTerminalScopeCases(bundlePath, prefix, mode)
assert.ok(stat(canary.pid), 'unrelated canary was killed')
} finally {
for (const unit of [...scopes, `orca-daemon-${prefix}-*`, 'orca_ptys_*']) {
try {
execFileSync('systemctl', ['--user', '--signal=SIGKILL', 'kill', unit], { stdio: 'ignore' })
} catch {}
}
}
}
main().catch((error) => {
console.error(error)
process.exitCode = 1
})