mirror of
https://github.com/stablyai/orca.git
synced 2026-11-04 19:17:28 +00:00
* refactor(daemon): rename the PTY process ceiling modules to terminal scopes Pure move ahead of making per-terminal scopes the Linux default. * fix(linux): run each terminal in its own systemd scope The daemon and every terminal shared one systemd scope, so one kernel OOM kill stopped the whole scope, systemd-oomd killed every terminal at once, and all terminals shared one TasksMax. When the daemon runs in the fresh scope its launcher made, each terminal now gets its own transient scope under a slice bound to the daemon's scope, as the opt-in process ceiling already did. OOMPolicy=continue is used where a probe scope accepts it, the ceiling becomes TasksMax per terminal, and oomd is asked to avoid the daemon's scope. The slice's binding replaces the death watch's slice argument and the exit-time slice stop. * test(linux): cover per-terminal scopes in the daemon scope Docker harness Runs on systemd 249, 252 and 257: an OOM kill in one terminal leaves the daemon and the other terminals running, the process ceiling applies per terminal, and stopping the daemon scope or SIGKILLing the daemon reaps the slice and every terminal. The baseline reports today's whole-daemon kill. * chore: prune the sync-spawn baseline after removing the exit-time slice stop * fix(linux): read the terminal scope wrappers as no foreground process yet systemd-run and env own a new terminal until the shell execs, which took up to ~270 ms per terminal when 50 started at once on Ubuntu 24.04. Reported as the foreground, they read as running work and could pass the four-poll child-process readiness gates before the shell existed. * fix(linux): simplify terminal scope state and keep scopes without pids delegation One nullable runtime replaces the unread ceiling state; losing the slice logs once and later terminals stay in the daemon scope. The probe no longer fails when pids.max is absent, so only the ceiling turns off there. The resolution is one on/off value, the launch id comes from the daemon scope alone, and the ManagedOOMPreference request no longer blocks startup. The foreground filter matches Orca's exact wrapper paths, so a user-run systemd-run stays visible. * refactor(linux): name the terminal slice from the daemon scope and start the oomd hint at init The slice and scope names now come from the daemon scope's own unit name, so nothing parses its prefix back out. The ManagedOOMPreference request moves out of the resolver into initializeTerminalScopes, where a spawn error can no longer land in the resolver's catch after the slice exists. * test(linux): run the terminal scope wrapper foreground test off Windows only * test(linux): tolerate an already unloaded scope after the baseline OOM stop * fix(linux): read Orca's scope wrapper as the shell, and bound startup systemd calls node-pty names the wrapper path both while it execs the shell and as its spawn file once the foreground group's leader has exited (true | claude, seq | less). Hiding it skipped the agent scan; reporting the shell instead matches what the unwrapped spawn reports. Each startup systemd call now times out after 1.5 s, so a wedged user manager turns terminal scopes off inside the launcher's 10 s wait instead of failing the launch. * fix(linux): mark a timed-out terminal scope setup in the daemon log A busy user manager and one that refused both reported slice_unavailable. The log-only off detail now carries timedOut: true for the first; the reported ceiling reason and the wire are unchanged.
203 lines
6.7 KiB
JavaScript
203 lines
6.7 KiB
JavaScript
const assert = require('node:assert/strict')
|
|
const fs = require('node:fs')
|
|
const { join } = require('node:path')
|
|
const { spawn, execFileSync } = require('node:child_process')
|
|
const { runTerminalScopeCases } = require('./terminal-scope-cases.cjs')
|
|
|
|
const [bundlePath, mode] = process.argv.slice(2)
|
|
assert.ok(['baseline', 'candidate'].includes(mode))
|
|
const fixture = join(__dirname, 'process-fixture.cjs')
|
|
const prefix = `retention-${mode}-${process.pid}-${Date.now()}`
|
|
const scopes = []
|
|
const workers = []
|
|
const cgroups = []
|
|
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
|
|
function stat(pid) {
|
|
try {
|
|
const contents = fs.readFileSync(`/proc/${pid}/stat`, 'utf8')
|
|
const fields = contents.slice(contents.lastIndexOf(') ') + 2).split(' ')
|
|
return fields[0] === 'Z'
|
|
? null
|
|
: { ppid: Number(fields[1]), pgid: Number(fields[2]), startTicks: fields[19] }
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
function rss(pid) {
|
|
try {
|
|
const status = fs.readFileSync(`/proc/${pid}/status`, 'utf8')
|
|
return Number(status.match(/^VmRSS:\s+(\d+)/m)?.[1] ?? 0)
|
|
} catch {
|
|
return 0
|
|
}
|
|
}
|
|
|
|
async function until(predicate, message, timeoutMs = 10000) {
|
|
const deadline = Date.now() + timeoutMs
|
|
while (Date.now() < deadline) {
|
|
if (predicate()) {
|
|
return
|
|
}
|
|
await delay(25)
|
|
}
|
|
throw new Error(message)
|
|
}
|
|
|
|
function readPid(path) {
|
|
return fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : null
|
|
}
|
|
|
|
function scopePath(pid) {
|
|
return fs.readFileSync(`/proc/${pid}/cgroup`, 'utf8').trim()
|
|
}
|
|
|
|
async function startCycle(cycle, ownership = 'exclusive') {
|
|
const nonce = `${prefix}-${cycle}`
|
|
const resultPath = `/tmp/${nonce}`
|
|
const unit = `orca-daemon-${nonce}.scope`
|
|
scopes.push(unit)
|
|
const runtime = spawn(
|
|
process.execPath,
|
|
[fixture, 'runtime', resultPath, bundlePath, nonce, ownership],
|
|
{ stdio: 'inherit' }
|
|
)
|
|
await new Promise((resolve, reject) => {
|
|
runtime.on('error', reject)
|
|
runtime.on('exit', (code) =>
|
|
code === 0 ? resolve() : reject(new Error(`runtime exited ${code}`))
|
|
)
|
|
})
|
|
let workerPid, daemonPid
|
|
await until(() => {
|
|
workerPid = readPid(resultPath)
|
|
daemonPid = readPid(`${resultPath}.daemon`)
|
|
return workerPid && daemonPid && stat(workerPid)?.ppid === 1
|
|
}, 'worker did not reparent before daemon stop')
|
|
assert.ok(stat(daemonPid), 'daemon did not survive its runtime exit')
|
|
assert.equal(scopePath(workerPid), scopePath(daemonPid), 'worker escaped its owning scope')
|
|
assert.ok(scopePath(workerPid).endsWith(unit), 'daemon did not enter requested scope')
|
|
const cgroupDirectory = join('/sys/fs/cgroup', scopePath(workerPid).split(':').slice(2).join(':'))
|
|
assert.ok(
|
|
fs.existsSync(join(cgroupDirectory, 'memory.current')),
|
|
'cgroup memory accounting missing'
|
|
)
|
|
await delay(300)
|
|
assert.ok(stat(daemonPid) && stat(workerPid), 'runtime exit killed terminal work')
|
|
return { workerPid, daemonPid, resultPath, unit, cgroupDirectory }
|
|
}
|
|
|
|
async function main() {
|
|
const canaryUnit = `orca-canary-${prefix}.scope`
|
|
scopes.push(canaryUnit)
|
|
const canary = spawn(
|
|
'systemd-run',
|
|
[
|
|
'--user',
|
|
'--scope',
|
|
'--collect',
|
|
'--quiet',
|
|
`--unit=${canaryUnit}`,
|
|
'--',
|
|
'/bin/sleep',
|
|
'120'
|
|
],
|
|
{ stdio: 'inherit' }
|
|
)
|
|
try {
|
|
await until(
|
|
() => stat(canary.pid) && scopePath(canary.pid).endsWith(canaryUnit),
|
|
'canary scope missing'
|
|
)
|
|
for (const [index, signal] of [
|
|
'SIGTERM',
|
|
'SIGKILL',
|
|
'natural-exit',
|
|
'process-group-SIGKILL'
|
|
].entries()) {
|
|
const cycle = index + 1
|
|
const { workerPid, daemonPid, resultPath, cgroupDirectory } = await startCycle(cycle)
|
|
workers.push(workerPid)
|
|
cgroups.push(cgroupDirectory)
|
|
if (signal === 'natural-exit') {
|
|
fs.writeFileSync(`${resultPath}.exit`, '')
|
|
} else if (signal === 'process-group-SIGKILL') {
|
|
assert.equal(stat(daemonPid)?.pgid, daemonPid, 'daemon does not own its process group')
|
|
process.kill(-daemonPid, 'SIGKILL')
|
|
} else {
|
|
process.kill(daemonPid, signal)
|
|
}
|
|
await until(() => !stat(daemonPid), 'daemon did not exit')
|
|
await (mode === 'candidate'
|
|
? until(() => !stat(workerPid), 'scope retained orphan after daemon exit', 12000)
|
|
: delay(700))
|
|
if (mode === 'candidate') {
|
|
await until(
|
|
() => !fs.existsSync(cgroupDirectory),
|
|
'scope retained a process after worker exit'
|
|
)
|
|
}
|
|
const live = workers.filter((pid) => stat(pid))
|
|
const retainedRssKiB = live.reduce((sum, pid) => sum + rss(pid), 0)
|
|
const retainedCgroupMemoryKiB = cgroups.reduce((sum, directory) => {
|
|
const memoryFile = join(directory, 'memory.current')
|
|
return (
|
|
sum + (fs.existsSync(memoryFile) ? Number(fs.readFileSync(memoryFile, 'utf8')) / 1024 : 0)
|
|
)
|
|
}, 0)
|
|
assert.equal(live.length, mode === 'baseline' ? cycle : 0)
|
|
assert.ok(stat(canary.pid), 'unrelated scope was killed')
|
|
console.log(
|
|
JSON.stringify({
|
|
mode,
|
|
cycle,
|
|
signal,
|
|
liveWorkers: live.length,
|
|
retainedRssKiB,
|
|
retainedCgroupMemoryKiB,
|
|
unrelatedCanary: 'live',
|
|
runtimeExitPreservedDaemon: true,
|
|
workerReparentedBeforeStop: true
|
|
})
|
|
)
|
|
}
|
|
if (mode === 'candidate') {
|
|
const { workerPid, daemonPid } = await startCycle('shared', 'shared')
|
|
process.kill(daemonPid, 'SIGKILL')
|
|
await until(() => !stat(daemonPid), 'shared-scope daemon did not exit')
|
|
await delay(700)
|
|
assert.ok(stat(workerPid), 'scope without exclusive ownership was stopped')
|
|
assert.ok(stat(canary.pid), 'unrelated canary was killed')
|
|
console.log(
|
|
JSON.stringify({ mode, sharedScopeWithoutOwnership: 'preserved', unrelatedCanary: 'live' })
|
|
)
|
|
const brokenPipe = await startCycle('broken-pipe', 'broken-pipe')
|
|
await delay(5700)
|
|
assert.ok(stat(brokenPipe.daemonPid), 'premature pipe closure killed the daemon')
|
|
assert.ok(stat(brokenPipe.workerPid), 'premature pipe closure killed live terminal work')
|
|
assert.ok(stat(canary.pid), 'unrelated canary was killed')
|
|
console.log(
|
|
JSON.stringify({
|
|
mode,
|
|
prematurePipeClosure: 'live-work-preserved',
|
|
unrelatedCanary: 'live'
|
|
})
|
|
)
|
|
}
|
|
await runTerminalScopeCases(bundlePath, prefix, mode)
|
|
assert.ok(stat(canary.pid), 'unrelated canary was killed')
|
|
} finally {
|
|
for (const unit of [...scopes, `orca-daemon-${prefix}-*`, 'orca_ptys_*']) {
|
|
try {
|
|
execFileSync('systemctl', ['--user', '--signal=SIGKILL', 'kill', unit], { stdio: 'ignore' })
|
|
} catch {}
|
|
}
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error)
|
|
process.exitCode = 1
|
|
})
|