test(daemon): add provenance-gated daemon PTY-churn RSS benchmark (STA-3515)

Replaces the untracked, unsound RSS probe. Fixes all five review findings:
ephemeral pid-verified inspector port, socket close + timeouts with cleanup
in finally, loud build-provenance gate (patch hash chain + fresh native
binaries + in-run daemon bundle rebuild), cross-platform paths/pipes/shell,
and a natural-GC primary series with forced-GC only as a labelled separate
series plus a post-run forced-GC discriminator sample.
This commit is contained in:
Brennan Benson
2026-08-27 19:27:46 -07:00
parent 4f14d6d757
commit fbc09932ed
4 changed files with 834 additions and 0 deletions
@@ -0,0 +1,112 @@
/**
* Ephemeral-port, pid-verifiable inspector session for daemon benchmarks.
*
* The target daemon is spawned with `--inspect=0`; `waitForInspectorUrl` reads
* the advertised ws:// URL from the child's OWN stderr, so the port is never
* guessed or shared and the target is the spawned process by construction
* (STA-3515 harness finding #1 — a fixed port can attach to an unrelated
* process). Callers must still assert `evaluateJson('process.pid')` equals the
* spawned child's pid before measuring.
*/
import { withTimeout } from './daemon-bench-ndjson-client.mjs'
export const INSPECTOR_TIMEOUT_MS = 10_000
export class InspectorSession {
constructor(wsUrl) {
this.wsUrl = wsUrl
this.ws = null
this.pending = new Map()
this.messageCounter = 0
}
async connect() {
this.ws = new WebSocket(this.wsUrl)
await withTimeout(
new Promise((resolvePromise, reject) => {
this.ws.addEventListener('open', () => resolvePromise(), { once: true })
this.ws.addEventListener('error', () => reject(new Error('inspector websocket error')), {
once: true
})
}),
INSPECTOR_TIMEOUT_MS,
'inspector websocket connect'
)
this.ws.addEventListener('message', (event) => {
let message
try {
message = JSON.parse(typeof event.data === 'string' ? event.data : '')
} catch {
return
}
const pending = message.id !== undefined ? this.pending.get(message.id) : undefined
if (pending) {
this.pending.delete(message.id)
clearTimeout(pending.timer)
if (message.error) {
pending.reject(new Error(`inspector ${pending.method}: ${message.error.message}`))
} else {
pending.resolve(message.result)
}
}
})
this.ws.addEventListener('close', () => {
for (const [id, pending] of this.pending) {
clearTimeout(pending.timer)
pending.reject(new Error('inspector websocket closed'))
this.pending.delete(id)
}
})
}
send(method, params) {
return new Promise((resolvePromise, reject) => {
const id = ++this.messageCounter
const timer = setTimeout(() => {
this.pending.delete(id)
reject(new Error(`inspector ${method} timed out after ${INSPECTOR_TIMEOUT_MS}ms`))
}, INSPECTOR_TIMEOUT_MS)
this.pending.set(id, { resolve: resolvePromise, reject, timer, method })
this.ws.send(JSON.stringify({ id, method, ...(params ? { params } : {}) }))
})
}
async evaluateJson(expression) {
const result = await this.send('Runtime.evaluate', {
expression: `JSON.stringify(${expression})`,
returnByValue: true
})
if (result.exceptionDetails) {
throw new Error(`inspector evaluate threw: ${result.exceptionDetails.text}`)
}
return JSON.parse(result.result.value)
}
close() {
try {
this.ws?.close()
} catch {
// already closed
}
}
}
export function waitForInspectorUrl(child) {
return withTimeout(
new Promise((resolvePromise, reject) => {
let buffer = ''
const onData = (chunk) => {
buffer += chunk.toString('utf8')
const match = /Debugger listening on (ws:\/\/[^\s]+)/.exec(buffer)
if (match) {
child.stderr.off('data', onData)
resolvePromise(match[1])
}
}
child.stderr.on('data', onData)
child.once('exit', () => reject(new Error('daemon exited before advertising inspector')))
}),
INSPECTOR_TIMEOUT_MS,
'inspector url discovery'
)
}
@@ -0,0 +1,180 @@
/**
* Minimal NDJSON client for benchmarks that talk to a self-spawned Orca
* terminal daemon: control + stream hello handshake, id-matched RPC with
* per-request timeouts, and close/error handling that rejects every pending
* request so a daemon crash can never strand an await (STA-3515 harness
* finding #2).
*/
import { randomUUID } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { connect } from 'node:net'
export const HELLO_TIMEOUT_MS = 10_000
export const RPC_TIMEOUT_MS = 30_000
export function withTimeout(promise, ms, what) {
let timer
return Promise.race([
promise,
new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms}ms`)), ms)
})
]).finally(() => clearTimeout(timer))
}
function connectSocket(path, timeoutMs) {
return new Promise((resolvePromise, reject) => {
const socket = connect(path)
const timer = setTimeout(() => {
socket.destroy()
reject(new Error(`daemon socket connect timed out after ${timeoutMs}ms`))
}, timeoutMs)
socket.once('connect', () => {
clearTimeout(timer)
resolvePromise(socket)
})
socket.once('error', (error) => {
clearTimeout(timer)
reject(error)
})
})
}
export class BenchDaemonClient {
constructor(socketPath, tokenPath, protocolVersion, onSessionOutputChars) {
this.socketPath = socketPath
this.tokenPath = tokenPath
this.protocolVersion = protocolVersion
this.onSessionOutputChars = onSessionOutputChars
this.clientId = randomUUID()
this.controlSocket = null
this.streamSocket = null
this.pending = new Map()
this.requestCounter = 0
}
async connect() {
this.controlSocket = await connectSocket(this.socketPath, HELLO_TIMEOUT_MS)
await this.hello(this.controlSocket, 'control')
this.attachNdjsonReader(this.controlSocket, (message) => this.settleResponse(message))
// A stream socket mirrors the real client shape: PTY output leaves the
// daemon instead of accumulating solely because nothing is attached. Its
// data events also PROVE each cycle's output actually flowed.
this.streamSocket = await connectSocket(this.socketPath, HELLO_TIMEOUT_MS)
await this.hello(this.streamSocket, 'stream')
this.attachNdjsonReader(this.streamSocket, (message) => {
if (message.type === 'event' && message.event === 'data') {
this.onSessionOutputChars?.(message.sessionId, message.payload?.data?.length ?? 0)
}
})
const rejectAll = (reason) => () => {
for (const [id, pending] of this.pending) {
clearTimeout(pending.timer)
pending.reject(new Error(reason))
this.pending.delete(id)
}
}
this.controlSocket.on('close', rejectAll('daemon control socket closed'))
this.controlSocket.on('error', rejectAll('daemon control socket errored'))
}
hello(socket, role) {
const token = readFileSync(this.tokenPath, 'utf8').trim()
return withTimeout(
new Promise((resolvePromise, reject) => {
let buffer = ''
const onData = (chunk) => {
buffer += chunk.toString('utf8')
const newline = buffer.indexOf('\n')
if (newline === -1) {
return
}
socket.off('data', onData)
socket.off('close', onClose)
try {
const response = JSON.parse(buffer.slice(0, newline))
if (response.ok) {
resolvePromise()
} else {
reject(new Error(`hello rejected: ${response.error ?? 'unknown'}`))
}
} catch (error) {
reject(error)
}
}
const onClose = () => reject(new Error('daemon closed socket before hello response'))
socket.on('data', onData)
socket.once('close', onClose)
socket.write(
`${JSON.stringify({ type: 'hello', version: this.protocolVersion, token, clientId: this.clientId, role })}\n`
)
}),
HELLO_TIMEOUT_MS,
`daemon ${role} hello`
)
}
attachNdjsonReader(socket, onMessage) {
let buffer = ''
socket.on('data', (chunk) => {
buffer += chunk.toString('utf8')
let newline = buffer.indexOf('\n')
while (newline !== -1) {
const line = buffer.slice(0, newline)
buffer = buffer.slice(newline + 1)
newline = buffer.indexOf('\n')
if (!line) {
continue
}
try {
onMessage(JSON.parse(line))
} catch {
// tolerate partial/foreign lines; the daemon owns the framing
}
}
})
}
settleResponse(message) {
const pending = message.id ? this.pending.get(message.id) : undefined
if (pending) {
this.pending.delete(message.id)
clearTimeout(pending.timer)
if (message.ok) {
pending.resolve(message.payload)
} else {
pending.reject(new Error(message.error ?? 'daemon request failed'))
}
}
}
request(type, payload, timeoutMs = RPC_TIMEOUT_MS) {
return new Promise((resolvePromise, reject) => {
const id = `bench-${++this.requestCounter}`
const timer = setTimeout(() => {
this.pending.delete(id)
reject(new Error(`daemon request ${type} timed out after ${timeoutMs}ms`))
}, timeoutMs)
this.pending.set(id, { resolve: resolvePromise, reject, timer })
try {
this.controlSocket.write(
`${JSON.stringify({ id, type, ...(payload !== undefined ? { payload } : {}) })}\n`
)
} catch (error) {
this.pending.delete(id)
clearTimeout(timer)
reject(error)
}
})
}
destroy() {
for (const [id, pending] of this.pending) {
clearTimeout(pending.timer)
pending.reject(new Error('client destroyed'))
this.pending.delete(id)
}
this.controlSocket?.destroy()
this.streamSocket?.destroy()
}
}
@@ -0,0 +1,136 @@
/**
* Build-provenance gate for daemon benchmarks (STA-3515 harness finding #3).
*
* A measurement is only attributable to a commit when the artifacts it loads
* were provably built from that commit. `assertDaemonBuildProvenance` FAILS
* LOUDLY unless every link in the chain holds:
*
* 1. `git status` is clean for daemon-relevant paths, so the working tree IS
* `git rev-parse HEAD` for everything measured;
* 2. sha256(config/patches/node-pty@1.1.0.patch) equals the patch hash in
* pnpm-lock.yaml AND the patch hash pnpm baked into the installed
* node-pty realpath — pnpm names the install dir after the patch content
* hash, so this proves the installed native SOURCE is the checked-out
* patch;
* 3. the installed node-pty resolves inside THIS worktree (a sibling
* worktree's symlinked install proves nothing about this HEAD);
* 4. every compiled node-pty binary is at least as new as the newest native
* source file, so a stale binary cannot masquerade as patched;
* 5. out/orcad/daemon-entry.js is rebuilt by THIS call (build-orcad.mjs runs
* here), so the daemon bundle is the checked-out HEAD by construction.
*/
import { execFileSync, spawnSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
import { createRequire } from 'node:module'
import { join, resolve, sep } from 'node:path'
export function assertDaemonBuildProvenance(repoRoot) {
const git = (...cmdArgs) =>
execFileSync('git', cmdArgs, { cwd: repoRoot, encoding: 'utf8' }).trim()
const headSha = git('rev-parse', 'HEAD')
const branch = git('rev-parse', '--abbrev-ref', 'HEAD')
const relevant = ['src/main/', 'src/shared/', 'config/patches/', 'pnpm-lock.yaml', 'package.json']
// NOTE: no .trim() on the whole output — porcelain lines start with a
// two-character status whose first column may be a meaningful space.
const dirty = execFileSync('git', ['status', '--porcelain'], { cwd: repoRoot, encoding: 'utf8' })
.split('\n')
.filter((line) => line.length > 3)
.map((line) => line.slice(3))
.filter((file) => relevant.some((prefix) => file.startsWith(prefix)))
if (dirty.length > 0) {
throw new Error(
`PROVENANCE FAILURE: daemon-relevant paths are dirty; results would not be attributable to HEAD ${headSha}:\n ${dirty.join('\n ')}`
)
}
const patchFile = join(repoRoot, 'config', 'patches', 'node-pty@1.1.0.patch')
const patchSha256 = createHash('sha256').update(readFileSync(patchFile)).digest('hex')
const lockfile = readFileSync(join(repoRoot, 'pnpm-lock.yaml'), 'utf8')
const lockHashMatch = /node-pty@1\.1\.0:\s*\n\s*hash: ([0-9a-f]{64})/.exec(lockfile)
if (!lockHashMatch || lockHashMatch[1] !== patchSha256) {
throw new Error(
`PROVENANCE FAILURE: pnpm-lock node-pty patch hash (${lockHashMatch?.[1] ?? 'missing'}) != sha256 of checked-out patch (${patchSha256})`
)
}
const requireFromRoot = createRequire(join(repoRoot, 'package.json'))
const installedPkgJson = requireFromRoot.resolve('node-pty/package.json')
const installedRealRoot = realpathSync(resolve(installedPkgJson, '..'))
if (!installedRealRoot.includes(`patch_hash=${patchSha256}`)) {
throw new Error(
`PROVENANCE FAILURE: installed node-pty at ${installedRealRoot} was not built from the checked-out patch (expected patch_hash=${patchSha256}). Run pnpm install.`
)
}
if (!installedRealRoot.startsWith(repoRoot + sep)) {
throw new Error(
`PROVENANCE FAILURE: node_modules/node-pty resolves outside this worktree (${installedRealRoot}); a sibling worktree's install cannot prove this HEAD`
)
}
const releaseDir = join(installedRealRoot, 'build', 'Release')
const nativeBinaries = existsSync(releaseDir)
? readdirSync(releaseDir).filter(
(name) => name.endsWith('.node') || name === 'spawn-helper' || name.endsWith('.exe')
)
: []
if (nativeBinaries.length === 0) {
throw new Error(`PROVENANCE FAILURE: no compiled node-pty binaries under ${releaseDir}`)
}
const srcDir = join(installedRealRoot, 'src')
let newestSourceMtime = 0
for (const entry of readdirSync(srcDir, { withFileTypes: true, recursive: true })) {
if (entry.isFile()) {
newestSourceMtime = Math.max(
newestSourceMtime,
statSync(join(entry.parentPath, entry.name)).mtimeMs
)
}
}
const staleBinaries = nativeBinaries.filter(
(name) => statSync(join(releaseDir, name)).mtimeMs < newestSourceMtime
)
if (staleBinaries.length > 0) {
throw new Error(
`PROVENANCE FAILURE: node-pty binaries older than patched sources (stale: ${staleBinaries.join(', ')}). Rebuild node-pty.`
)
}
// Rebuild the daemon bundle from the (clean) working tree = HEAD by construction.
console.log('[provenance] rebuilding daemon bundle from HEAD…')
const daemonBundle = join(repoRoot, 'out', 'orcad', 'daemon-entry.js')
const buildStartedAt = Date.now()
const build = spawnSync(
process.execPath,
[join(repoRoot, 'config', 'scripts', 'build-orcad.mjs')],
{
cwd: repoRoot,
stdio: 'inherit'
}
)
if (build.status !== 0) {
throw new Error(`PROVENANCE FAILURE: daemon bundle build exited ${build.status}`)
}
if (!existsSync(daemonBundle) || statSync(daemonBundle).mtimeMs < buildStartedAt) {
throw new Error('PROVENANCE FAILURE: daemon bundle missing or not refreshed by this build')
}
const provenance = {
headSha,
// Parents make a local merge HEAD attributable to its upstream inputs.
headParents: git('log', '-1', '--format=%P').split(' '),
branch,
patchSha256,
installedNodePtyRealpath: installedRealRoot,
nativeBinaries,
daemonBundleBuiltAt: new Date(buildStartedAt).toISOString(),
nodeVersion: process.version,
platform: process.platform,
arch: process.arch
}
console.log(
`[provenance] OK: HEAD ${headSha} (${branch}), node-pty patch ${patchSha256.slice(0, 12)}…`
)
return provenance
}
@@ -0,0 +1,406 @@
#!/usr/bin/env node
/**
* Orca terminal-daemon RSS measurement under PTY churn (STA-3515).
*
* Measures whether daemon memory grows without bound across repeated PTY
* create/output/kill cycles, and separates:
* - the NATURAL series: RSS + V8 heap sampled with NO forced GC, so transient
* JS retention stays visible;
* - the FORCED-GC series: a separate daemon run where GC is forced via the
* inspector immediately before every sample, showing the reclaimable floor.
* The natural run additionally takes one clearly-labelled post-run forced-GC
* sample, which is the leak-vs-retention discriminator: memory that survives a
* final full GC after all sessions are dead is either allocator retention or a
* real leak; memory that drops was transient JS retention.
*
* Every run starts with the build-provenance gate in
* daemon-build-provenance-gate.mjs, which FAILS LOUDLY unless the daemon
* bundle and the patched node-pty binaries are provably built from the
* checked-out HEAD.
*
* The daemon is spawned with `--inspect=0` (ephemeral port). The inspector URL
* is taken from the child's own stderr — never from a fixed port — and
* ownership is asserted by evaluating `process.pid` over the socket and
* comparing it to the spawned child's pid before any measurement.
*
* Usage:
* node tests/tools/benchmarks/daemon-pty-churn-rss-bench.mjs --label baseline
* [--cycles 400] [--sample-every 20] [--output-bytes 65536]
* [--series both|natural|forced-gc] [--settle-ms 30] [--probe-fds]
*
* Results: tests/tools/benchmarks/results/daemon-pty-churn-rss-<label>-<stamp>.json
*/
import { execFileSync, fork } from 'node:child_process'
import { randomUUID } from 'node:crypto'
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import os from 'node:os'
import { join, resolve } from 'node:path'
import { setTimeout as delay } from 'node:timers/promises'
import { assertDaemonBuildProvenance } from './daemon-build-provenance-gate.mjs'
import { BenchDaemonClient, RPC_TIMEOUT_MS, withTimeout } from './daemon-bench-ndjson-client.mjs'
import { InspectorSession, waitForInspectorUrl } from './daemon-bench-inspector-session.mjs'
const scriptDir = import.meta.dirname
const repoRoot = resolve(scriptDir, '..', '..', '..')
const DAEMON_BUNDLE = join(repoRoot, 'out', 'orcad', 'daemon-entry.js')
const READY_TIMEOUT_MS = 20_000
function parseArgs(argv) {
const args = {
label: 'run',
cycles: 400,
sampleEvery: 20,
outputBytes: 65_536,
series: 'both',
settleMs: 30,
probeFds: false
}
for (let i = 2; i < argv.length; i++) {
const next = () => argv[++i]
const num = (name) => {
const value = Number(next())
if (!Number.isFinite(value) || value < 0) {
throw new Error(`${name} requires a non-negative number`)
}
return value
}
switch (argv[i]) {
case '--label':
args.label = next()
break
case '--cycles':
args.cycles = num('--cycles')
break
case '--sample-every':
args.sampleEvery = num('--sample-every')
break
case '--output-bytes':
args.outputBytes = num('--output-bytes')
break
case '--settle-ms':
args.settleMs = num('--settle-ms')
break
case '--series':
args.series = next()
if (!['both', 'natural', 'forced-gc'].includes(args.series)) {
throw new Error('--series must be both, natural, or forced-gc')
}
break
case '--probe-fds':
args.probeFds = true
break
default:
throw new Error(`Unknown argument: ${argv[i]}`)
}
}
return args
}
function readProtocolVersion() {
const source = readFileSync(
join(repoRoot, 'src', 'main', 'daemon', 'daemon-protocol-version.ts'),
'utf8'
)
const match = /export const PROTOCOL_VERSION = (\d+)/.exec(source)
if (!match) {
throw new Error('could not read PROTOCOL_VERSION from daemon-protocol-version.ts')
}
return Number(match[1])
}
function shellForPlatform() {
return process.platform === 'win32' ? 'cmd.exe' : '/bin/sh'
}
function outputCommand(outputBytes) {
// node one-liner so output generation is identical on every platform.
return `"${process.execPath}" -e "process.stdout.write('x'.repeat(${outputBytes}))"`
}
function sampleFdCounts(pid) {
if (process.platform !== 'darwin') {
return null
}
try {
const lines = execFileSync('lsof', ['-nP', '-p', String(pid)], { encoding: 'utf8' })
.trimEnd()
.split('\n')
.slice(1)
return {
fds: lines.length,
kqueues: lines.filter((line) => line.includes(' KQUEUE ')).length,
ptmx: lines.filter((line) => line.includes('/dev/ptmx')).length
}
} catch {
return null
}
}
function memorySample(seriesName, forcedGc, cycles, startedAt, memory) {
return {
series: seriesName,
forcedGc,
cycles,
elapsedMs: Date.now() - startedAt,
rssKb: Math.round(memory.rss / 1024),
heapUsedKb: Math.round(memory.heapUsed / 1024),
heapTotalKb: Math.round(memory.heapTotal / 1024),
externalKb: Math.round(memory.external / 1024),
arrayBuffersKb: Math.round(memory.arrayBuffers / 1024)
}
}
async function runSeries({ seriesName, args, protocolVersion, forceGcBeforeSample }) {
const runDir = mkdtempSync(join(os.tmpdir(), 'orca-rss-bench-'))
const nonce = randomUUID().slice(0, 8)
const socketPath =
process.platform === 'win32'
? `\\\\?\\pipe\\orca-rss-bench-${nonce}`
: join(runDir, 'daemon.sock')
const tokenPath = join(runDir, 'daemon.token')
const child = fork(DAEMON_BUNDLE, ['--socket', socketPath, '--token', tokenPath], {
cwd: repoRoot,
env: { ...process.env, ORCA_USER_DATA_PATH: runDir },
execArgv: ['--inspect=0'],
stdio: ['ignore', 'ignore', 'pipe', 'ipc']
})
const childPid = child.pid
let childExited = false
child.once('exit', () => {
childExited = true
})
let stderrTail = ''
const inspectorUrlPromise = waitForInspectorUrl(child)
child.stderr.on('data', (chunk) => {
stderrTail = (stderrTail + chunk.toString('utf8')).slice(-8192)
})
const outputCharsBySession = new Map()
const client = new BenchDaemonClient(socketPath, tokenPath, protocolVersion, (sessionId, chars) =>
outputCharsBySession.set(sessionId, (outputCharsBySession.get(sessionId) ?? 0) + chars)
)
const inspector = new InspectorSession(await inspectorUrlPromise)
const samples = []
const startedAt = Date.now()
try {
await withTimeout(
new Promise((resolvePromise, reject) => {
child.once('message', (message) => {
if (message?.type === 'ready') {
resolvePromise()
} else {
reject(new Error(`unexpected first daemon message: ${JSON.stringify(message)}`))
}
})
child.once('exit', (code) => reject(new Error(`daemon exited before ready: ${code}`)))
}),
READY_TIMEOUT_MS,
'daemon ready'
)
await inspector.connect()
// Ownership assertion: the inspector target MUST be the daemon we spawned.
const reportedPid = await inspector.evaluateJson('process.pid')
if (reportedPid !== childPid) {
throw new Error(
`PROVENANCE FAILURE: inspector target pid ${reportedPid} != spawned daemon pid ${childPid}; refusing to measure an unrelated process`
)
}
await client.connect()
const takeSample = async (cycles) => {
if (forceGcBeforeSample) {
await inspector.send('HeapProfiler.collectGarbage')
await delay(50)
}
const memory = await inspector.evaluateJson('process.memoryUsage()')
samples.push({
...memorySample(seriesName, forceGcBeforeSample, cycles, startedAt, memory),
...(args.probeFds ? { fdCounts: sampleFdCounts(childPid) } : {})
})
}
await takeSample(0)
const command = args.outputBytes > 0 ? outputCommand(args.outputBytes) : null
let fullOutputCycles = 0
let shortOutputCycles = 0
for (let cycle = 1; cycle <= args.cycles; cycle++) {
const sessionId = `rss-bench-${seriesName}-${cycle}`
await client.request('createOrAttach', {
sessionId,
cols: 80,
rows: 24,
shellOverride: shellForPlatform(),
shellReadySupported: false,
cancelAfterMs: RPC_TIMEOUT_MS - 100
})
if (command) {
await client.request('write', { sessionId, data: `${command}\r` })
// Churn must be proven, not assumed: wait until the daemon actually
// streamed the requested output back (echo + payload), bounded.
const outputDeadline = Date.now() + 5000
while (
(outputCharsBySession.get(sessionId) ?? 0) < args.outputBytes &&
Date.now() < outputDeadline
) {
await delay(10)
}
if ((outputCharsBySession.get(sessionId) ?? 0) >= args.outputBytes) {
fullOutputCycles++
} else {
shortOutputCycles++
}
if (args.settleMs > 0) {
await delay(args.settleMs)
}
}
await client.request('kill', { sessionId, immediate: true })
outputCharsBySession.delete(sessionId)
if (cycle % args.sampleEvery === 0) {
await takeSample(cycle)
}
}
if (command && shortOutputCycles > 0) {
console.warn(
`[bench] WARNING: ${shortOutputCycles}/${args.cycles} cycles did not reach ${args.outputBytes} output chars before the wait deadline`
)
}
const listed = await client.request('listSessions', undefined)
const liveSessions = listed?.sessions?.length ?? null
// Leak-vs-retention discriminator for the natural series: one labelled
// forced-GC sample AFTER the run, once every session is dead.
let postRunForcedGc = null
if (!forceGcBeforeSample) {
await delay(500)
await inspector.send('HeapProfiler.collectGarbage')
await delay(250)
const memory = await inspector.evaluateJson('process.memoryUsage()')
postRunForcedGc = memorySample(
`${seriesName}-post-run-forced-gc`,
true,
args.cycles,
startedAt,
memory
)
}
await client.request('shutdown', { killSessions: true }, 10_000).catch(() => {})
return {
samples,
postRunForcedGc,
liveSessions,
fullOutputCycles,
shortOutputCycles,
durationMs: Date.now() - startedAt
}
} catch (error) {
error.message = `${error.message}\ndaemon stderr tail:\n${stderrTail}`
throw error
} finally {
inspector.close()
client.destroy()
if (!childExited) {
// Only the exact child we spawned — never a pattern or name kill.
child.kill('SIGTERM')
await Promise.race([
new Promise((resolveExit) => child.once('exit', resolveExit)),
delay(5000)
])
if (!childExited) {
child.kill('SIGKILL')
}
}
rmSync(runDir, { recursive: true, force: true })
}
}
function summarizeSeries(samples) {
if (samples.length === 0) {
return null
}
const first = samples[0]
const last = samples.at(-1)
const peak = samples.reduce((max, sample) => Math.max(max, sample.rssKb), 0)
const half = samples[Math.floor(samples.length / 2)]
return {
firstRssKb: first.rssKb,
lastRssKb: last.rssKb,
peakRssKb: peak,
rssGrowthKb: last.rssKb - first.rssKb,
secondHalfRssGrowthKb: last.rssKb - half.rssKb,
firstHeapUsedKb: first.heapUsedKb,
lastHeapUsedKb: last.heapUsedKb,
cycles: last.cycles,
durationMs: last.elapsedMs
}
}
async function main() {
const args = parseArgs(process.argv)
const provenance = assertDaemonBuildProvenance(repoRoot)
const protocolVersion = readProtocolVersion()
const results = { args, provenance, series: {} }
if (args.series !== 'forced-gc') {
console.log(`[bench] natural series: ${args.cycles} cycles, no forced GC…`)
results.series.natural = await runSeries({
seriesName: 'natural',
args,
protocolVersion,
forceGcBeforeSample: false
})
console.log(
`[bench] natural done in ${Math.round(results.series.natural.durationMs / 1000)}s: ` +
`rss ${results.series.natural.samples[0].rssKb}KB → ${results.series.natural.samples.at(-1).rssKb}KB, ` +
`post-run forced-GC rss ${results.series.natural.postRunForcedGc?.rssKb}KB`
)
}
if (args.series !== 'natural') {
console.log(`[bench] forced-gc series: ${args.cycles} cycles, GC before every sample…`)
results.series.forcedGc = await runSeries({
seriesName: 'forced-gc',
args,
protocolVersion,
forceGcBeforeSample: true
})
console.log(
`[bench] forced-gc done in ${Math.round(results.series.forcedGc.durationMs / 1000)}s: ` +
`rss ${results.series.forcedGc.samples[0].rssKb}KB → ${results.series.forcedGc.samples.at(-1).rssKb}KB`
)
}
results.summary = {
natural: summarizeSeries(results.series.natural?.samples ?? []),
forcedGc: summarizeSeries(results.series.forcedGc?.samples ?? []),
naturalPostRunForcedGc: results.series.natural?.postRunForcedGc ?? null
}
const resultsDir = join(scriptDir, 'results')
mkdirSync(resultsDir, { recursive: true })
const stamp = new Date().toISOString().replace(/[:.]/g, '-')
const outPath = join(resultsDir, `daemon-pty-churn-rss-${args.label}-${stamp}.json`)
const serialized = JSON.stringify(results, null, 2)
// Results get committed as evidence — strip host-identifying home paths.
const homeEscaped = JSON.stringify(os.homedir()).slice(1, -1)
writeFileSync(outPath, serialized.split(homeEscaped).join('~'))
console.log(`\n[bench] results written to ${outPath}`)
for (const [name, summary] of Object.entries(results.summary)) {
if (summary && summary.firstRssKb !== undefined) {
console.log(
`| ${name} | rss ${summary.firstRssKb}→${summary.lastRssKb}KB (peak ${summary.peakRssKb}) | ` +
`heapUsed ${summary.firstHeapUsedKb}→${summary.lastHeapUsedKb}KB | ` +
`${summary.cycles} cycles / ${Math.round(summary.durationMs / 1000)}s |`
)
}
}
}
main().catch((error) => {
console.error(error)
process.exit(1)
})