mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 16:03:49 +00:00
test(daemon): add provenance-gated daemon PTY-churn RSS benchmark (STA-3515)
Replaces the untracked, unsound RSS probe. Fixes all five review findings: ephemeral pid-verified inspector port, socket close + timeouts with cleanup in finally, loud build-provenance gate (patch hash chain + fresh native binaries + in-run daemon bundle rebuild), cross-platform paths/pipes/shell, and a natural-GC primary series with forced-GC only as a labelled separate series plus a post-run forced-GC discriminator sample.
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Ephemeral-port, pid-verifiable inspector session for daemon benchmarks.
|
||||
*
|
||||
* The target daemon is spawned with `--inspect=0`; `waitForInspectorUrl` reads
|
||||
* the advertised ws:// URL from the child's OWN stderr, so the port is never
|
||||
* guessed or shared and the target is the spawned process by construction
|
||||
* (STA-3515 harness finding #1 — a fixed port can attach to an unrelated
|
||||
* process). Callers must still assert `evaluateJson('process.pid')` equals the
|
||||
* spawned child's pid before measuring.
|
||||
*/
|
||||
import { withTimeout } from './daemon-bench-ndjson-client.mjs'
|
||||
|
||||
export const INSPECTOR_TIMEOUT_MS = 10_000
|
||||
|
||||
export class InspectorSession {
|
||||
constructor(wsUrl) {
|
||||
this.wsUrl = wsUrl
|
||||
this.ws = null
|
||||
this.pending = new Map()
|
||||
this.messageCounter = 0
|
||||
}
|
||||
|
||||
async connect() {
|
||||
this.ws = new WebSocket(this.wsUrl)
|
||||
await withTimeout(
|
||||
new Promise((resolvePromise, reject) => {
|
||||
this.ws.addEventListener('open', () => resolvePromise(), { once: true })
|
||||
this.ws.addEventListener('error', () => reject(new Error('inspector websocket error')), {
|
||||
once: true
|
||||
})
|
||||
}),
|
||||
INSPECTOR_TIMEOUT_MS,
|
||||
'inspector websocket connect'
|
||||
)
|
||||
this.ws.addEventListener('message', (event) => {
|
||||
let message
|
||||
try {
|
||||
message = JSON.parse(typeof event.data === 'string' ? event.data : '')
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const pending = message.id !== undefined ? this.pending.get(message.id) : undefined
|
||||
if (pending) {
|
||||
this.pending.delete(message.id)
|
||||
clearTimeout(pending.timer)
|
||||
if (message.error) {
|
||||
pending.reject(new Error(`inspector ${pending.method}: ${message.error.message}`))
|
||||
} else {
|
||||
pending.resolve(message.result)
|
||||
}
|
||||
}
|
||||
})
|
||||
this.ws.addEventListener('close', () => {
|
||||
for (const [id, pending] of this.pending) {
|
||||
clearTimeout(pending.timer)
|
||||
pending.reject(new Error('inspector websocket closed'))
|
||||
this.pending.delete(id)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
send(method, params) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const id = ++this.messageCounter
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(id)
|
||||
reject(new Error(`inspector ${method} timed out after ${INSPECTOR_TIMEOUT_MS}ms`))
|
||||
}, INSPECTOR_TIMEOUT_MS)
|
||||
this.pending.set(id, { resolve: resolvePromise, reject, timer, method })
|
||||
this.ws.send(JSON.stringify({ id, method, ...(params ? { params } : {}) }))
|
||||
})
|
||||
}
|
||||
|
||||
async evaluateJson(expression) {
|
||||
const result = await this.send('Runtime.evaluate', {
|
||||
expression: `JSON.stringify(${expression})`,
|
||||
returnByValue: true
|
||||
})
|
||||
if (result.exceptionDetails) {
|
||||
throw new Error(`inspector evaluate threw: ${result.exceptionDetails.text}`)
|
||||
}
|
||||
return JSON.parse(result.result.value)
|
||||
}
|
||||
|
||||
close() {
|
||||
try {
|
||||
this.ws?.close()
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function waitForInspectorUrl(child) {
|
||||
return withTimeout(
|
||||
new Promise((resolvePromise, reject) => {
|
||||
let buffer = ''
|
||||
const onData = (chunk) => {
|
||||
buffer += chunk.toString('utf8')
|
||||
const match = /Debugger listening on (ws:\/\/[^\s]+)/.exec(buffer)
|
||||
if (match) {
|
||||
child.stderr.off('data', onData)
|
||||
resolvePromise(match[1])
|
||||
}
|
||||
}
|
||||
child.stderr.on('data', onData)
|
||||
child.once('exit', () => reject(new Error('daemon exited before advertising inspector')))
|
||||
}),
|
||||
INSPECTOR_TIMEOUT_MS,
|
||||
'inspector url discovery'
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Minimal NDJSON client for benchmarks that talk to a self-spawned Orca
|
||||
* terminal daemon: control + stream hello handshake, id-matched RPC with
|
||||
* per-request timeouts, and close/error handling that rejects every pending
|
||||
* request so a daemon crash can never strand an await (STA-3515 harness
|
||||
* finding #2).
|
||||
*/
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { connect } from 'node:net'
|
||||
|
||||
export const HELLO_TIMEOUT_MS = 10_000
|
||||
export const RPC_TIMEOUT_MS = 30_000
|
||||
|
||||
export function withTimeout(promise, ms, what) {
|
||||
let timer
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms}ms`)), ms)
|
||||
})
|
||||
]).finally(() => clearTimeout(timer))
|
||||
}
|
||||
|
||||
function connectSocket(path, timeoutMs) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const socket = connect(path)
|
||||
const timer = setTimeout(() => {
|
||||
socket.destroy()
|
||||
reject(new Error(`daemon socket connect timed out after ${timeoutMs}ms`))
|
||||
}, timeoutMs)
|
||||
socket.once('connect', () => {
|
||||
clearTimeout(timer)
|
||||
resolvePromise(socket)
|
||||
})
|
||||
socket.once('error', (error) => {
|
||||
clearTimeout(timer)
|
||||
reject(error)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export class BenchDaemonClient {
|
||||
constructor(socketPath, tokenPath, protocolVersion, onSessionOutputChars) {
|
||||
this.socketPath = socketPath
|
||||
this.tokenPath = tokenPath
|
||||
this.protocolVersion = protocolVersion
|
||||
this.onSessionOutputChars = onSessionOutputChars
|
||||
this.clientId = randomUUID()
|
||||
this.controlSocket = null
|
||||
this.streamSocket = null
|
||||
this.pending = new Map()
|
||||
this.requestCounter = 0
|
||||
}
|
||||
|
||||
async connect() {
|
||||
this.controlSocket = await connectSocket(this.socketPath, HELLO_TIMEOUT_MS)
|
||||
await this.hello(this.controlSocket, 'control')
|
||||
this.attachNdjsonReader(this.controlSocket, (message) => this.settleResponse(message))
|
||||
// A stream socket mirrors the real client shape: PTY output leaves the
|
||||
// daemon instead of accumulating solely because nothing is attached. Its
|
||||
// data events also PROVE each cycle's output actually flowed.
|
||||
this.streamSocket = await connectSocket(this.socketPath, HELLO_TIMEOUT_MS)
|
||||
await this.hello(this.streamSocket, 'stream')
|
||||
this.attachNdjsonReader(this.streamSocket, (message) => {
|
||||
if (message.type === 'event' && message.event === 'data') {
|
||||
this.onSessionOutputChars?.(message.sessionId, message.payload?.data?.length ?? 0)
|
||||
}
|
||||
})
|
||||
const rejectAll = (reason) => () => {
|
||||
for (const [id, pending] of this.pending) {
|
||||
clearTimeout(pending.timer)
|
||||
pending.reject(new Error(reason))
|
||||
this.pending.delete(id)
|
||||
}
|
||||
}
|
||||
this.controlSocket.on('close', rejectAll('daemon control socket closed'))
|
||||
this.controlSocket.on('error', rejectAll('daemon control socket errored'))
|
||||
}
|
||||
|
||||
hello(socket, role) {
|
||||
const token = readFileSync(this.tokenPath, 'utf8').trim()
|
||||
return withTimeout(
|
||||
new Promise((resolvePromise, reject) => {
|
||||
let buffer = ''
|
||||
const onData = (chunk) => {
|
||||
buffer += chunk.toString('utf8')
|
||||
const newline = buffer.indexOf('\n')
|
||||
if (newline === -1) {
|
||||
return
|
||||
}
|
||||
socket.off('data', onData)
|
||||
socket.off('close', onClose)
|
||||
try {
|
||||
const response = JSON.parse(buffer.slice(0, newline))
|
||||
if (response.ok) {
|
||||
resolvePromise()
|
||||
} else {
|
||||
reject(new Error(`hello rejected: ${response.error ?? 'unknown'}`))
|
||||
}
|
||||
} catch (error) {
|
||||
reject(error)
|
||||
}
|
||||
}
|
||||
const onClose = () => reject(new Error('daemon closed socket before hello response'))
|
||||
socket.on('data', onData)
|
||||
socket.once('close', onClose)
|
||||
socket.write(
|
||||
`${JSON.stringify({ type: 'hello', version: this.protocolVersion, token, clientId: this.clientId, role })}\n`
|
||||
)
|
||||
}),
|
||||
HELLO_TIMEOUT_MS,
|
||||
`daemon ${role} hello`
|
||||
)
|
||||
}
|
||||
|
||||
attachNdjsonReader(socket, onMessage) {
|
||||
let buffer = ''
|
||||
socket.on('data', (chunk) => {
|
||||
buffer += chunk.toString('utf8')
|
||||
let newline = buffer.indexOf('\n')
|
||||
while (newline !== -1) {
|
||||
const line = buffer.slice(0, newline)
|
||||
buffer = buffer.slice(newline + 1)
|
||||
newline = buffer.indexOf('\n')
|
||||
if (!line) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
onMessage(JSON.parse(line))
|
||||
} catch {
|
||||
// tolerate partial/foreign lines; the daemon owns the framing
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
settleResponse(message) {
|
||||
const pending = message.id ? this.pending.get(message.id) : undefined
|
||||
if (pending) {
|
||||
this.pending.delete(message.id)
|
||||
clearTimeout(pending.timer)
|
||||
if (message.ok) {
|
||||
pending.resolve(message.payload)
|
||||
} else {
|
||||
pending.reject(new Error(message.error ?? 'daemon request failed'))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
request(type, payload, timeoutMs = RPC_TIMEOUT_MS) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const id = `bench-${++this.requestCounter}`
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(id)
|
||||
reject(new Error(`daemon request ${type} timed out after ${timeoutMs}ms`))
|
||||
}, timeoutMs)
|
||||
this.pending.set(id, { resolve: resolvePromise, reject, timer })
|
||||
try {
|
||||
this.controlSocket.write(
|
||||
`${JSON.stringify({ id, type, ...(payload !== undefined ? { payload } : {}) })}\n`
|
||||
)
|
||||
} catch (error) {
|
||||
this.pending.delete(id)
|
||||
clearTimeout(timer)
|
||||
reject(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
destroy() {
|
||||
for (const [id, pending] of this.pending) {
|
||||
clearTimeout(pending.timer)
|
||||
pending.reject(new Error('client destroyed'))
|
||||
this.pending.delete(id)
|
||||
}
|
||||
this.controlSocket?.destroy()
|
||||
this.streamSocket?.destroy()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* Build-provenance gate for daemon benchmarks (STA-3515 harness finding #3).
|
||||
*
|
||||
* A measurement is only attributable to a commit when the artifacts it loads
|
||||
* were provably built from that commit. `assertDaemonBuildProvenance` FAILS
|
||||
* LOUDLY unless every link in the chain holds:
|
||||
*
|
||||
* 1. `git status` is clean for daemon-relevant paths, so the working tree IS
|
||||
* `git rev-parse HEAD` for everything measured;
|
||||
* 2. sha256(config/patches/node-pty@1.1.0.patch) equals the patch hash in
|
||||
* pnpm-lock.yaml AND the patch hash pnpm baked into the installed
|
||||
* node-pty realpath — pnpm names the install dir after the patch content
|
||||
* hash, so this proves the installed native SOURCE is the checked-out
|
||||
* patch;
|
||||
* 3. the installed node-pty resolves inside THIS worktree (a sibling
|
||||
* worktree's symlinked install proves nothing about this HEAD);
|
||||
* 4. every compiled node-pty binary is at least as new as the newest native
|
||||
* source file, so a stale binary cannot masquerade as patched;
|
||||
* 5. out/orcad/daemon-entry.js is rebuilt by THIS call (build-orcad.mjs runs
|
||||
* here), so the daemon bundle is the checked-out HEAD by construction.
|
||||
*/
|
||||
import { execFileSync, spawnSync } from 'node:child_process'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { join, resolve, sep } from 'node:path'
|
||||
|
||||
export function assertDaemonBuildProvenance(repoRoot) {
|
||||
const git = (...cmdArgs) =>
|
||||
execFileSync('git', cmdArgs, { cwd: repoRoot, encoding: 'utf8' }).trim()
|
||||
const headSha = git('rev-parse', 'HEAD')
|
||||
const branch = git('rev-parse', '--abbrev-ref', 'HEAD')
|
||||
|
||||
const relevant = ['src/main/', 'src/shared/', 'config/patches/', 'pnpm-lock.yaml', 'package.json']
|
||||
// NOTE: no .trim() on the whole output — porcelain lines start with a
|
||||
// two-character status whose first column may be a meaningful space.
|
||||
const dirty = execFileSync('git', ['status', '--porcelain'], { cwd: repoRoot, encoding: 'utf8' })
|
||||
.split('\n')
|
||||
.filter((line) => line.length > 3)
|
||||
.map((line) => line.slice(3))
|
||||
.filter((file) => relevant.some((prefix) => file.startsWith(prefix)))
|
||||
if (dirty.length > 0) {
|
||||
throw new Error(
|
||||
`PROVENANCE FAILURE: daemon-relevant paths are dirty; results would not be attributable to HEAD ${headSha}:\n ${dirty.join('\n ')}`
|
||||
)
|
||||
}
|
||||
|
||||
const patchFile = join(repoRoot, 'config', 'patches', 'node-pty@1.1.0.patch')
|
||||
const patchSha256 = createHash('sha256').update(readFileSync(patchFile)).digest('hex')
|
||||
const lockfile = readFileSync(join(repoRoot, 'pnpm-lock.yaml'), 'utf8')
|
||||
const lockHashMatch = /node-pty@1\.1\.0:\s*\n\s*hash: ([0-9a-f]{64})/.exec(lockfile)
|
||||
if (!lockHashMatch || lockHashMatch[1] !== patchSha256) {
|
||||
throw new Error(
|
||||
`PROVENANCE FAILURE: pnpm-lock node-pty patch hash (${lockHashMatch?.[1] ?? 'missing'}) != sha256 of checked-out patch (${patchSha256})`
|
||||
)
|
||||
}
|
||||
|
||||
const requireFromRoot = createRequire(join(repoRoot, 'package.json'))
|
||||
const installedPkgJson = requireFromRoot.resolve('node-pty/package.json')
|
||||
const installedRealRoot = realpathSync(resolve(installedPkgJson, '..'))
|
||||
if (!installedRealRoot.includes(`patch_hash=${patchSha256}`)) {
|
||||
throw new Error(
|
||||
`PROVENANCE FAILURE: installed node-pty at ${installedRealRoot} was not built from the checked-out patch (expected patch_hash=${patchSha256}). Run pnpm install.`
|
||||
)
|
||||
}
|
||||
if (!installedRealRoot.startsWith(repoRoot + sep)) {
|
||||
throw new Error(
|
||||
`PROVENANCE FAILURE: node_modules/node-pty resolves outside this worktree (${installedRealRoot}); a sibling worktree's install cannot prove this HEAD`
|
||||
)
|
||||
}
|
||||
|
||||
const releaseDir = join(installedRealRoot, 'build', 'Release')
|
||||
const nativeBinaries = existsSync(releaseDir)
|
||||
? readdirSync(releaseDir).filter(
|
||||
(name) => name.endsWith('.node') || name === 'spawn-helper' || name.endsWith('.exe')
|
||||
)
|
||||
: []
|
||||
if (nativeBinaries.length === 0) {
|
||||
throw new Error(`PROVENANCE FAILURE: no compiled node-pty binaries under ${releaseDir}`)
|
||||
}
|
||||
const srcDir = join(installedRealRoot, 'src')
|
||||
let newestSourceMtime = 0
|
||||
for (const entry of readdirSync(srcDir, { withFileTypes: true, recursive: true })) {
|
||||
if (entry.isFile()) {
|
||||
newestSourceMtime = Math.max(
|
||||
newestSourceMtime,
|
||||
statSync(join(entry.parentPath, entry.name)).mtimeMs
|
||||
)
|
||||
}
|
||||
}
|
||||
const staleBinaries = nativeBinaries.filter(
|
||||
(name) => statSync(join(releaseDir, name)).mtimeMs < newestSourceMtime
|
||||
)
|
||||
if (staleBinaries.length > 0) {
|
||||
throw new Error(
|
||||
`PROVENANCE FAILURE: node-pty binaries older than patched sources (stale: ${staleBinaries.join(', ')}). Rebuild node-pty.`
|
||||
)
|
||||
}
|
||||
|
||||
// Rebuild the daemon bundle from the (clean) working tree = HEAD by construction.
|
||||
console.log('[provenance] rebuilding daemon bundle from HEAD…')
|
||||
const daemonBundle = join(repoRoot, 'out', 'orcad', 'daemon-entry.js')
|
||||
const buildStartedAt = Date.now()
|
||||
const build = spawnSync(
|
||||
process.execPath,
|
||||
[join(repoRoot, 'config', 'scripts', 'build-orcad.mjs')],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
stdio: 'inherit'
|
||||
}
|
||||
)
|
||||
if (build.status !== 0) {
|
||||
throw new Error(`PROVENANCE FAILURE: daemon bundle build exited ${build.status}`)
|
||||
}
|
||||
if (!existsSync(daemonBundle) || statSync(daemonBundle).mtimeMs < buildStartedAt) {
|
||||
throw new Error('PROVENANCE FAILURE: daemon bundle missing or not refreshed by this build')
|
||||
}
|
||||
|
||||
const provenance = {
|
||||
headSha,
|
||||
// Parents make a local merge HEAD attributable to its upstream inputs.
|
||||
headParents: git('log', '-1', '--format=%P').split(' '),
|
||||
branch,
|
||||
patchSha256,
|
||||
installedNodePtyRealpath: installedRealRoot,
|
||||
nativeBinaries,
|
||||
daemonBundleBuiltAt: new Date(buildStartedAt).toISOString(),
|
||||
nodeVersion: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch
|
||||
}
|
||||
console.log(
|
||||
`[provenance] OK: HEAD ${headSha} (${branch}), node-pty patch ${patchSha256.slice(0, 12)}…`
|
||||
)
|
||||
return provenance
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Orca terminal-daemon RSS measurement under PTY churn (STA-3515).
|
||||
*
|
||||
* Measures whether daemon memory grows without bound across repeated PTY
|
||||
* create/output/kill cycles, and separates:
|
||||
* - the NATURAL series: RSS + V8 heap sampled with NO forced GC, so transient
|
||||
* JS retention stays visible;
|
||||
* - the FORCED-GC series: a separate daemon run where GC is forced via the
|
||||
* inspector immediately before every sample, showing the reclaimable floor.
|
||||
* The natural run additionally takes one clearly-labelled post-run forced-GC
|
||||
* sample, which is the leak-vs-retention discriminator: memory that survives a
|
||||
* final full GC after all sessions are dead is either allocator retention or a
|
||||
* real leak; memory that drops was transient JS retention.
|
||||
*
|
||||
* Every run starts with the build-provenance gate in
|
||||
* daemon-build-provenance-gate.mjs, which FAILS LOUDLY unless the daemon
|
||||
* bundle and the patched node-pty binaries are provably built from the
|
||||
* checked-out HEAD.
|
||||
*
|
||||
* The daemon is spawned with `--inspect=0` (ephemeral port). The inspector URL
|
||||
* is taken from the child's own stderr — never from a fixed port — and
|
||||
* ownership is asserted by evaluating `process.pid` over the socket and
|
||||
* comparing it to the spawned child's pid before any measurement.
|
||||
*
|
||||
* Usage:
|
||||
* node tests/tools/benchmarks/daemon-pty-churn-rss-bench.mjs --label baseline
|
||||
* [--cycles 400] [--sample-every 20] [--output-bytes 65536]
|
||||
* [--series both|natural|forced-gc] [--settle-ms 30] [--probe-fds]
|
||||
*
|
||||
* Results: tests/tools/benchmarks/results/daemon-pty-churn-rss-<label>-<stamp>.json
|
||||
*/
|
||||
import { execFileSync, fork } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { assertDaemonBuildProvenance } from './daemon-build-provenance-gate.mjs'
|
||||
import { BenchDaemonClient, RPC_TIMEOUT_MS, withTimeout } from './daemon-bench-ndjson-client.mjs'
|
||||
import { InspectorSession, waitForInspectorUrl } from './daemon-bench-inspector-session.mjs'
|
||||
|
||||
const scriptDir = import.meta.dirname
|
||||
const repoRoot = resolve(scriptDir, '..', '..', '..')
|
||||
const DAEMON_BUNDLE = join(repoRoot, 'out', 'orcad', 'daemon-entry.js')
|
||||
const READY_TIMEOUT_MS = 20_000
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
label: 'run',
|
||||
cycles: 400,
|
||||
sampleEvery: 20,
|
||||
outputBytes: 65_536,
|
||||
series: 'both',
|
||||
settleMs: 30,
|
||||
probeFds: false
|
||||
}
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
const next = () => argv[++i]
|
||||
const num = (name) => {
|
||||
const value = Number(next())
|
||||
if (!Number.isFinite(value) || value < 0) {
|
||||
throw new Error(`${name} requires a non-negative number`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
switch (argv[i]) {
|
||||
case '--label':
|
||||
args.label = next()
|
||||
break
|
||||
case '--cycles':
|
||||
args.cycles = num('--cycles')
|
||||
break
|
||||
case '--sample-every':
|
||||
args.sampleEvery = num('--sample-every')
|
||||
break
|
||||
case '--output-bytes':
|
||||
args.outputBytes = num('--output-bytes')
|
||||
break
|
||||
case '--settle-ms':
|
||||
args.settleMs = num('--settle-ms')
|
||||
break
|
||||
case '--series':
|
||||
args.series = next()
|
||||
if (!['both', 'natural', 'forced-gc'].includes(args.series)) {
|
||||
throw new Error('--series must be both, natural, or forced-gc')
|
||||
}
|
||||
break
|
||||
case '--probe-fds':
|
||||
args.probeFds = true
|
||||
break
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${argv[i]}`)
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
function readProtocolVersion() {
|
||||
const source = readFileSync(
|
||||
join(repoRoot, 'src', 'main', 'daemon', 'daemon-protocol-version.ts'),
|
||||
'utf8'
|
||||
)
|
||||
const match = /export const PROTOCOL_VERSION = (\d+)/.exec(source)
|
||||
if (!match) {
|
||||
throw new Error('could not read PROTOCOL_VERSION from daemon-protocol-version.ts')
|
||||
}
|
||||
return Number(match[1])
|
||||
}
|
||||
|
||||
function shellForPlatform() {
|
||||
return process.platform === 'win32' ? 'cmd.exe' : '/bin/sh'
|
||||
}
|
||||
|
||||
function outputCommand(outputBytes) {
|
||||
// node one-liner so output generation is identical on every platform.
|
||||
return `"${process.execPath}" -e "process.stdout.write('x'.repeat(${outputBytes}))"`
|
||||
}
|
||||
|
||||
function sampleFdCounts(pid) {
|
||||
if (process.platform !== 'darwin') {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const lines = execFileSync('lsof', ['-nP', '-p', String(pid)], { encoding: 'utf8' })
|
||||
.trimEnd()
|
||||
.split('\n')
|
||||
.slice(1)
|
||||
return {
|
||||
fds: lines.length,
|
||||
kqueues: lines.filter((line) => line.includes(' KQUEUE ')).length,
|
||||
ptmx: lines.filter((line) => line.includes('/dev/ptmx')).length
|
||||
}
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function memorySample(seriesName, forcedGc, cycles, startedAt, memory) {
|
||||
return {
|
||||
series: seriesName,
|
||||
forcedGc,
|
||||
cycles,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
rssKb: Math.round(memory.rss / 1024),
|
||||
heapUsedKb: Math.round(memory.heapUsed / 1024),
|
||||
heapTotalKb: Math.round(memory.heapTotal / 1024),
|
||||
externalKb: Math.round(memory.external / 1024),
|
||||
arrayBuffersKb: Math.round(memory.arrayBuffers / 1024)
|
||||
}
|
||||
}
|
||||
|
||||
async function runSeries({ seriesName, args, protocolVersion, forceGcBeforeSample }) {
|
||||
const runDir = mkdtempSync(join(os.tmpdir(), 'orca-rss-bench-'))
|
||||
const nonce = randomUUID().slice(0, 8)
|
||||
const socketPath =
|
||||
process.platform === 'win32'
|
||||
? `\\\\?\\pipe\\orca-rss-bench-${nonce}`
|
||||
: join(runDir, 'daemon.sock')
|
||||
const tokenPath = join(runDir, 'daemon.token')
|
||||
|
||||
const child = fork(DAEMON_BUNDLE, ['--socket', socketPath, '--token', tokenPath], {
|
||||
cwd: repoRoot,
|
||||
env: { ...process.env, ORCA_USER_DATA_PATH: runDir },
|
||||
execArgv: ['--inspect=0'],
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc']
|
||||
})
|
||||
const childPid = child.pid
|
||||
let childExited = false
|
||||
child.once('exit', () => {
|
||||
childExited = true
|
||||
})
|
||||
let stderrTail = ''
|
||||
const inspectorUrlPromise = waitForInspectorUrl(child)
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderrTail = (stderrTail + chunk.toString('utf8')).slice(-8192)
|
||||
})
|
||||
|
||||
const outputCharsBySession = new Map()
|
||||
const client = new BenchDaemonClient(socketPath, tokenPath, protocolVersion, (sessionId, chars) =>
|
||||
outputCharsBySession.set(sessionId, (outputCharsBySession.get(sessionId) ?? 0) + chars)
|
||||
)
|
||||
const inspector = new InspectorSession(await inspectorUrlPromise)
|
||||
const samples = []
|
||||
const startedAt = Date.now()
|
||||
try {
|
||||
await withTimeout(
|
||||
new Promise((resolvePromise, reject) => {
|
||||
child.once('message', (message) => {
|
||||
if (message?.type === 'ready') {
|
||||
resolvePromise()
|
||||
} else {
|
||||
reject(new Error(`unexpected first daemon message: ${JSON.stringify(message)}`))
|
||||
}
|
||||
})
|
||||
child.once('exit', (code) => reject(new Error(`daemon exited before ready: ${code}`)))
|
||||
}),
|
||||
READY_TIMEOUT_MS,
|
||||
'daemon ready'
|
||||
)
|
||||
|
||||
await inspector.connect()
|
||||
// Ownership assertion: the inspector target MUST be the daemon we spawned.
|
||||
const reportedPid = await inspector.evaluateJson('process.pid')
|
||||
if (reportedPid !== childPid) {
|
||||
throw new Error(
|
||||
`PROVENANCE FAILURE: inspector target pid ${reportedPid} != spawned daemon pid ${childPid}; refusing to measure an unrelated process`
|
||||
)
|
||||
}
|
||||
|
||||
await client.connect()
|
||||
|
||||
const takeSample = async (cycles) => {
|
||||
if (forceGcBeforeSample) {
|
||||
await inspector.send('HeapProfiler.collectGarbage')
|
||||
await delay(50)
|
||||
}
|
||||
const memory = await inspector.evaluateJson('process.memoryUsage()')
|
||||
samples.push({
|
||||
...memorySample(seriesName, forceGcBeforeSample, cycles, startedAt, memory),
|
||||
...(args.probeFds ? { fdCounts: sampleFdCounts(childPid) } : {})
|
||||
})
|
||||
}
|
||||
|
||||
await takeSample(0)
|
||||
const command = args.outputBytes > 0 ? outputCommand(args.outputBytes) : null
|
||||
let fullOutputCycles = 0
|
||||
let shortOutputCycles = 0
|
||||
for (let cycle = 1; cycle <= args.cycles; cycle++) {
|
||||
const sessionId = `rss-bench-${seriesName}-${cycle}`
|
||||
await client.request('createOrAttach', {
|
||||
sessionId,
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
shellOverride: shellForPlatform(),
|
||||
shellReadySupported: false,
|
||||
cancelAfterMs: RPC_TIMEOUT_MS - 100
|
||||
})
|
||||
if (command) {
|
||||
await client.request('write', { sessionId, data: `${command}\r` })
|
||||
// Churn must be proven, not assumed: wait until the daemon actually
|
||||
// streamed the requested output back (echo + payload), bounded.
|
||||
const outputDeadline = Date.now() + 5000
|
||||
while (
|
||||
(outputCharsBySession.get(sessionId) ?? 0) < args.outputBytes &&
|
||||
Date.now() < outputDeadline
|
||||
) {
|
||||
await delay(10)
|
||||
}
|
||||
if ((outputCharsBySession.get(sessionId) ?? 0) >= args.outputBytes) {
|
||||
fullOutputCycles++
|
||||
} else {
|
||||
shortOutputCycles++
|
||||
}
|
||||
if (args.settleMs > 0) {
|
||||
await delay(args.settleMs)
|
||||
}
|
||||
}
|
||||
await client.request('kill', { sessionId, immediate: true })
|
||||
outputCharsBySession.delete(sessionId)
|
||||
if (cycle % args.sampleEvery === 0) {
|
||||
await takeSample(cycle)
|
||||
}
|
||||
}
|
||||
if (command && shortOutputCycles > 0) {
|
||||
console.warn(
|
||||
`[bench] WARNING: ${shortOutputCycles}/${args.cycles} cycles did not reach ${args.outputBytes} output chars before the wait deadline`
|
||||
)
|
||||
}
|
||||
|
||||
const listed = await client.request('listSessions', undefined)
|
||||
const liveSessions = listed?.sessions?.length ?? null
|
||||
|
||||
// Leak-vs-retention discriminator for the natural series: one labelled
|
||||
// forced-GC sample AFTER the run, once every session is dead.
|
||||
let postRunForcedGc = null
|
||||
if (!forceGcBeforeSample) {
|
||||
await delay(500)
|
||||
await inspector.send('HeapProfiler.collectGarbage')
|
||||
await delay(250)
|
||||
const memory = await inspector.evaluateJson('process.memoryUsage()')
|
||||
postRunForcedGc = memorySample(
|
||||
`${seriesName}-post-run-forced-gc`,
|
||||
true,
|
||||
args.cycles,
|
||||
startedAt,
|
||||
memory
|
||||
)
|
||||
}
|
||||
|
||||
await client.request('shutdown', { killSessions: true }, 10_000).catch(() => {})
|
||||
return {
|
||||
samples,
|
||||
postRunForcedGc,
|
||||
liveSessions,
|
||||
fullOutputCycles,
|
||||
shortOutputCycles,
|
||||
durationMs: Date.now() - startedAt
|
||||
}
|
||||
} catch (error) {
|
||||
error.message = `${error.message}\ndaemon stderr tail:\n${stderrTail}`
|
||||
throw error
|
||||
} finally {
|
||||
inspector.close()
|
||||
client.destroy()
|
||||
if (!childExited) {
|
||||
// Only the exact child we spawned — never a pattern or name kill.
|
||||
child.kill('SIGTERM')
|
||||
await Promise.race([
|
||||
new Promise((resolveExit) => child.once('exit', resolveExit)),
|
||||
delay(5000)
|
||||
])
|
||||
if (!childExited) {
|
||||
child.kill('SIGKILL')
|
||||
}
|
||||
}
|
||||
rmSync(runDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeSeries(samples) {
|
||||
if (samples.length === 0) {
|
||||
return null
|
||||
}
|
||||
const first = samples[0]
|
||||
const last = samples.at(-1)
|
||||
const peak = samples.reduce((max, sample) => Math.max(max, sample.rssKb), 0)
|
||||
const half = samples[Math.floor(samples.length / 2)]
|
||||
return {
|
||||
firstRssKb: first.rssKb,
|
||||
lastRssKb: last.rssKb,
|
||||
peakRssKb: peak,
|
||||
rssGrowthKb: last.rssKb - first.rssKb,
|
||||
secondHalfRssGrowthKb: last.rssKb - half.rssKb,
|
||||
firstHeapUsedKb: first.heapUsedKb,
|
||||
lastHeapUsedKb: last.heapUsedKb,
|
||||
cycles: last.cycles,
|
||||
durationMs: last.elapsedMs
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv)
|
||||
const provenance = assertDaemonBuildProvenance(repoRoot)
|
||||
const protocolVersion = readProtocolVersion()
|
||||
const results = { args, provenance, series: {} }
|
||||
|
||||
if (args.series !== 'forced-gc') {
|
||||
console.log(`[bench] natural series: ${args.cycles} cycles, no forced GC…`)
|
||||
results.series.natural = await runSeries({
|
||||
seriesName: 'natural',
|
||||
args,
|
||||
protocolVersion,
|
||||
forceGcBeforeSample: false
|
||||
})
|
||||
console.log(
|
||||
`[bench] natural done in ${Math.round(results.series.natural.durationMs / 1000)}s: ` +
|
||||
`rss ${results.series.natural.samples[0].rssKb}KB → ${results.series.natural.samples.at(-1).rssKb}KB, ` +
|
||||
`post-run forced-GC rss ${results.series.natural.postRunForcedGc?.rssKb}KB`
|
||||
)
|
||||
}
|
||||
if (args.series !== 'natural') {
|
||||
console.log(`[bench] forced-gc series: ${args.cycles} cycles, GC before every sample…`)
|
||||
results.series.forcedGc = await runSeries({
|
||||
seriesName: 'forced-gc',
|
||||
args,
|
||||
protocolVersion,
|
||||
forceGcBeforeSample: true
|
||||
})
|
||||
console.log(
|
||||
`[bench] forced-gc done in ${Math.round(results.series.forcedGc.durationMs / 1000)}s: ` +
|
||||
`rss ${results.series.forcedGc.samples[0].rssKb}KB → ${results.series.forcedGc.samples.at(-1).rssKb}KB`
|
||||
)
|
||||
}
|
||||
|
||||
results.summary = {
|
||||
natural: summarizeSeries(results.series.natural?.samples ?? []),
|
||||
forcedGc: summarizeSeries(results.series.forcedGc?.samples ?? []),
|
||||
naturalPostRunForcedGc: results.series.natural?.postRunForcedGc ?? null
|
||||
}
|
||||
|
||||
const resultsDir = join(scriptDir, 'results')
|
||||
mkdirSync(resultsDir, { recursive: true })
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-')
|
||||
const outPath = join(resultsDir, `daemon-pty-churn-rss-${args.label}-${stamp}.json`)
|
||||
const serialized = JSON.stringify(results, null, 2)
|
||||
// Results get committed as evidence — strip host-identifying home paths.
|
||||
const homeEscaped = JSON.stringify(os.homedir()).slice(1, -1)
|
||||
writeFileSync(outPath, serialized.split(homeEscaped).join('~'))
|
||||
console.log(`\n[bench] results written to ${outPath}`)
|
||||
|
||||
for (const [name, summary] of Object.entries(results.summary)) {
|
||||
if (summary && summary.firstRssKb !== undefined) {
|
||||
console.log(
|
||||
`| ${name} | rss ${summary.firstRssKb}→${summary.lastRssKb}KB (peak ${summary.peakRssKb}) | ` +
|
||||
`heapUsed ${summary.firstHeapUsedKb}→${summary.lastHeapUsedKb}KB | ` +
|
||||
`${summary.cycles} cycles / ${Math.round(summary.durationMs / 1000)}s |`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
})
|
||||
Reference in New Issue
Block a user