diff --git a/tests/tools/benchmarks/daemon-bench-inspector-session.mjs b/tests/tools/benchmarks/daemon-bench-inspector-session.mjs new file mode 100644 index 00000000000..0749f51ef90 --- /dev/null +++ b/tests/tools/benchmarks/daemon-bench-inspector-session.mjs @@ -0,0 +1,112 @@ +/** + * Ephemeral-port, pid-verifiable inspector session for daemon benchmarks. + * + * The target daemon is spawned with `--inspect=0`; `waitForInspectorUrl` reads + * the advertised ws:// URL from the child's OWN stderr, so the port is never + * guessed or shared and the target is the spawned process by construction + * (STA-3515 harness finding #1 — a fixed port can attach to an unrelated + * process). Callers must still assert `evaluateJson('process.pid')` equals the + * spawned child's pid before measuring. + */ +import { withTimeout } from './daemon-bench-ndjson-client.mjs' + +export const INSPECTOR_TIMEOUT_MS = 10_000 + +export class InspectorSession { + constructor(wsUrl) { + this.wsUrl = wsUrl + this.ws = null + this.pending = new Map() + this.messageCounter = 0 + } + + async connect() { + this.ws = new WebSocket(this.wsUrl) + await withTimeout( + new Promise((resolvePromise, reject) => { + this.ws.addEventListener('open', () => resolvePromise(), { once: true }) + this.ws.addEventListener('error', () => reject(new Error('inspector websocket error')), { + once: true + }) + }), + INSPECTOR_TIMEOUT_MS, + 'inspector websocket connect' + ) + this.ws.addEventListener('message', (event) => { + let message + try { + message = JSON.parse(typeof event.data === 'string' ? event.data : '') + } catch { + return + } + const pending = message.id !== undefined ? this.pending.get(message.id) : undefined + if (pending) { + this.pending.delete(message.id) + clearTimeout(pending.timer) + if (message.error) { + pending.reject(new Error(`inspector ${pending.method}: ${message.error.message}`)) + } else { + pending.resolve(message.result) + } + } + }) + this.ws.addEventListener('close', () => { + for (const [id, pending] of this.pending) { + clearTimeout(pending.timer) + pending.reject(new Error('inspector websocket closed')) + this.pending.delete(id) + } + }) + } + + send(method, params) { + return new Promise((resolvePromise, reject) => { + const id = ++this.messageCounter + const timer = setTimeout(() => { + this.pending.delete(id) + reject(new Error(`inspector ${method} timed out after ${INSPECTOR_TIMEOUT_MS}ms`)) + }, INSPECTOR_TIMEOUT_MS) + this.pending.set(id, { resolve: resolvePromise, reject, timer, method }) + this.ws.send(JSON.stringify({ id, method, ...(params ? { params } : {}) })) + }) + } + + async evaluateJson(expression) { + const result = await this.send('Runtime.evaluate', { + expression: `JSON.stringify(${expression})`, + returnByValue: true + }) + if (result.exceptionDetails) { + throw new Error(`inspector evaluate threw: ${result.exceptionDetails.text}`) + } + return JSON.parse(result.result.value) + } + + close() { + try { + this.ws?.close() + } catch { + // already closed + } + } +} + +export function waitForInspectorUrl(child) { + return withTimeout( + new Promise((resolvePromise, reject) => { + let buffer = '' + const onData = (chunk) => { + buffer += chunk.toString('utf8') + const match = /Debugger listening on (ws:\/\/[^\s]+)/.exec(buffer) + if (match) { + child.stderr.off('data', onData) + resolvePromise(match[1]) + } + } + child.stderr.on('data', onData) + child.once('exit', () => reject(new Error('daemon exited before advertising inspector'))) + }), + INSPECTOR_TIMEOUT_MS, + 'inspector url discovery' + ) +} diff --git a/tests/tools/benchmarks/daemon-bench-ndjson-client.mjs b/tests/tools/benchmarks/daemon-bench-ndjson-client.mjs new file mode 100644 index 00000000000..9c714d1b674 --- /dev/null +++ b/tests/tools/benchmarks/daemon-bench-ndjson-client.mjs @@ -0,0 +1,180 @@ +/** + * Minimal NDJSON client for benchmarks that talk to a self-spawned Orca + * terminal daemon: control + stream hello handshake, id-matched RPC with + * per-request timeouts, and close/error handling that rejects every pending + * request so a daemon crash can never strand an await (STA-3515 harness + * finding #2). + */ +import { randomUUID } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { connect } from 'node:net' + +export const HELLO_TIMEOUT_MS = 10_000 +export const RPC_TIMEOUT_MS = 30_000 + +export function withTimeout(promise, ms, what) { + let timer + return Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms}ms`)), ms) + }) + ]).finally(() => clearTimeout(timer)) +} + +function connectSocket(path, timeoutMs) { + return new Promise((resolvePromise, reject) => { + const socket = connect(path) + const timer = setTimeout(() => { + socket.destroy() + reject(new Error(`daemon socket connect timed out after ${timeoutMs}ms`)) + }, timeoutMs) + socket.once('connect', () => { + clearTimeout(timer) + resolvePromise(socket) + }) + socket.once('error', (error) => { + clearTimeout(timer) + reject(error) + }) + }) +} + +export class BenchDaemonClient { + constructor(socketPath, tokenPath, protocolVersion, onSessionOutputChars) { + this.socketPath = socketPath + this.tokenPath = tokenPath + this.protocolVersion = protocolVersion + this.onSessionOutputChars = onSessionOutputChars + this.clientId = randomUUID() + this.controlSocket = null + this.streamSocket = null + this.pending = new Map() + this.requestCounter = 0 + } + + async connect() { + this.controlSocket = await connectSocket(this.socketPath, HELLO_TIMEOUT_MS) + await this.hello(this.controlSocket, 'control') + this.attachNdjsonReader(this.controlSocket, (message) => this.settleResponse(message)) + // A stream socket mirrors the real client shape: PTY output leaves the + // daemon instead of accumulating solely because nothing is attached. Its + // data events also PROVE each cycle's output actually flowed. + this.streamSocket = await connectSocket(this.socketPath, HELLO_TIMEOUT_MS) + await this.hello(this.streamSocket, 'stream') + this.attachNdjsonReader(this.streamSocket, (message) => { + if (message.type === 'event' && message.event === 'data') { + this.onSessionOutputChars?.(message.sessionId, message.payload?.data?.length ?? 0) + } + }) + const rejectAll = (reason) => () => { + for (const [id, pending] of this.pending) { + clearTimeout(pending.timer) + pending.reject(new Error(reason)) + this.pending.delete(id) + } + } + this.controlSocket.on('close', rejectAll('daemon control socket closed')) + this.controlSocket.on('error', rejectAll('daemon control socket errored')) + } + + hello(socket, role) { + const token = readFileSync(this.tokenPath, 'utf8').trim() + return withTimeout( + new Promise((resolvePromise, reject) => { + let buffer = '' + const onData = (chunk) => { + buffer += chunk.toString('utf8') + const newline = buffer.indexOf('\n') + if (newline === -1) { + return + } + socket.off('data', onData) + socket.off('close', onClose) + try { + const response = JSON.parse(buffer.slice(0, newline)) + if (response.ok) { + resolvePromise() + } else { + reject(new Error(`hello rejected: ${response.error ?? 'unknown'}`)) + } + } catch (error) { + reject(error) + } + } + const onClose = () => reject(new Error('daemon closed socket before hello response')) + socket.on('data', onData) + socket.once('close', onClose) + socket.write( + `${JSON.stringify({ type: 'hello', version: this.protocolVersion, token, clientId: this.clientId, role })}\n` + ) + }), + HELLO_TIMEOUT_MS, + `daemon ${role} hello` + ) + } + + attachNdjsonReader(socket, onMessage) { + let buffer = '' + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8') + let newline = buffer.indexOf('\n') + while (newline !== -1) { + const line = buffer.slice(0, newline) + buffer = buffer.slice(newline + 1) + newline = buffer.indexOf('\n') + if (!line) { + continue + } + try { + onMessage(JSON.parse(line)) + } catch { + // tolerate partial/foreign lines; the daemon owns the framing + } + } + }) + } + + settleResponse(message) { + const pending = message.id ? this.pending.get(message.id) : undefined + if (pending) { + this.pending.delete(message.id) + clearTimeout(pending.timer) + if (message.ok) { + pending.resolve(message.payload) + } else { + pending.reject(new Error(message.error ?? 'daemon request failed')) + } + } + } + + request(type, payload, timeoutMs = RPC_TIMEOUT_MS) { + return new Promise((resolvePromise, reject) => { + const id = `bench-${++this.requestCounter}` + const timer = setTimeout(() => { + this.pending.delete(id) + reject(new Error(`daemon request ${type} timed out after ${timeoutMs}ms`)) + }, timeoutMs) + this.pending.set(id, { resolve: resolvePromise, reject, timer }) + try { + this.controlSocket.write( + `${JSON.stringify({ id, type, ...(payload !== undefined ? { payload } : {}) })}\n` + ) + } catch (error) { + this.pending.delete(id) + clearTimeout(timer) + reject(error) + } + }) + } + + destroy() { + for (const [id, pending] of this.pending) { + clearTimeout(pending.timer) + pending.reject(new Error('client destroyed')) + this.pending.delete(id) + } + this.controlSocket?.destroy() + this.streamSocket?.destroy() + } +} diff --git a/tests/tools/benchmarks/daemon-build-provenance-gate.mjs b/tests/tools/benchmarks/daemon-build-provenance-gate.mjs new file mode 100644 index 00000000000..e4a7cfd125c --- /dev/null +++ b/tests/tools/benchmarks/daemon-build-provenance-gate.mjs @@ -0,0 +1,136 @@ +/** + * Build-provenance gate for daemon benchmarks (STA-3515 harness finding #3). + * + * A measurement is only attributable to a commit when the artifacts it loads + * were provably built from that commit. `assertDaemonBuildProvenance` FAILS + * LOUDLY unless every link in the chain holds: + * + * 1. `git status` is clean for daemon-relevant paths, so the working tree IS + * `git rev-parse HEAD` for everything measured; + * 2. sha256(config/patches/node-pty@1.1.0.patch) equals the patch hash in + * pnpm-lock.yaml AND the patch hash pnpm baked into the installed + * node-pty realpath — pnpm names the install dir after the patch content + * hash, so this proves the installed native SOURCE is the checked-out + * patch; + * 3. the installed node-pty resolves inside THIS worktree (a sibling + * worktree's symlinked install proves nothing about this HEAD); + * 4. every compiled node-pty binary is at least as new as the newest native + * source file, so a stale binary cannot masquerade as patched; + * 5. out/orcad/daemon-entry.js is rebuilt by THIS call (build-orcad.mjs runs + * here), so the daemon bundle is the checked-out HEAD by construction. + */ +import { execFileSync, spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs' +import { createRequire } from 'node:module' +import { join, resolve, sep } from 'node:path' + +export function assertDaemonBuildProvenance(repoRoot) { + const git = (...cmdArgs) => + execFileSync('git', cmdArgs, { cwd: repoRoot, encoding: 'utf8' }).trim() + const headSha = git('rev-parse', 'HEAD') + const branch = git('rev-parse', '--abbrev-ref', 'HEAD') + + const relevant = ['src/main/', 'src/shared/', 'config/patches/', 'pnpm-lock.yaml', 'package.json'] + // NOTE: no .trim() on the whole output — porcelain lines start with a + // two-character status whose first column may be a meaningful space. + const dirty = execFileSync('git', ['status', '--porcelain'], { cwd: repoRoot, encoding: 'utf8' }) + .split('\n') + .filter((line) => line.length > 3) + .map((line) => line.slice(3)) + .filter((file) => relevant.some((prefix) => file.startsWith(prefix))) + if (dirty.length > 0) { + throw new Error( + `PROVENANCE FAILURE: daemon-relevant paths are dirty; results would not be attributable to HEAD ${headSha}:\n ${dirty.join('\n ')}` + ) + } + + const patchFile = join(repoRoot, 'config', 'patches', 'node-pty@1.1.0.patch') + const patchSha256 = createHash('sha256').update(readFileSync(patchFile)).digest('hex') + const lockfile = readFileSync(join(repoRoot, 'pnpm-lock.yaml'), 'utf8') + const lockHashMatch = /node-pty@1\.1\.0:\s*\n\s*hash: ([0-9a-f]{64})/.exec(lockfile) + if (!lockHashMatch || lockHashMatch[1] !== patchSha256) { + throw new Error( + `PROVENANCE FAILURE: pnpm-lock node-pty patch hash (${lockHashMatch?.[1] ?? 'missing'}) != sha256 of checked-out patch (${patchSha256})` + ) + } + + const requireFromRoot = createRequire(join(repoRoot, 'package.json')) + const installedPkgJson = requireFromRoot.resolve('node-pty/package.json') + const installedRealRoot = realpathSync(resolve(installedPkgJson, '..')) + if (!installedRealRoot.includes(`patch_hash=${patchSha256}`)) { + throw new Error( + `PROVENANCE FAILURE: installed node-pty at ${installedRealRoot} was not built from the checked-out patch (expected patch_hash=${patchSha256}). Run pnpm install.` + ) + } + if (!installedRealRoot.startsWith(repoRoot + sep)) { + throw new Error( + `PROVENANCE FAILURE: node_modules/node-pty resolves outside this worktree (${installedRealRoot}); a sibling worktree's install cannot prove this HEAD` + ) + } + + const releaseDir = join(installedRealRoot, 'build', 'Release') + const nativeBinaries = existsSync(releaseDir) + ? readdirSync(releaseDir).filter( + (name) => name.endsWith('.node') || name === 'spawn-helper' || name.endsWith('.exe') + ) + : [] + if (nativeBinaries.length === 0) { + throw new Error(`PROVENANCE FAILURE: no compiled node-pty binaries under ${releaseDir}`) + } + const srcDir = join(installedRealRoot, 'src') + let newestSourceMtime = 0 + for (const entry of readdirSync(srcDir, { withFileTypes: true, recursive: true })) { + if (entry.isFile()) { + newestSourceMtime = Math.max( + newestSourceMtime, + statSync(join(entry.parentPath, entry.name)).mtimeMs + ) + } + } + const staleBinaries = nativeBinaries.filter( + (name) => statSync(join(releaseDir, name)).mtimeMs < newestSourceMtime + ) + if (staleBinaries.length > 0) { + throw new Error( + `PROVENANCE FAILURE: node-pty binaries older than patched sources (stale: ${staleBinaries.join(', ')}). Rebuild node-pty.` + ) + } + + // Rebuild the daemon bundle from the (clean) working tree = HEAD by construction. + console.log('[provenance] rebuilding daemon bundle from HEAD…') + const daemonBundle = join(repoRoot, 'out', 'orcad', 'daemon-entry.js') + const buildStartedAt = Date.now() + const build = spawnSync( + process.execPath, + [join(repoRoot, 'config', 'scripts', 'build-orcad.mjs')], + { + cwd: repoRoot, + stdio: 'inherit' + } + ) + if (build.status !== 0) { + throw new Error(`PROVENANCE FAILURE: daemon bundle build exited ${build.status}`) + } + if (!existsSync(daemonBundle) || statSync(daemonBundle).mtimeMs < buildStartedAt) { + throw new Error('PROVENANCE FAILURE: daemon bundle missing or not refreshed by this build') + } + + const provenance = { + headSha, + // Parents make a local merge HEAD attributable to its upstream inputs. + headParents: git('log', '-1', '--format=%P').split(' '), + branch, + patchSha256, + installedNodePtyRealpath: installedRealRoot, + nativeBinaries, + daemonBundleBuiltAt: new Date(buildStartedAt).toISOString(), + nodeVersion: process.version, + platform: process.platform, + arch: process.arch + } + console.log( + `[provenance] OK: HEAD ${headSha} (${branch}), node-pty patch ${patchSha256.slice(0, 12)}…` + ) + return provenance +} diff --git a/tests/tools/benchmarks/daemon-pty-churn-rss-bench.mjs b/tests/tools/benchmarks/daemon-pty-churn-rss-bench.mjs new file mode 100644 index 00000000000..f6ecae5fb34 --- /dev/null +++ b/tests/tools/benchmarks/daemon-pty-churn-rss-bench.mjs @@ -0,0 +1,406 @@ +#!/usr/bin/env node +/** + * Orca terminal-daemon RSS measurement under PTY churn (STA-3515). + * + * Measures whether daemon memory grows without bound across repeated PTY + * create/output/kill cycles, and separates: + * - the NATURAL series: RSS + V8 heap sampled with NO forced GC, so transient + * JS retention stays visible; + * - the FORCED-GC series: a separate daemon run where GC is forced via the + * inspector immediately before every sample, showing the reclaimable floor. + * The natural run additionally takes one clearly-labelled post-run forced-GC + * sample, which is the leak-vs-retention discriminator: memory that survives a + * final full GC after all sessions are dead is either allocator retention or a + * real leak; memory that drops was transient JS retention. + * + * Every run starts with the build-provenance gate in + * daemon-build-provenance-gate.mjs, which FAILS LOUDLY unless the daemon + * bundle and the patched node-pty binaries are provably built from the + * checked-out HEAD. + * + * The daemon is spawned with `--inspect=0` (ephemeral port). The inspector URL + * is taken from the child's own stderr — never from a fixed port — and + * ownership is asserted by evaluating `process.pid` over the socket and + * comparing it to the spawned child's pid before any measurement. + * + * Usage: + * node tests/tools/benchmarks/daemon-pty-churn-rss-bench.mjs --label baseline + * [--cycles 400] [--sample-every 20] [--output-bytes 65536] + * [--series both|natural|forced-gc] [--settle-ms 30] [--probe-fds] + * + * Results: tests/tools/benchmarks/results/daemon-pty-churn-rss-