diff --git a/config/scripts/relay-gc-host.cjs b/config/scripts/relay-gc-host.cjs index 3c8395e30fb..eca72e5ab29 100644 --- a/config/scripts/relay-gc-host.cjs +++ b/config/scripts/relay-gc-host.cjs @@ -1,8 +1,14 @@ // Runs the real relay bundle under --expose-gc and reports post-collection memory on SIGUSR2. // Nothing about the relay changes: this only adds a signal handler before loading it. -const { writeFileSync } = require('node:fs') +// +// If ORCA_HEAP_SNAPSHOT_REQUEST names a file and that file exists when the signal arrives, its +// contents are read as a destination path and a heap snapshot is written there after the +// collection. The request file is consumed, so one request yields one snapshot. +const { writeFileSync, readFileSync, existsSync, unlinkSync } = require('node:fs') const report = process.env.ORCA_GC_REPORT +const snapshotRequest = process.env.ORCA_HEAP_SNAPSHOT_REQUEST + process.on('SIGUSR2', () => { try { // Twice: the first pass can resurrect via finalizers, the second settles it. @@ -11,8 +17,22 @@ process.on('SIGUSR2', () => { } catch { /* --expose-gc absent; report raw numbers so the caller can tell */ } + let snapshot = null try { - writeFileSync(report, JSON.stringify({ at: Date.now(), mem: process.memoryUsage() })) + if (snapshotRequest && existsSync(snapshotRequest)) { + const dest = readFileSync(snapshotRequest, 'utf8').trim() + unlinkSync(snapshotRequest) + if (dest) { + require('node:v8').writeHeapSnapshot(dest) + snapshot = dest + } + } + } catch { + /* a failed snapshot must not stop the memory report */ + } + try { + // Written last: the caller polls for this, so it must not appear before the snapshot is done. + writeFileSync(report, JSON.stringify({ at: Date.now(), mem: process.memoryUsage(), snapshot })) } catch { /* best-effort */ } diff --git a/config/scripts/relay-heap-snapshot-diff.mjs b/config/scripts/relay-heap-snapshot-diff.mjs new file mode 100644 index 00000000000..86a1606a453 --- /dev/null +++ b/config/scripts/relay-heap-snapshot-diff.mjs @@ -0,0 +1,88 @@ +// Aggregates a V8 heap snapshot by constructor and diffs two of them. +// +// Reports object COUNT and total SELF size per constructor, not true retained size -- retained size +// needs a dominator tree, and for finding what churn retains, a constructor whose instance count +// climbs with the cycle count is the signal. Counts are exact. +// +// Usage: node relay-heap-snapshot-diff.mjs [topN] +import { readFileSync } from 'node:fs' + +function aggregate(path) { + const snap = JSON.parse(readFileSync(path, 'utf8')) + const fields = snap.snapshot.meta.node_fields + const typeNames = snap.snapshot.meta.node_types[0] + const width = fields.length + const iType = fields.indexOf('type') + const iName = fields.indexOf('name') + const iSelf = fields.indexOf('self_size') + const nodes = snap.nodes + const strings = snap.strings + const byCtor = new Map() + let totalSelf = 0 + for (let off = 0; off < nodes.length; off += width) { + const type = typeNames[nodes[off + iType]] + const name = strings[nodes[off + iName]] + const self = nodes[off + iSelf] + totalSelf += self + // Key on type+name: "object/Foo" and "string" land in distinct buckets. + const key = `${type}/${name}` + const cur = byCtor.get(key) + if (cur) { + cur.count++ + cur.self += self + } else { + byCtor.set(key, { count: 1, self }) + } + } + return { byCtor, totalSelf, nodeCount: nodes.length / width } +} + +const [beforePath, afterPath, topRaw] = process.argv.slice(2) +if (!beforePath || !afterPath) { + console.error('usage: relay-heap-snapshot-diff.mjs [topN]') + process.exit(1) +} +const top = Number.parseInt(topRaw ?? '25', 10) + +const a = aggregate(beforePath) +const b = aggregate(afterPath) + +const keys = new Set([...a.byCtor.keys(), ...b.byCtor.keys()]) +const rows = [] +for (const key of keys) { + const x = a.byCtor.get(key) ?? { count: 0, self: 0 } + const y = b.byCtor.get(key) ?? { count: 0, self: 0 } + const dCount = y.count - x.count + const dSelf = y.self - x.self + if (dCount === 0 && dSelf === 0) { + continue + } + rows.push({ + key, + beforeCount: x.count, + afterCount: y.count, + dCount, + dSelfKb: +(dSelf / 1024).toFixed(1) + }) +} + +console.log( + JSON.stringify( + { + before: { nodeCount: a.nodeCount, totalSelfMb: +(a.totalSelf / 1048576).toFixed(3) }, + after: { nodeCount: b.nodeCount, totalSelfMb: +(b.totalSelf / 1048576).toFixed(3) }, + deltaSelfMb: +((b.totalSelf - a.totalSelf) / 1048576).toFixed(3), + deltaNodeCount: b.nodeCount - a.nodeCount + }, + null, + 1 + ) +) +console.log('\n--- top growth by self size ---') +for (const r of rows.sort((p, q) => q.dSelfKb - p.dSelfKb).slice(0, top)) { + console.log(JSON.stringify(r)) +} +console.log('\n--- top growth by instance count ---') +for (const r of rows.sort((p, q) => q.dCount - p.dCount).slice(0, top)) { + console.log(JSON.stringify(r)) +} diff --git a/config/scripts/relay-pty-gc-cycles-probe.mjs b/config/scripts/relay-pty-gc-cycles-probe.mjs index 963d0b85dd6..46035fcb774 100644 --- a/config/scripts/relay-pty-gc-cycles-probe.mjs +++ b/config/scripts/relay-pty-gc-cycles-probe.mjs @@ -17,7 +17,10 @@ const SOCK = join(RUNDIR, 'relay.sock') const GC_REPORT = join(RUNDIR, 'gc.json') const HEADER = 13 const PTYS = 20 -const CYCLES = 10 +// Why 30 and not 10: the first ~10 cycles are still inside V8's JIT warmup, where retained heap +// climbs about 0.064 MB/cycle and looks like a linear leak. It decays to ~0.017 MB/cycle over the +// second decade and is flat across the last three. Reading 10 cycles alone produces a false leak. +const CYCLES = Number.parseInt(process.env.ORCA_PROBE_CYCLES ?? '30', 10) rmSync(RUNDIR, { recursive: true, force: true }) mkdirSync(RUNDIR, { recursive: true }) diff --git a/config/scripts/relay-pty-heap-capture-probe.mjs b/config/scripts/relay-pty-heap-capture-probe.mjs new file mode 100644 index 00000000000..f2b45f3b55c --- /dev/null +++ b/config/scripts/relay-pty-heap-capture-probe.mjs @@ -0,0 +1,236 @@ +// HOW TO RUN: copy this file, relay-gc-host.cjs, the built `relay.js` and its `.version` into one +// directory on the target host, install a matching `node-pty` beside them, then `node `. +// +// Same cycle shape as relay-pty-gc-cycles-probe.mjs, but captures a heap snapshot after the forced +// collection at two chosen cycles so the residual can be attributed to a constructor rather than +// guessed at. Feed the two files to relay-heap-snapshot-diff.mjs. +import { spawn } from 'node:child_process' +import net from 'node:net' +import { readFileSync, readdirSync, existsSync, rmSync, mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +const HERE = import.meta.dirname +const VERSION = readFileSync(join(HERE, '.version'), 'utf8').trim() +const RUNDIR = '/tmp/orca-pty-heap-capture' +const SOCK = join(RUNDIR, 'relay.sock') +const GC_REPORT = join(RUNDIR, 'gc.json') +const SNAP_REQUEST = join(RUNDIR, 'snapshot.request') +const HEADER = 13 +const PTYS = 20 +const CYCLES = 10 +const SNAPSHOT_AT = new Set([2, 10]) + +rmSync(RUNDIR, { recursive: true, force: true }) +mkdirSync(RUNDIR, { recursive: true }) + +const enc = (type, id, ack, payload) => { + const h = Buffer.alloc(HEADER) + h[0] = type + h.writeUInt32BE(id, 1) + h.writeUInt32BE(ack, 5) + h.writeUInt32BE(payload.length, 9) + return Buffer.concat([h, payload]) +} +const dec = (buf) => { + const out = [] + let off = 0 + while (buf.length - off >= HEADER) { + const len = buf.readUInt32BE(off + 9) + if (buf.length - off - HEADER < len) { + break + } + out.push({ type: buf[off], payload: buf.subarray(off + HEADER, off + HEADER + len) }) + off += HEADER + len + } + return { frames: out, rest: buf.subarray(off) } +} + +function connect() { + return new Promise((resolve, reject) => { + const sock = net.connect(SOCK) + let buf = Buffer.alloc(0) + sock.on('error', reject) + sock.on('data', (d) => { + buf = Buffer.concat([buf, d]) + const { frames, rest } = dec(buf) + buf = rest + for (const f of frames) { + if (f.type !== 2) { + continue + } + const m = JSON.parse(f.payload.toString()) + if (m.type === 'orca-relay-handshake-ok') { + sock.removeAllListeners('data') + sock._carry = buf + sock._seq = 0 + resolve(sock) + } else { + reject(new Error(m.type)) + } + } + }) + sock.on('connect', () => + sock.write( + enc( + 2, + 0, + 0, + Buffer.from(JSON.stringify({ type: 'orca-relay-handshake', version: VERSION })) + ) + ) + ) + }) +} + +let rpcId = 1 +function rpc(sock, method, params, timeoutMs = 25000) { + return new Promise((resolve, reject) => { + const id = rpcId++ + let buf = sock._carry ?? Buffer.alloc(0) + sock._carry = Buffer.alloc(0) + const timer = setTimeout(() => { + sock.off('data', onData) + reject(new Error(`${method} timed out`)) + }, timeoutMs) + const onData = (d) => { + buf = Buffer.concat([buf, d]) + const { frames, rest } = dec(buf) + buf = rest + for (const f of frames) { + if (f.type !== 1) { + continue + } + let m + try { + m = JSON.parse(f.payload.toString()) + } catch { + continue + } + if (m.id === id) { + clearTimeout(timer) + sock.off('data', onData) + sock._carry = buf + if (m.error) { + reject(new Error(JSON.stringify(m.error))) + } else { + resolve(m.result) + } + } + } + } + sock.on('data', onData) + sock.write( + enc(1, ++sock._seq, 0, Buffer.from(JSON.stringify({ jsonrpc: '2.0', id, method, params }))) + ) + }) +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) +const fds = (pid) => { + try { + return readdirSync(`/proc/${pid}/fd`).length + } catch { + return -1 + } +} + +async function forcedGc(pid, snapshotPath) { + writeFileSync(GC_REPORT, '') + if (snapshotPath) { + writeFileSync(SNAP_REQUEST, snapshotPath) + } + process.kill(pid, 'SIGUSR2') + // A snapshot write is slow; the report is written last so it gates on completion. + for (let i = 0; i < 600; i++) { + await sleep(200) + const raw = readFileSync(GC_REPORT, 'utf8') + if (raw.trim()) { + return JSON.parse(raw) + } + } + return null +} + +async function main() { + const child = spawn( + process.execPath, + [ + '--expose-gc', + join(HERE, 'gc-host.cjs'), + '--detached', + '--sock-path', + SOCK, + '--grace-time', + '0', + '--log-file', + join(RUNDIR, 'relay.log') + ], + { + cwd: RUNDIR, + stdio: ['ignore', 'pipe', 'pipe'], + env: { + ...process.env, + ORCA_GC_REPORT: GC_REPORT, + ORCA_HEAP_SNAPSHOT_REQUEST: SNAP_REQUEST, + ORCA_RELAY_EMPTY_STARTUP_GRACE_MS: '3600000', + ORCA_RELAY_IDLE_GRACE_MS: '3600000' + } + } + ) + child.stderr.on('data', () => {}) + child.stdout.on('data', () => {}) + for (let i = 0; i < 150 && !existsSync(SOCK); i++) { + await sleep(100) + } + await sleep(500) + const pid = child.pid + const obs = await connect() + const rows = [] + + for (let c = 1; c <= CYCLES; c++) { + const holder = await connect() + const ids = [] + for (let i = 0; i < PTYS; i++) { + const r = await rpc(holder, 'pty.spawn', { + cwd: RUNDIR, + cols: 80, + rows: 24, + shell: '/bin/sh', + args: [] + }) + ids.push(r.id) + } + holder.destroy() + await sleep(500) + for (const id of ids) { + await rpc(obs, 'pty.shutdown', { id }) + } + for (let i = 0; i < 120; i++) { + const st = await rpc(obs, 'relay.status', {}) + if (st.ptys.active === 0) { + break + } + await sleep(500) + } + const wantSnapshot = SNAPSHOT_AT.has(c) ? join(RUNDIR, `cycle-${c}.heapsnapshot`) : null + const rep = await forcedGc(pid, wantSnapshot) + rows.push({ + cycle: c, + ptysCreatedSoFar: c * PTYS, + fds: fds(pid), + heapAfterGcMb: +(rep.mem.heapUsed / 1048576).toFixed(3), + snapshot: rep.snapshot + }) + } + + console.log(JSON.stringify(rows, null, 1)) + obs.destroy() + child.kill('SIGKILL') + await sleep(300) + console.log(`RELAY_PID=${pid}`) +} + +main().catch((e) => { + console.error(`PROBE_FAILED: ${e.message}`) + process.exit(1) +})