From 414799e41a092f71bf20109457cd0daf935c8505 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:30:22 -0800 Subject: [PATCH] Add active-chat resident memory comparison to 4a harness --- .../native-chat/native-chat-4a-comparison.ts | 209 ++++++------ .../native-chat/native-chat-4a-fixture.ts | 44 ++- .../native-chat/native-chat-4a-ops-memory.ts | 306 ++++++++++++++++++ .../native-chat/native-chat-4a-probes.ts | 7 +- .../native-chat/native-chat-4a-report.ts | 106 +++++- .../native-chat/native-chat-4a-run-context.ts | 62 +++- .../native-chat/native-chat-4a-session.ts | 5 +- .../native-chat/native-chat-4a-workloads.ts | 7 +- 8 files changed, 631 insertions(+), 115 deletions(-) create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-memory.ts diff --git a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts index cd879a64aea..d1d197b03ab 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts @@ -2,21 +2,15 @@ // Dry run (no app): pnpm exec tsx config/scripts/performance/native-chat/native-chat-4a-comparison.ts --profile full --dry-run // Full run: ... --arm main=@ --arm candidate=@ --profile full --out [--large-source] // Re-table: ... --out --report-only [--arm main=@ --arm candidate=@] -import { - existsSync, - mkdirSync, - readdirSync, - readFileSync, - renameSync, - writeFileSync -} from 'node:fs' +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' import os from 'node:os' import path from 'node:path' import { parseArgs } from 'node:util' import type { Arm } from './native-chat-4a-arm' import { assertOneRun, parseArmSpec, seededBy } from './native-chat-4a-arm-provenance' import type { Manifest } from './native-chat-4a-fixture' -import type { RunContext } from './native-chat-4a-run-context' +import { profileRunner } from './native-chat-4a-run-context' +import * as memory from './native-chat-4a-ops-memory' import { armMarkdown, writeComparison, type ArmResult, type Row } from './native-chat-4a-report' import { conversionToolOutputBytes, @@ -34,7 +28,8 @@ const FAMILIES = [ 'records', 'retained', 'open', - 'rewind' + 'rewind', + 'memory' ] as const type Family = (typeof FAMILIES)[number] @@ -60,76 +55,39 @@ const workload = workloadFor( ) const log = (line: string): void => console.error(`[4a ${new Date().toISOString()}] ${line}`) +type SeededProfile = { seedDir: string; manifest: Manifest } + const seededChatCount = (manifest: Manifest): number => 1 + Number(manifest.mediumSessionId !== null) + manifest.fillerSessionIds.length async function runArm(input: { arm: Arm out: string - seedDir: string - manifest: Manifest + profile?: SeededProfile + memoryProfile?: SeededProfile + memorySeedFailure?: string stubCommand: string families: Set largeSource?: { seedDir: string; manifest: Manifest } }): Promise { - const { copyProfile, launchArm, stopArm } = await import('./native-chat-4a-arm') - const { resetTags } = await import('./native-chat-4a-run-context') const lifecycle = await import('./native-chat-4a-ops-lifecycle') const conversion = await import('./native-chat-4a-ops-conversion') - const { ConversionObserver } = await import('./native-chat-4a-sqlite-observer') const queue = await import('./native-chat-4a-ops-queue') const retained = await import('./native-chat-4a-ops-retained') const stream = await import('./native-chat-4a-ops-stream') const reads = await import('./native-chat-4a-ops-reads') const shapes = await import('./native-chat-4a-ops-shapes') - const profileDir = path.join(input.out, 'profile') const rows: Row[] = [] const startedAt = new Date().toISOString() - // Each seed copy's first launch converts on the candidate; the observer starts before it, reading only. - const onProfile = async ( - seedDir: string, - manifest: typeof input.manifest, - phase: string, - body: (ctx: RunContext, observer: InstanceType) => Promise - ) => { - if (existsSync(profileDir)) { - renameSync(profileDir, path.join(input.out, `stale-profile-${Date.now()}`)) - } - copyProfile(seedDir, profileDir) - resetTags() - const observer = new ConversionObserver(profileDir) - const open = () => launchArm(input.arm, profileDir, input.stubCommand) - const ctx = { - arm: input.arm, - launch: await open().catch(async (error: unknown) => { - await observer.stop() - throw error - }), - workload, - manifest, - rows, - log, - relaunch: async () => undefined - } - ctx.relaunch = async () => { - await stopArm(ctx.launch) - ctx.launch = await open() - } - try { - await body(ctx, observer) - } finally { - await observer.stop() - await stopArm(ctx.launch) - renameSync( - profileDir, - path.join( - input.out, - `after-${input.arm.name}-${phase}-${manifest.toolOutputBytes}-${input.arm.sha.slice(0, 8)}` - ) - ) - } - } + const onProfile = profileRunner({ + arm: input.arm, + out: input.out, + stubCommand: input.stubCommand, + workload, + rows, + log + }) const run = async (family: Family, body: () => Promise) => { if (!input.families.has(family)) { @@ -141,42 +99,62 @@ async function runArm(input: { } catch (error) { const reason = `family failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300) - rows.push({ - family, - operation: '(family)', - variant: 'all', - samples: [], - context: {}, - skipped: reason - }) + if (family === 'memory') { + memory.skipMissingMemoryRows(rows, workload, reason) + } else { + rows.push({ + family, + operation: '(family)', + variant: 'all', + samples: [], + context: {}, + skipped: reason + }) + } log(`${input.arm.name}: ${family} ${reason}`) } } - const label = `${workload.history.largeTurns}-turn/${seededChatCount(input.manifest)}-chat profile` - if (input.families.has('conversion')) { - // A copy of its own, left idle, so the sweep's whole duration and per-chat intervals carry no foreground work. - await onProfile(input.seedDir, input.manifest, 'sweep', (ctx, observer) => - run('conversion', () => conversion.sweepRows(ctx, observer, label)) - ) - } - await onProfile(input.seedDir, input.manifest, 'families', async (ctx, observer) => { - const seedFiles = conversion.seededDbFiles(input.seedDir) - // First: the seeded tail markers and unconverted state exist only before any other family writes. - await run('conversion', () => conversion.firstLaunchRows(ctx, seedFiles, label, observer)) - await conversion.awaitSweep(ctx, observer, label) - await run('send', () => queue.sendAndQueueRows(ctx)) - await run('stream', () => stream.streamingRows(ctx)) - await run('history', async () => { - await reads.historyRows(ctx) - await shapes.subagentRows(ctx) - await shapes.itemShapeRows(ctx) + const profile = input.profile + if (profile) { + const label = `${workload.history.largeTurns}-turn/${seededChatCount(profile.manifest)}-chat profile` + if (input.families.has('conversion')) { + // A copy of its own, left idle, so the sweep's whole duration and per-chat intervals carry no foreground work. + await onProfile(profile.seedDir, profile.manifest, 'sweep', (ctx, observer) => + run('conversion', () => conversion.sweepRows(ctx, observer, label)) + ) + } + await onProfile(profile.seedDir, profile.manifest, 'families', async (ctx, observer) => { + const seedFiles = conversion.seededDbFiles(profile.seedDir) + // First: the seeded tail markers and unconverted state exist only before any other family writes. + await run('conversion', () => conversion.firstLaunchRows(ctx, seedFiles, label, observer)) + await conversion.awaitSweep(ctx, observer, label) + await run('send', () => queue.sendAndQueueRows(ctx)) + await run('stream', () => stream.streamingRows(ctx)) + await run('history', async () => { + await reads.historyRows(ctx) + await shapes.subagentRows(ctx) + await shapes.itemShapeRows(ctx) + }) + await run('outline', () => reads.outlineAndForkRows(ctx)) + await run('records', () => reads.recordRows(ctx)) + await run('retained', () => retained.retainedAdmissionRows(ctx)) + await run('open', () => lifecycle.openRows(ctx)) + await run('rewind', () => lifecycle.rewindRows(ctx)) }) - await run('outline', () => reads.outlineAndForkRows(ctx)) - await run('records', () => reads.recordRows(ctx)) - await run('retained', () => retained.retainedAdmissionRows(ctx)) - await run('open', () => lifecycle.openRows(ctx)) - await run('rewind', () => lifecycle.rewindRows(ctx)) + } + await run('memory', async () => { + if (!input.memoryProfile) { + throw new Error(input.memorySeedFailure ?? 'memory seed unavailable') + } + const source = input.memoryProfile + await onProfile( + source.seedDir, + source.manifest, + 'memory', + memory.convertedMemoryRows, + memory.memoryWorkload(workload) + ) }) if (input.largeSource && input.families.has('conversion')) { const source = input.largeSource @@ -192,6 +170,7 @@ async function runArm(input: { ) ) } + const manifest = profile?.manifest ?? input.memoryProfile?.manifest return { arm: input.arm.name, sha: input.arm.sha, @@ -200,22 +179,28 @@ async function runArm(input: { finishedAt: new Date().toISOString(), rows, fixture: { - repoId: input.manifest.workspace.repoId, - worktreeIds: input.manifest.workspace.worktreeIds.join(' '), - seedArm: input.manifest.seedArm, - seedSha: input.manifest.seedSha, - recipe: input.manifest.recipe, + repoId: manifest?.workspace.repoId ?? 'unavailable', + worktreeIds: manifest?.workspace.worktreeIds.join(' ') ?? 'unavailable', + seedArm: manifest?.seedArm ?? 'unavailable', + seedSha: manifest?.seedSha ?? 'unavailable', + recipe: manifest?.recipe ?? 'unavailable', largeSourceRecipe: input.largeSource?.manifest.recipe ?? 'none', - largeItems: input.manifest.largeItems, - storageBytes: input.manifest.storageBytes, - seedMs: input.manifest.seedMs + largeItems: manifest?.largeItems ?? 0, + storageBytes: manifest?.storageBytes ?? 0, + seedMs: manifest?.seedMs ?? 0, + memoryRecipe: + input.memoryProfile?.manifest.recipe ?? input.memorySeedFailure ?? 'not selected', + memorySeedMs: input.memoryProfile?.manifest.seedMs ?? 'unavailable', + memorySeedProfileLogicalBytes: input.memoryProfile?.manifest.profileBytes ?? 'unavailable' } } } async function main(): Promise { if (values['dry-run']) { - console.log(JSON.stringify(dryRunWorkload(workload), null, 2)) + console.log( + JSON.stringify({ ...dryRunWorkload(workload), memory: memory.memoryShape(workload) }, null, 2) + ) return } const out = path.resolve(values.out ?? path.join(os.tmpdir(), `orca-4a-${workload.profile}`)) @@ -260,20 +245,31 @@ async function main(): Promise { : createSeededTestRepo({ publishPath: false }) writeFileSync(repoFile, repoPath) const seedArm = arms.find((arm) => arm.name === values['seed-arm']) ?? arms[0]! - const seed = (toolOutputBytes: number, seeded: Workload) => + const seed = (toolOutputBytes: number, seeded: Workload, runDir = out) => seedProfile({ arm: seedArm, workload: seeded, - runDir: out, + runDir, profileDir: path.join(out, 'profile'), repoPath, stubCommand, toolOutputBytes, log }) - const manifest = seededBy(seedArm, await seed(workload.history.toolOutputBytes, workload)) + const manifest = [...families].some((family) => family !== 'memory') + ? seededBy(seedArm, await seed(workload.history.toolOutputBytes, workload)) + : null + const { memoryProfile, memorySeedFailure } = families.has('memory') + ? await memory.seedMemoryProfile( + out, + workload, + async (seeded, runDir) => + seededBy(seedArm, await seed(seeded.history.toolOutputBytes, seeded, runDir)), + log + ) + : {} const large = - values['large-source'] && workload.conversionSourceBytes > 0 + families.has('conversion') && values['large-source'] && workload.conversionSourceBytes > 0 ? seededBy( seedArm, await seed(conversionToolOutputBytes(workload), { @@ -287,8 +283,9 @@ async function main(): Promise { const result = await runArm({ arm, out, - seedDir: seedDirFor(out, manifest), - manifest, + ...(manifest ? { profile: { seedDir: seedDirFor(out, manifest), manifest } } : {}), + memoryProfile, + memorySeedFailure, stubCommand, families, ...(large ? { largeSource: { seedDir: seedDirFor(out, large), manifest: large } } : {}) diff --git a/config/scripts/performance/native-chat/native-chat-4a-fixture.ts b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts index 40ed48a527b..2e88fa26a58 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-fixture.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts @@ -32,19 +32,42 @@ export type Manifest = { largeItems: number mediumItems: number storageBytes: number + /** Logical file bytes including the isolated provider home; absent in older cached seeds. */ + profileBytes?: number seedMs: number } const SLEEP_FILLERS_EVERY = 20 /** Count the large chat's rows by paging back from the tail; refuses a fixture that seeded fewer rows than its recipe. */ -async function countItems(page: Page, sessionId: string): Promise { +async function countItems( + page: Page, + sessionId: string, + expectedTools?: { count: number; bytes: number } +): Promise { let count = 0 + let tools = 0 let cursor: unknown for (;;) { const read = await history(page, sessionId, cursor ? 'before' : 'tail', 200, cursor) count += read.page.items.length + if (expectedTools) { + for (const { body } of read.page.items) { + if (body?.kind !== 'tool-call') { + continue + } + if (body.state !== 'completed' || body.output?.byteLength !== expectedTools.bytes) { + throw new Error(`seed: ${sessionId} has an incomplete tool or wrong output size`) + } + tools += 1 + } + } if (!read.page.hasOlder) { + if (expectedTools && tools !== expectedTools.count) { + throw new Error( + `seed: ${sessionId} has ${tools} completed tools, expected ${expectedTools.count}` + ) + } return count } cursor = read.page.window.nextCursor @@ -92,7 +115,16 @@ async function seedRecipeChat( } await waitForDurable(page, created.sessionId, `4A-TAIL-${chat}-${turn - 1}`, 600_000) await waitForIdle(page, created.sessionId, 600_000) - const items = await countItems(page, created.sessionId) + const items = await countItems( + page, + created.sessionId, + workload.history.fillerTools === undefined + ? undefined + : { + count: turns * workload.history.tools, + bytes: workload.history.toolOutputBytes + } + ) // Hosts may group a call with its result; fewer than prompt+reply per turn means turns were lost. if (items < turns * 2) { throw new Error(`seed: ${chat} chat holds ${items} rows for ${turns} turns`) @@ -175,6 +207,13 @@ export async function seedProfile(input: { await waitForDurable(page, filler.sessionId, `4A-TAIL-filler-${chat}-${fillerTurnIndex}`) } fillerSessionIds.push(filler.sessionId) + if (input.workload.history.fillerTools !== undefined) { + await waitForIdle(page, filler.sessionId, 600_000) + await countItems(page, filler.sessionId, { + count: input.workload.history.fillerTurns * input.workload.history.fillerTools, + bytes: input.workload.history.fillerToolOutputBytes ?? 500 + }) + } // Bound live provider processes: retained chats stay, their stubs stop. if ((chat + 1) % SLEEP_FILLERS_EVERY === 0) { await rpc(page, 'agentSession.sleepWorkspace', { workspaceId }) @@ -210,6 +249,7 @@ export async function seedProfile(input: { largeItems: large.items, mediumItems: medium?.items ?? 0, storageBytes: storageBytes(input.profileDir).total, + profileBytes: storageBytes(input.profileDir, true).total, seedMs: Date.now() - started } renameSync(input.profileDir, seedDir) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-memory.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-memory.ts new file mode 100644 index 00000000000..7b92eef84be --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-memory.ts @@ -0,0 +1,306 @@ +// Main-process residency on an independent, uniform completed-tool seed; subscriptions retain no payloads in the driver. +import { existsSync, mkdirSync, renameSync } from 'node:fs' +import path from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { seedDirFor, seededChats, type Manifest } from './native-chat-4a-fixture' +import type { Row } from './native-chat-4a-report' +import { stat } from './native-chat-4a-report' +import { page, type RunContext } from './native-chat-4a-run-context' +import { history } from './native-chat-4a-session' +import type { ConversionObserver } from './native-chat-4a-sqlite-observer' +import type { Workload } from './native-chat-4a-workloads' + +const METRICS = ['heapUsed', 'rss', 'external', 'arrayBuffers'] as const +const PHASES = [ + 'before activation', + 'after activation', + 'activation delta', + 'before activation post-GC', + 'after activation post-GC', + 'activation delta post-GC' +] as const +type Phase = (typeof PHASES)[number] +type Memory = Partial> +const WINDOW = 5 +const INTERVAL_MS = 1000 +const SETTLE_TIMEOUT_MS = 60_000 + +/** Kept outside the established history workload, including its cache directory and recipe digest. */ +export function memoryWorkload(workload: Workload): Workload { + const tools = workload.profile === 'full' ? 480 : 48 + return { + ...workload, + history: { + largeTurns: 1, + mediumTurns: 0, + tools, + toolOutputBytes: 32 * 1024, + promptBytes: 300, + replyBytes: 400, + queuedEvery: 0, + fillerChats: (workload.profile === 'full' ? 500 : 20) - 1, + fillerTurns: 1, + fillerTools: tools, + fillerToolOutputBytes: 32 * 1024 + }, + diffs: { count: 0, patchBytes: 0 } + } +} + +export function memoryShape(workload: Workload): Record { + const { history: h } = memoryWorkload(workload) + return { + chats: h.fillerChats + 1, + completedToolsPerChat: h.tools, + toolOutputBytes: h.toolOutputBytes, + outputPayloadBytes: (h.fillerChats + 1) * h.tools * h.toolOutputBytes, + activation: 'agentSession.history then held agentSession.subscribe per chat; no provider start', + source: 'Electron main process.memoryUsage(); RSS is resident bytes, not physical footprint', + settling: 'last 5 samples, 1s apart, heapUsed/RSS range <= max(5% of minimum, 8MiB); 60s limit' + } +} + +/** Even a launch or seed failure produces a named skipped row for every requested counter and phase. */ +export function skipMissingMemoryRows(rows: Row[], workload: Workload, reason: string): void { + for (const phase of PHASES) { + for (const metric of METRICS) { + const existing = rows.find( + (row) => row.family === 'memory' && row.operation === metric && row.variant === phase + ) + if (existing && existing.samples.length === 0 && !existing.skipped) { + existing.skipped = reason + } + if (!existing) { + rows.push({ + family: 'memory', + operation: metric, + variant: phase, + samples: [], + context: memoryShape(workload), + skipped: reason + }) + } + } + } +} + +export async function memoryRows(ctx: RunContext): Promise { + const context = { + ...memoryShape(ctx.workload), + seedRecipe: ctx.manifest.recipe, + seedArm: ctx.manifest.seedArm, + seedSha: ctx.manifest.seedSha, + seedMs: ctx.manifest.seedMs, + seedStorageBytes: ctx.manifest.storageBytes, + seedProfileLogicalBytes: ctx.manifest.profileBytes ?? 'not recorded in cached seed', + mainPid: ctx.launch.pid + } + const rows = new Map() + for (const phase of PHASES) { + const group = METRICS.map((metric): Row => ({ + family: 'memory', + operation: metric, + variant: phase, + samples: [], + context + })) + ctx.rows.push(...group) + rows.set(phase, group) + } + const groupFor = (phase: Phase): Row[] => rows.get(phase)! + const skip = (phase: Phase, reason: string): void => { + for (const row of groupFor(phase)) { + row.skipped = reason + } + } + const snapshot = (): Promise => + ctx.launch.app.evaluate(() => { + const { heapUsed, rss, external, arrayBuffers } = process.memoryUsage() + return { heapUsed, rss, external, arrayBuffers } + }) + const measure = async (phase: Phase): Promise => { + const started = Date.now() + const samples: Memory[] = [] + let stable = false + do { + if (samples.length > 0) { + await delay(INTERVAL_MS) + } + samples.push(await snapshot()) + if (samples.length > WINDOW) { + samples.shift() + } + stable = + samples.length === WINDOW && + (['heapUsed', 'rss'] as const).every((metric) => { + const values = samples.map((sample) => sample[metric]) + if (values.some((value) => value === undefined || !Number.isFinite(value))) { + return false + } + const present = values.filter((value): value is number => typeof value === 'number') + const min = Math.min(...present) + return Math.max(...present) - min <= Math.max(min * 0.05, 8 * 1024 ** 2) + }) + } while (!stable && Date.now() - started < SETTLE_TIMEOUT_MS) + for (const row of groupFor(phase)) { + row.context = { ...context, settleMs: Date.now() - started } + for (const sample of samples) { + const value = METRICS.find((metric) => metric === row.operation) + const bytes = value ? sample[value] : undefined + if (typeof bytes === 'number' && Number.isFinite(bytes)) { + row.samples.push({ extra: { bytes } }) + } + } + if (!stable) { + row.skipped = 'heapUsed/RSS did not settle within 60s; diagnostic samples only' + } else if (row.samples.length !== WINDOW) { + row.skipped = 'counter unavailable in process.memoryUsage()' + } + } + if (!stable) { + throw new Error(`${phase}: heapUsed/RSS did not settle within 60s`) + } + } + const afterGc = async (phase: Phase): Promise => { + try { + const collected = await ctx.launch.app.evaluate(() => { + if (typeof global.gc !== 'function') { + return false + } + global.gc() + return true + }) + if (!collected) { + skip(phase, 'main global.gc unavailable; no product hook or launch flag added') + return + } + await measure(phase) + } catch (error) { + skip(phase, `GC measurement failed: ${String(error).slice(0, 200)}`) + } + } + const delta = (before: Phase, after: Phase, phase: Phase): void => { + for (const row of groupFor(phase)) { + const left = groupFor(before).find((value) => value.operation === row.operation)! + const right = groupFor(after).find((value) => value.operation === row.operation)! + const a = stat(left.samples.map((sample) => sample.extra?.bytes)) + const b = stat(right.samples.map((sample) => sample.extra?.bytes)) + if (left.skipped || right.skipped || !a || !b) { + row.skipped = left.skipped ?? right.skipped ?? 'baseline or active counter unavailable' + } else { + row.samples.push({ extra: { bytes: b.p50 - a.p50 } }) + } + } + } + const subscriptions = await page(ctx).evaluateHandle(() => { + const handles: { unsubscribe: () => void }[] = [] + const errors: string[] = [] + return { handles, errors } + }) + try { + await measure('before activation') + await afterGc('before activation post-GC') + const chats = seededChats(ctx.manifest) + if (chats.length !== ctx.workload.history.fillerChats + 1) { + throw new Error('memory seed chat count mismatch') + } + for (const [index, sessionId] of chats.entries()) { + await history(page(ctx), sessionId, 'tail', 1) + await subscriptions.evaluate(async (state, sessionId) => { + const handle = await window.api.runtime.subscribe( + { method: 'agentSession.subscribe', params: { sessionId } }, + (reply) => { + if (!reply.ok && state.errors.length < 10) { + state.errors.push(`${sessionId}: ${JSON.stringify(reply.error)}`) + } + if ( + reply.ok && + typeof reply.result === 'object' && + reply.result !== null && + 'type' in reply.result && + reply.result.type === 'end' && + state.errors.length < 10 + ) { + state.errors.push(`${sessionId}: subscription ended before sampling finished`) + } + } + ) + state.handles.push(handle) + }, sessionId) + if ((index + 1) % 20 === 0) { + ctx.log(`${ctx.arm.name}: memory ${index + 1}/${chats.length} subscribed`) + } + } + const checkSubscriptions = async (): Promise => { + const errors = await subscriptions.evaluate((state) => state.errors) + if (errors.length > 0) { + throw new Error(`memory subscription failed: ${errors.join('; ').slice(0, 300)}`) + } + } + await checkSubscriptions() + await measure('after activation') + await afterGc('after activation post-GC') + await checkSubscriptions() + } catch (error) { + const reason = `memory measurement failed: ${String(error).slice(0, 250)}` + for (const phase of PHASES) { + if ( + phase.startsWith('after') || + groupFor(phase).every((row) => row.samples.length === 0 && !row.skipped) + ) { + skip(phase, reason) + } + } + } finally { + await subscriptions + .evaluate((state) => { + for (const handle of state.handles) { + handle.unsubscribe() + } + state.handles.length = 0 + }) + .finally(() => subscriptions.dispose()) + delta('before activation', 'after activation', 'activation delta') + delta('before activation post-GC', 'after activation post-GC', 'activation delta post-GC') + } +} + +export async function seedMemoryProfile( + out: string, + workload: Workload, + seed: (workload: Workload, runDir: string) => Promise, + log: (line: string) => void +): Promise<{ + memoryProfile?: { seedDir: string; manifest: Manifest } + memorySeedFailure?: string +}> { + const runDir = path.join(out, 'memory') + mkdirSync(runDir, { recursive: true }) + try { + const manifest = await seed(memoryWorkload(workload), runDir) + return { memoryProfile: { seedDir: seedDirFor(runDir, manifest), manifest } } + } catch (error) { + const memorySeedFailure = `memory seed failed: ${String(error).slice(0, 250)}` + log(memorySeedFailure) + const profile = path.join(out, 'profile') + if (existsSync(profile)) { + renameSync(profile, path.join(out, `failed-memory-seed-${Date.now()}`)) + } + return { memorySeedFailure } + } +} + +export async function convertedMemoryRows( + ctx: RunContext, + observer: ConversionObserver +): Promise { + // Conversion completes before the baseline on this otherwise untouched copy. + if ( + observer.databaseFile && + !(await observer.allReady(seededChats(ctx.manifest), 2 * 60 * 60_000)) + ) { + throw new Error('memory profile conversion did not finish within 2h') + } + await observer.stop() + await memoryRows(ctx) +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-probes.ts b/config/scripts/performance/native-chat/native-chat-4a-probes.ts index 4ae01aa88e1..9f80b503e42 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-probes.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-probes.ts @@ -204,7 +204,10 @@ export async function hostWorkMark(app: ElectronApplication): Promise<() => Prom } /** All SQLite/journal files under the profile, so WAL growth and file growth read the same on both storage designs. */ -export function storageBytes(userDataDir: string): { db: number; wal: number; total: number } { +export function storageBytes( + userDataDir: string, + includeHome = false +): { db: number; wal: number; total: number } { let db = 0 let wal = 0 let total = 0 @@ -212,7 +215,7 @@ export function storageBytes(userDataDir: string): { db: number; wal: number; to for (const entry of readdirSync(dir, { withFileTypes: true })) { const file = path.join(dir, entry.name) if (entry.isDirectory()) { - if (entry.name !== 'home') { + if (includeHome || entry.name !== 'home') { visit(file) } continue diff --git a/config/scripts/performance/native-chat/native-chat-4a-report.ts b/config/scripts/performance/native-chat/native-chat-4a-report.ts index 4947bc5cfef..4cde1a193f1 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-report.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-report.ts @@ -116,6 +116,102 @@ function contextCell(row: Row | undefined): string { .join(', ') } +function memoryValue(row: Row | undefined): number | null { + return row?.skipped + ? null + : (stat(row?.samples.map((sample) => sample.extra?.bytes) ?? [])?.p50 ?? null) +} + +/** Positive reduction means the candidate uses fewer bytes; negative activation deltas have no relative denominator. */ +export function memoryComparison(base: ArmResult, next: ArmResult) { + const left = new Map( + base.rows.filter((row) => row.family === 'memory').map((row) => [key(row), row]) + ) + const right = new Map( + next.rows.filter((row) => row.family === 'memory').map((row) => [key(row), row]) + ) + return [...new Set([...left.keys(), ...right.keys()])].map((id) => { + const a = left.get(id) + const b = right.get(id) + if (a && b && !a.skipped && !b.skipped) { + for (const field of [ + 'seedRecipe', + 'seedSha', + 'chats', + 'completedToolsPerChat', + 'toolOutputBytes' + ]) { + if (a.context[field] === undefined || a.context[field] !== b.context[field]) { + throw new Error( + `memory comparison: arms disagree on or lack ${field}; use one identical seed` + ) + } + } + } + const mainBytes = memoryValue(a) + const candidateBytes = memoryValue(b) + const reductionBytes = + mainBytes === null || candidateBytes === null ? null : mainBytes - candidateBytes + return { + operation: (a ?? b)!.operation, + phase: (a ?? b)!.variant, + mainBytes, + candidateBytes, + reductionBytes, + reductionPercent: + reductionBytes !== null && mainBytes !== null && mainBytes > 0 + ? (reductionBytes / mainBytes) * 100 + : null, + mainSkipped: a?.skipped ?? (mainBytes === null ? 'measurement missing' : null), + candidateSkipped: b?.skipped ?? (candidateBytes === null ? 'measurement missing' : null) + } + }) +} + +function memoryMarkdown(base: ArmResult, next?: ArmResult): string[] { + const compared = memoryComparison(base, next ?? base) + if (compared.length === 0) { + return [] + } + const bytes = (value: number | null, skipped?: string | null): string => + skipped + ? `skipped: ${skipped}` + : value === null + ? 'skipped: measurement missing' + : value.toFixed(0) + const lines = [ + '', + 'Main-process memory (bytes; median of the final stable 5-sample window). Each arm uses its own copy of one separate uniform seed. Conversion completes before the baseline. Every chat is opened through history and held subscribed until both active samples finish; no provider starts. The baseline can already include chats restored at startup, so compare absolute active bytes as well as activation deltas. These counts model the eight-hour completed-tool workload; this is not an eight-hour soak.', + '', + next + ? `| Counter | Phase | ${base.arm} bytes | ${next.arm} bytes | Reduction bytes | Reduction % |` + : `| Counter | Phase | ${base.arm} bytes |`, + next ? '| --- | --- | --- | --- | --- | --- |' : '| --- | --- | --- |' + ] + for (const row of compared) { + const cells = [row.operation, row.phase, bytes(row.mainBytes, row.mainSkipped)] + if (next) { + cells.push( + bytes(row.candidateBytes, row.candidateSkipped), + bytes(row.reductionBytes), + row.reductionPercent === null ? '—' : `${row.reductionPercent.toFixed(2)}%` + ) + } + lines.push(`| ${cells.join(' | ')} |`) + } + lines.push( + '', + 'RSS is process.memoryUsage().rss for the whole main process including workers, not macOS physical footprint. heapUsed/external/arrayBuffers cover the main V8 isolate. external includes arrayBuffers; do not sum them. Activation deltas subtract phase medians. GC runs only when main already exposes global.gc; unavailable or failed counters are explicitly skipped. Positive reduction means fewer candidate bytes. Seed time and logical file bytes (including the isolated provider home, not APFS allocated blocks) are recorded in each arm fixture and row Shape.' + ) + for (const result of next ? [base, next] : [base]) { + const row = result.rows.find((value) => value.family === 'memory') + if (row) { + lines.push('', `${result.arm} memory Shape: ${contextCell(row)}`) + } + } + return lines +} + export function comparisonMarkdown(base: ArmResult, next: ArmResult): string { const nextRows = new Map(next.rows.map((row) => [key(row), row])) const baseRows = new Map(base.rows.map((row) => [key(row), row])) @@ -131,6 +227,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string { ] const seen = new Set() for (const row of ordered) { + if (row.family === 'memory') { + continue + } if (seen.has(key(row))) { continue } @@ -151,6 +250,7 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string { '', WRITER_SPAN_DISCLOSURE ) + lines.push(...memoryMarkdown(base, next)) return `${lines.join('\n')}\n` } @@ -164,11 +264,15 @@ export function armMarkdown(result: ArmResult): string { '| --- | --- | --- | --- | --- | --- |' ] for (const row of result.rows) { + if (row.family === 'memory') { + continue + } const summary = summarizeRow(row) lines.push( `| ${row.family} | ${row.operation} | ${caseCell(row)} | ${row.skipped ? `skipped: ${row.skipped}` : cell(primary(summary))} | ${cell(summary.hostMs)} | ${contextCell(row)} |` ) } + lines.push(...memoryMarkdown(result)) return `${lines.join('\n')}\n` } @@ -197,7 +301,7 @@ export function writeComparison( } writeFileSync( path.join(out, `${name}.json`), - `${JSON.stringify({ workload, machine, arms: summaries }, null, 2)}\n` + `${JSON.stringify({ workload, machine, arms: summaries, memoryComparison: memoryComparison(base, next) }, null, 2)}\n` ) writeFileSync(path.join(out, `${name}.md`), comparisonMarkdown(base, next)) log(`wrote ${path.join(out, `${name}.md`)}`) diff --git a/config/scripts/performance/native-chat/native-chat-4a-run-context.ts b/config/scripts/performance/native-chat/native-chat-4a-run-context.ts index 999f11a2003..136957876f3 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-run-context.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-run-context.ts @@ -1,6 +1,9 @@ // What every 4a operation family receives, plus the one helper that turns a timed action into a row sample. +import { existsSync, renameSync } from 'node:fs' +import path from 'node:path' import type { Page } from '@stablyai/playwright-test' -import type { Arm, Launch } from './native-chat-4a-arm' +import { copyProfile, launchArm, stopArm, type Arm, type Launch } from './native-chat-4a-arm' +import { ConversionObserver } from './native-chat-4a-sqlite-observer' import type { Manifest } from './native-chat-4a-fixture' import { hostWorkMark, rendererNow } from './native-chat-4a-probes' import type { Row, Sample } from './native-chat-4a-report' @@ -64,3 +67,60 @@ export const resetTags = (): void => { } export const tagFor = (prefix: string): string => `${prefix}-${String((unique += 1)).padStart(5, '0')}` + +// Each seed copy's first launch converts on the candidate; the observer starts before it, reading only. +export function profileRunner(input: { + arm: Arm + out: string + stubCommand: string + workload: Workload + rows: Row[] + log: (line: string) => void +}) { + return async ( + seedDir: string, + manifest: Manifest, + phase: string, + body: (ctx: RunContext, observer: ConversionObserver) => Promise, + workload = input.workload + ): Promise => { + const { rows, log } = input + const profileDir = path.join(input.out, 'profile') + if (existsSync(profileDir)) { + renameSync(profileDir, path.join(input.out, `stale-profile-${Date.now()}`)) + } + copyProfile(seedDir, profileDir) + resetTags() + const observer = new ConversionObserver(profileDir) + const open = () => launchArm(input.arm, profileDir, input.stubCommand) + const ctx = { + arm: input.arm, + launch: await open().catch(async (error: unknown) => { + await observer.stop() + throw error + }), + workload, + manifest, + rows, + log, + relaunch: async () => undefined + } + ctx.relaunch = async () => { + await stopArm(ctx.launch) + ctx.launch = await open() + } + try { + await body(ctx, observer) + } finally { + await observer.stop() + await stopArm(ctx.launch) + renameSync( + profileDir, + path.join( + input.out, + `after-${input.arm.name}-${phase}-${manifest.toolOutputBytes}-${input.arm.sha.slice(0, 8)}` + ) + ) + } + } +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-session.ts b/config/scripts/performance/native-chat/native-chat-4a-session.ts index 3cc679cc5b0..fa69bcad867 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-session.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-session.ts @@ -6,7 +6,10 @@ import type { Page } from '@stablyai/playwright-test' import { operationId, type Arm } from './native-chat-4a-arm' import { rendererTime, rpc, visibleChatState } from './native-chat-4a-probes' -type HistoryItem = { itemId: string; body?: { kind?: string; role?: string; state?: string } } +type HistoryItem = { + itemId: string + body?: { kind?: string; role?: string; state?: string; output?: { byteLength?: number } } +} export type HistoryPage = { epoch: string fence?: number diff --git a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts index 2793f0d8d03..157efb2d2d5 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts @@ -29,6 +29,9 @@ export type Workload = { queuedEvery: number fillerChats: number fillerTurns: number + /** Only the separate uniform memory recipe overrides the small fillers. */ + fillerTools?: number + fillerToolOutputBytes?: number } /** Streams are long; `samples` per stream case, separate from the warm row count. */ stream: { cases: StreamCase[]; chunkBytes: number; intervalMs: number; samples: number } @@ -161,8 +164,8 @@ export function fillerTurn(chat: number, turn: number, workload: Workload): stri const op = { op: 'history', tag: `filler-${chat}-${turn}`, - tools: 2, - toolOutputBytes: 500, + tools: workload.history.fillerTools ?? 2, + toolOutputBytes: workload.history.fillerToolOutputBytes ?? 500, replyBytes: 400 } return providerText.formatDirective(op, workload.history.promptBytes)