Add active-chat resident memory comparison to 4a harness

config/scripts/performance/native-chat/native-chat-4a-comparison.ts

@@ -2,21 +2,15 @@
 // Dry run (no app): pnpm exec tsx config/scripts/performance/native-chat/native-chat-4a-comparison.ts --profile full --dry-run
 // Full run:        ... --arm main=<built main checkout>@<full sha> --arm candidate=<built candidate checkout>@<full sha> --profile full --out <dir> [--large-source]
 // Re-table:        ... --out <dir> --report-only [--arm main=@<full sha> --arm candidate=@<full sha>]
-import {
-  existsSync,
-  mkdirSync,
-  readdirSync,
-  readFileSync,
-  renameSync,
-  writeFileSync
-} from 'node:fs'
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
 import { parseArgs } from 'node:util'
 import type { Arm } from './native-chat-4a-arm'
 import { assertOneRun, parseArmSpec, seededBy } from './native-chat-4a-arm-provenance'
 import type { Manifest } from './native-chat-4a-fixture'
-import type { RunContext } from './native-chat-4a-run-context'
+import { profileRunner } from './native-chat-4a-run-context'
+import * as memory from './native-chat-4a-ops-memory'
 import { armMarkdown, writeComparison, type ArmResult, type Row } from './native-chat-4a-report'
 import {
   conversionToolOutputBytes,
@@ -34,7 +28,8 @@ const FAMILIES = [
   'records',
   'retained',
   'open',
-  'rewind'
+  'rewind',
+  'memory'
 ] as const
 type Family = (typeof FAMILIES)[number]
 
@@ -60,76 +55,39 @@ const workload = workloadFor(
 )
 const log = (line: string): void => console.error(`[4a ${new Date().toISOString()}] ${line}`)
 
+type SeededProfile = { seedDir: string; manifest: Manifest }
+
 const seededChatCount = (manifest: Manifest): number =>
   1 + Number(manifest.mediumSessionId !== null) + manifest.fillerSessionIds.length
 
 async function runArm(input: {
   arm: Arm
   out: string
-  seedDir: string
-  manifest: Manifest
+  profile?: SeededProfile
+  memoryProfile?: SeededProfile
+  memorySeedFailure?: string
   stubCommand: string
   families: Set<Family>
   largeSource?: { seedDir: string; manifest: Manifest }
 }): Promise<ArmResult> {
-  const { copyProfile, launchArm, stopArm } = await import('./native-chat-4a-arm')
-  const { resetTags } = await import('./native-chat-4a-run-context')
   const lifecycle = await import('./native-chat-4a-ops-lifecycle')
   const conversion = await import('./native-chat-4a-ops-conversion')
-  const { ConversionObserver } = await import('./native-chat-4a-sqlite-observer')
   const queue = await import('./native-chat-4a-ops-queue')
   const retained = await import('./native-chat-4a-ops-retained')
   const stream = await import('./native-chat-4a-ops-stream')
   const reads = await import('./native-chat-4a-ops-reads')
   const shapes = await import('./native-chat-4a-ops-shapes')
-  const profileDir = path.join(input.out, 'profile')
   const rows: Row[] = []
   const startedAt = new Date().toISOString()
 
-  // Each seed copy's first launch converts on the candidate; the observer starts before it, reading only.
-  const onProfile = async (
-    seedDir: string,
-    manifest: typeof input.manifest,
-    phase: string,
-    body: (ctx: RunContext, observer: InstanceType<typeof ConversionObserver>) => Promise<void>
-  ) => {
-    if (existsSync(profileDir)) {
-      renameSync(profileDir, path.join(input.out, `stale-profile-${Date.now()}`))
-    }
-    copyProfile(seedDir, profileDir)
-    resetTags()
-    const observer = new ConversionObserver(profileDir)
-    const open = () => launchArm(input.arm, profileDir, input.stubCommand)
-    const ctx = {
-      arm: input.arm,
-      launch: await open().catch(async (error: unknown) => {
-        await observer.stop()
-        throw error
-      }),
-      workload,
-      manifest,
-      rows,
-      log,
-      relaunch: async () => undefined
-    }
-    ctx.relaunch = async () => {
-      await stopArm(ctx.launch)
-      ctx.launch = await open()
-    }
-    try {
-      await body(ctx, observer)
-    } finally {
-      await observer.stop()
-      await stopArm(ctx.launch)
-      renameSync(
-        profileDir,
-        path.join(
-          input.out,
-          `after-${input.arm.name}-${phase}-${manifest.toolOutputBytes}-${input.arm.sha.slice(0, 8)}`
-        )
-      )
-    }
-  }
+  const onProfile = profileRunner({
+    arm: input.arm,
+    out: input.out,
+    stubCommand: input.stubCommand,
+    workload,
+    rows,
+    log
+  })
 
   const run = async (family: Family, body: () => Promise<void>) => {
     if (!input.families.has(family)) {
@@ -141,42 +99,62 @@ async function runArm(input: {
     } catch (error) {
       const reason =
         `family failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300)
-      rows.push({
-        family,
-        operation: '(family)',
-        variant: 'all',
-        samples: [],
-        context: {},
-        skipped: reason
-      })
+      if (family === 'memory') {
+        memory.skipMissingMemoryRows(rows, workload, reason)
+      } else {
+        rows.push({
+          family,
+          operation: '(family)',
+          variant: 'all',
+          samples: [],
+          context: {},
+          skipped: reason
+        })
+      }
       log(`${input.arm.name}: ${family} ${reason}`)
     }
   }
 
-  const label = `${workload.history.largeTurns}-turn/${seededChatCount(input.manifest)}-chat profile`
-  if (input.families.has('conversion')) {
-    // A copy of its own, left idle, so the sweep's whole duration and per-chat intervals carry no foreground work.
-    await onProfile(input.seedDir, input.manifest, 'sweep', (ctx, observer) =>
-      run('conversion', () => conversion.sweepRows(ctx, observer, label))
-    )
-  }
-  await onProfile(input.seedDir, input.manifest, 'families', async (ctx, observer) => {
-    const seedFiles = conversion.seededDbFiles(input.seedDir)
-    // First: the seeded tail markers and unconverted state exist only before any other family writes.
-    await run('conversion', () => conversion.firstLaunchRows(ctx, seedFiles, label, observer))
-    await conversion.awaitSweep(ctx, observer, label)
-    await run('send', () => queue.sendAndQueueRows(ctx))
-    await run('stream', () => stream.streamingRows(ctx))
-    await run('history', async () => {
-      await reads.historyRows(ctx)
-      await shapes.subagentRows(ctx)
-      await shapes.itemShapeRows(ctx)
+  const profile = input.profile
+  if (profile) {
+    const label = `${workload.history.largeTurns}-turn/${seededChatCount(profile.manifest)}-chat profile`
+    if (input.families.has('conversion')) {
+      // A copy of its own, left idle, so the sweep's whole duration and per-chat intervals carry no foreground work.
+      await onProfile(profile.seedDir, profile.manifest, 'sweep', (ctx, observer) =>
+        run('conversion', () => conversion.sweepRows(ctx, observer, label))
+      )
+    }
+    await onProfile(profile.seedDir, profile.manifest, 'families', async (ctx, observer) => {
+      const seedFiles = conversion.seededDbFiles(profile.seedDir)
+      // First: the seeded tail markers and unconverted state exist only before any other family writes.
+      await run('conversion', () => conversion.firstLaunchRows(ctx, seedFiles, label, observer))
+      await conversion.awaitSweep(ctx, observer, label)
+      await run('send', () => queue.sendAndQueueRows(ctx))
+      await run('stream', () => stream.streamingRows(ctx))
+      await run('history', async () => {
+        await reads.historyRows(ctx)
+        await shapes.subagentRows(ctx)
+        await shapes.itemShapeRows(ctx)
+      })
+      await run('outline', () => reads.outlineAndForkRows(ctx))
+      await run('records', () => reads.recordRows(ctx))
+      await run('retained', () => retained.retainedAdmissionRows(ctx))
+      await run('open', () => lifecycle.openRows(ctx))
+      await run('rewind', () => lifecycle.rewindRows(ctx))
     })
-    await run('outline', () => reads.outlineAndForkRows(ctx))
-    await run('records', () => reads.recordRows(ctx))
-    await run('retained', () => retained.retainedAdmissionRows(ctx))
-    await run('open', () => lifecycle.openRows(ctx))
-    await run('rewind', () => lifecycle.rewindRows(ctx))
+  }
+  await run('memory', async () => {
+    if (!input.memoryProfile) {
+      throw new Error(input.memorySeedFailure ?? 'memory seed unavailable')
+    }
+    const source = input.memoryProfile
+    await onProfile(
+      source.seedDir,
+      source.manifest,
+      'memory',
+      memory.convertedMemoryRows,
+      memory.memoryWorkload(workload)
+    )
   })
   if (input.largeSource && input.families.has('conversion')) {
     const source = input.largeSource
@@ -192,6 +170,7 @@ async function runArm(input: {
       )
     )
   }
+  const manifest = profile?.manifest ?? input.memoryProfile?.manifest
   return {
     arm: input.arm.name,
     sha: input.arm.sha,
@@ -200,22 +179,28 @@ async function runArm(input: {
     finishedAt: new Date().toISOString(),
     rows,
     fixture: {
-      repoId: input.manifest.workspace.repoId,
-      worktreeIds: input.manifest.workspace.worktreeIds.join(' '),
-      seedArm: input.manifest.seedArm,
-      seedSha: input.manifest.seedSha,
-      recipe: input.manifest.recipe,
+      repoId: manifest?.workspace.repoId ?? 'unavailable',
+      worktreeIds: manifest?.workspace.worktreeIds.join(' ') ?? 'unavailable',
+      seedArm: manifest?.seedArm ?? 'unavailable',
+      seedSha: manifest?.seedSha ?? 'unavailable',
+      recipe: manifest?.recipe ?? 'unavailable',
       largeSourceRecipe: input.largeSource?.manifest.recipe ?? 'none',
-      largeItems: input.manifest.largeItems,
-      storageBytes: input.manifest.storageBytes,
-      seedMs: input.manifest.seedMs
+      largeItems: manifest?.largeItems ?? 0,
+      storageBytes: manifest?.storageBytes ?? 0,
+      seedMs: manifest?.seedMs ?? 0,
+      memoryRecipe:
+        input.memoryProfile?.manifest.recipe ?? input.memorySeedFailure ?? 'not selected',
+      memorySeedMs: input.memoryProfile?.manifest.seedMs ?? 'unavailable',
+      memorySeedProfileLogicalBytes: input.memoryProfile?.manifest.profileBytes ?? 'unavailable'
     }
   }
 }
 
 async function main(): Promise<void> {
   if (values['dry-run']) {
-    console.log(JSON.stringify(dryRunWorkload(workload), null, 2))
+    console.log(
+      JSON.stringify({ ...dryRunWorkload(workload), memory: memory.memoryShape(workload) }, null, 2)
+    )
     return
   }
   const out = path.resolve(values.out ?? path.join(os.tmpdir(), `orca-4a-${workload.profile}`))
@@ -260,20 +245,31 @@ async function main(): Promise<void> {
     : createSeededTestRepo({ publishPath: false })
   writeFileSync(repoFile, repoPath)
   const seedArm = arms.find((arm) => arm.name === values['seed-arm']) ?? arms[0]!
-  const seed = (toolOutputBytes: number, seeded: Workload) =>
+  const seed = (toolOutputBytes: number, seeded: Workload, runDir = out) =>
     seedProfile({
       arm: seedArm,
       workload: seeded,
-      runDir: out,
+      runDir,
       profileDir: path.join(out, 'profile'),
       repoPath,
       stubCommand,
       toolOutputBytes,
       log
     })
-  const manifest = seededBy(seedArm, await seed(workload.history.toolOutputBytes, workload))
+  const manifest = [...families].some((family) => family !== 'memory')
+    ? seededBy(seedArm, await seed(workload.history.toolOutputBytes, workload))
+    : null
+  const { memoryProfile, memorySeedFailure } = families.has('memory')
+    ? await memory.seedMemoryProfile(
+        out,
+        workload,
+        async (seeded, runDir) =>
+          seededBy(seedArm, await seed(seeded.history.toolOutputBytes, seeded, runDir)),
+        log
+      )
+    : {}
   const large =
-    values['large-source'] && workload.conversionSourceBytes > 0
+    families.has('conversion') && values['large-source'] && workload.conversionSourceBytes > 0
       ? seededBy(
           seedArm,
           await seed(conversionToolOutputBytes(workload), {
@@ -287,8 +283,9 @@ async function main(): Promise<void> {
     const result = await runArm({
       arm,
       out,
-      seedDir: seedDirFor(out, manifest),
-      manifest,
+      ...(manifest ? { profile: { seedDir: seedDirFor(out, manifest), manifest } } : {}),
+      memoryProfile,
+      memorySeedFailure,
       stubCommand,
       families,
       ...(large ? { largeSource: { seedDir: seedDirFor(out, large), manifest: large } } : {})

config/scripts/performance/native-chat/native-chat-4a-fixture.ts

@@ -32,19 +32,42 @@ export type Manifest = {
   largeItems: number
   mediumItems: number
   storageBytes: number
+  /** Logical file bytes including the isolated provider home; absent in older cached seeds. */
+  profileBytes?: number
   seedMs: number
 }
 
 const SLEEP_FILLERS_EVERY = 20
 
 /** Count the large chat's rows by paging back from the tail; refuses a fixture that seeded fewer rows than its recipe. */
-async function countItems(page: Page, sessionId: string): Promise<number> {
+async function countItems(
+  page: Page,
+  sessionId: string,
+  expectedTools?: { count: number; bytes: number }
+): Promise<number> {
   let count = 0
+  let tools = 0
   let cursor: unknown
   for (;;) {
     const read = await history(page, sessionId, cursor ? 'before' : 'tail', 200, cursor)
     count += read.page.items.length
+    if (expectedTools) {
+      for (const { body } of read.page.items) {
+        if (body?.kind !== 'tool-call') {
+          continue
+        }
+        if (body.state !== 'completed' || body.output?.byteLength !== expectedTools.bytes) {
+          throw new Error(`seed: ${sessionId} has an incomplete tool or wrong output size`)
+        }
+        tools += 1
+      }
+    }
     if (!read.page.hasOlder) {
+      if (expectedTools && tools !== expectedTools.count) {
+        throw new Error(
+          `seed: ${sessionId} has ${tools} completed tools, expected ${expectedTools.count}`
+        )
+      }
       return count
     }
     cursor = read.page.window.nextCursor
@@ -92,7 +115,16 @@ async function seedRecipeChat(
   }
   await waitForDurable(page, created.sessionId, `4A-TAIL-${chat}-${turn - 1}`, 600_000)
   await waitForIdle(page, created.sessionId, 600_000)
-  const items = await countItems(page, created.sessionId)
+  const items = await countItems(
+    page,
+    created.sessionId,
+    workload.history.fillerTools === undefined
+      ? undefined
+      : {
+          count: turns * workload.history.tools,
+          bytes: workload.history.toolOutputBytes
+        }
+  )
   // Hosts may group a call with its result; fewer than prompt+reply per turn means turns were lost.
   if (items < turns * 2) {
     throw new Error(`seed: ${chat} chat holds ${items} rows for ${turns} turns`)
@@ -175,6 +207,13 @@ export async function seedProfile(input: {
         await waitForDurable(page, filler.sessionId, `4A-TAIL-filler-${chat}-${fillerTurnIndex}`)
       }
       fillerSessionIds.push(filler.sessionId)
+      if (input.workload.history.fillerTools !== undefined) {
+        await waitForIdle(page, filler.sessionId, 600_000)
+        await countItems(page, filler.sessionId, {
+          count: input.workload.history.fillerTurns * input.workload.history.fillerTools,
+          bytes: input.workload.history.fillerToolOutputBytes ?? 500
+        })
+      }
       // Bound live provider processes: retained chats stay, their stubs stop.
       if ((chat + 1) % SLEEP_FILLERS_EVERY === 0) {
         await rpc(page, 'agentSession.sleepWorkspace', { workspaceId })
@@ -210,6 +249,7 @@ export async function seedProfile(input: {
       largeItems: large.items,
       mediumItems: medium?.items ?? 0,
       storageBytes: storageBytes(input.profileDir).total,
+      profileBytes: storageBytes(input.profileDir, true).total,
       seedMs: Date.now() - started
     }
     renameSync(input.profileDir, seedDir)

config/scripts/performance/native-chat/native-chat-4a-ops-memory.ts

@@ -0,0 +1,306 @@
+// Main-process residency on an independent, uniform completed-tool seed; subscriptions retain no payloads in the driver.
+import { existsSync, mkdirSync, renameSync } from 'node:fs'
+import path from 'node:path'
+import { setTimeout as delay } from 'node:timers/promises'
+import { seedDirFor, seededChats, type Manifest } from './native-chat-4a-fixture'
+import type { Row } from './native-chat-4a-report'
+import { stat } from './native-chat-4a-report'
+import { page, type RunContext } from './native-chat-4a-run-context'
+import { history } from './native-chat-4a-session'
+import type { ConversionObserver } from './native-chat-4a-sqlite-observer'
+import type { Workload } from './native-chat-4a-workloads'
+
+const METRICS = ['heapUsed', 'rss', 'external', 'arrayBuffers'] as const
+const PHASES = [
+  'before activation',
+  'after activation',
+  'activation delta',
+  'before activation post-GC',
+  'after activation post-GC',
+  'activation delta post-GC'
+] as const
+type Phase = (typeof PHASES)[number]
+type Memory = Partial<Record<(typeof METRICS)[number], number>>
+const WINDOW = 5
+const INTERVAL_MS = 1000
+const SETTLE_TIMEOUT_MS = 60_000
+
+/** Kept outside the established history workload, including its cache directory and recipe digest. */
+export function memoryWorkload(workload: Workload): Workload {
+  const tools = workload.profile === 'full' ? 480 : 48
+  return {
+    ...workload,
+    history: {
+      largeTurns: 1,
+      mediumTurns: 0,
+      tools,
+      toolOutputBytes: 32 * 1024,
+      promptBytes: 300,
+      replyBytes: 400,
+      queuedEvery: 0,
+      fillerChats: (workload.profile === 'full' ? 500 : 20) - 1,
+      fillerTurns: 1,
+      fillerTools: tools,
+      fillerToolOutputBytes: 32 * 1024
+    },
+    diffs: { count: 0, patchBytes: 0 }
+  }
+}
+
+export function memoryShape(workload: Workload): Record<string, number | string> {
+  const { history: h } = memoryWorkload(workload)
+  return {
+    chats: h.fillerChats + 1,
+    completedToolsPerChat: h.tools,
+    toolOutputBytes: h.toolOutputBytes,
+    outputPayloadBytes: (h.fillerChats + 1) * h.tools * h.toolOutputBytes,
+    activation: 'agentSession.history then held agentSession.subscribe per chat; no provider start',
+    source: 'Electron main process.memoryUsage(); RSS is resident bytes, not physical footprint',
+    settling: 'last 5 samples, 1s apart, heapUsed/RSS range <= max(5% of minimum, 8MiB); 60s limit'
+  }
+}
+
+/** Even a launch or seed failure produces a named skipped row for every requested counter and phase. */
+export function skipMissingMemoryRows(rows: Row[], workload: Workload, reason: string): void {
+  for (const phase of PHASES) {
+    for (const metric of METRICS) {
+      const existing = rows.find(
+        (row) => row.family === 'memory' && row.operation === metric && row.variant === phase
+      )
+      if (existing && existing.samples.length === 0 && !existing.skipped) {
+        existing.skipped = reason
+      }
+      if (!existing) {
+        rows.push({
+          family: 'memory',
+          operation: metric,
+          variant: phase,
+          samples: [],
+          context: memoryShape(workload),
+          skipped: reason
+        })
+      }
+    }
+  }
+}
+
+export async function memoryRows(ctx: RunContext): Promise<void> {
+  const context = {
+    ...memoryShape(ctx.workload),
+    seedRecipe: ctx.manifest.recipe,
+    seedArm: ctx.manifest.seedArm,
+    seedSha: ctx.manifest.seedSha,
+    seedMs: ctx.manifest.seedMs,
+    seedStorageBytes: ctx.manifest.storageBytes,
+    seedProfileLogicalBytes: ctx.manifest.profileBytes ?? 'not recorded in cached seed',
+    mainPid: ctx.launch.pid
+  }
+  const rows = new Map<Phase, Row[]>()
+  for (const phase of PHASES) {
+    const group = METRICS.map((metric): Row => ({
+      family: 'memory',
+      operation: metric,
+      variant: phase,
+      samples: [],
+      context
+    }))
+    ctx.rows.push(...group)
+    rows.set(phase, group)
+  }
+  const groupFor = (phase: Phase): Row[] => rows.get(phase)!
+  const skip = (phase: Phase, reason: string): void => {
+    for (const row of groupFor(phase)) {
+      row.skipped = reason
+    }
+  }
+  const snapshot = (): Promise<Memory> =>
+    ctx.launch.app.evaluate(() => {
+      const { heapUsed, rss, external, arrayBuffers } = process.memoryUsage()
+      return { heapUsed, rss, external, arrayBuffers }
+    })
+  const measure = async (phase: Phase): Promise<void> => {
+    const started = Date.now()
+    const samples: Memory[] = []
+    let stable = false
+    do {
+      if (samples.length > 0) {
+        await delay(INTERVAL_MS)
+      }
+      samples.push(await snapshot())
+      if (samples.length > WINDOW) {
+        samples.shift()
+      }
+      stable =
+        samples.length === WINDOW &&
+        (['heapUsed', 'rss'] as const).every((metric) => {
+          const values = samples.map((sample) => sample[metric])
+          if (values.some((value) => value === undefined || !Number.isFinite(value))) {
+            return false
+          }
+          const present = values.filter((value): value is number => typeof value === 'number')
+          const min = Math.min(...present)
+          return Math.max(...present) - min <= Math.max(min * 0.05, 8 * 1024 ** 2)
+        })
+    } while (!stable && Date.now() - started < SETTLE_TIMEOUT_MS)
+    for (const row of groupFor(phase)) {
+      row.context = { ...context, settleMs: Date.now() - started }
+      for (const sample of samples) {
+        const value = METRICS.find((metric) => metric === row.operation)
+        const bytes = value ? sample[value] : undefined
+        if (typeof bytes === 'number' && Number.isFinite(bytes)) {
+          row.samples.push({ extra: { bytes } })
+        }
+      }
+      if (!stable) {
+        row.skipped = 'heapUsed/RSS did not settle within 60s; diagnostic samples only'
+      } else if (row.samples.length !== WINDOW) {
+        row.skipped = 'counter unavailable in process.memoryUsage()'
+      }
+    }
+    if (!stable) {
+      throw new Error(`${phase}: heapUsed/RSS did not settle within 60s`)
+    }
+  }
+  const afterGc = async (phase: Phase): Promise<void> => {
+    try {
+      const collected = await ctx.launch.app.evaluate(() => {
+        if (typeof global.gc !== 'function') {
+          return false
+        }
+        global.gc()
+        return true
+      })
+      if (!collected) {
+        skip(phase, 'main global.gc unavailable; no product hook or launch flag added')
+        return
+      }
+      await measure(phase)
+    } catch (error) {
+      skip(phase, `GC measurement failed: ${String(error).slice(0, 200)}`)
+    }
+  }
+  const delta = (before: Phase, after: Phase, phase: Phase): void => {
+    for (const row of groupFor(phase)) {
+      const left = groupFor(before).find((value) => value.operation === row.operation)!
+      const right = groupFor(after).find((value) => value.operation === row.operation)!
+      const a = stat(left.samples.map((sample) => sample.extra?.bytes))
+      const b = stat(right.samples.map((sample) => sample.extra?.bytes))
+      if (left.skipped || right.skipped || !a || !b) {
+        row.skipped = left.skipped ?? right.skipped ?? 'baseline or active counter unavailable'
+      } else {
+        row.samples.push({ extra: { bytes: b.p50 - a.p50 } })
+      }
+    }
+  }
+  const subscriptions = await page(ctx).evaluateHandle(() => {
+    const handles: { unsubscribe: () => void }[] = []
+    const errors: string[] = []
+    return { handles, errors }
+  })
+  try {
+    await measure('before activation')
+    await afterGc('before activation post-GC')
+    const chats = seededChats(ctx.manifest)
+    if (chats.length !== ctx.workload.history.fillerChats + 1) {
+      throw new Error('memory seed chat count mismatch')
+    }
+    for (const [index, sessionId] of chats.entries()) {
+      await history(page(ctx), sessionId, 'tail', 1)
+      await subscriptions.evaluate(async (state, sessionId) => {
+        const handle = await window.api.runtime.subscribe(
+          { method: 'agentSession.subscribe', params: { sessionId } },
+          (reply) => {
+            if (!reply.ok && state.errors.length < 10) {
+              state.errors.push(`${sessionId}: ${JSON.stringify(reply.error)}`)
+            }
+            if (
+              reply.ok &&
+              typeof reply.result === 'object' &&
+              reply.result !== null &&
+              'type' in reply.result &&
+              reply.result.type === 'end' &&
+              state.errors.length < 10
+            ) {
+              state.errors.push(`${sessionId}: subscription ended before sampling finished`)
+            }
+          }
+        )
+        state.handles.push(handle)
+      }, sessionId)
+      if ((index + 1) % 20 === 0) {
+        ctx.log(`${ctx.arm.name}: memory ${index + 1}/${chats.length} subscribed`)
+      }
+    }
+    const checkSubscriptions = async (): Promise<void> => {
+      const errors = await subscriptions.evaluate((state) => state.errors)
+      if (errors.length > 0) {
+        throw new Error(`memory subscription failed: ${errors.join('; ').slice(0, 300)}`)
+      }
+    }
+    await checkSubscriptions()
+    await measure('after activation')
+    await afterGc('after activation post-GC')
+    await checkSubscriptions()
+  } catch (error) {
+    const reason = `memory measurement failed: ${String(error).slice(0, 250)}`
+    for (const phase of PHASES) {
+      if (
+        phase.startsWith('after') ||
+        groupFor(phase).every((row) => row.samples.length === 0 && !row.skipped)
+      ) {
+        skip(phase, reason)
+      }
+    }
+  } finally {
+    await subscriptions
+      .evaluate((state) => {
+        for (const handle of state.handles) {
+          handle.unsubscribe()
+        }
+        state.handles.length = 0
+      })
+      .finally(() => subscriptions.dispose())
+    delta('before activation', 'after activation', 'activation delta')
+    delta('before activation post-GC', 'after activation post-GC', 'activation delta post-GC')
+  }
+}
+
+export async function seedMemoryProfile(
+  out: string,
+  workload: Workload,
+  seed: (workload: Workload, runDir: string) => Promise<Manifest>,
+  log: (line: string) => void
+): Promise<{
+  memoryProfile?: { seedDir: string; manifest: Manifest }
+  memorySeedFailure?: string
+}> {
+  const runDir = path.join(out, 'memory')
+  mkdirSync(runDir, { recursive: true })
+  try {
+    const manifest = await seed(memoryWorkload(workload), runDir)
+    return { memoryProfile: { seedDir: seedDirFor(runDir, manifest), manifest } }
+  } catch (error) {
+    const memorySeedFailure = `memory seed failed: ${String(error).slice(0, 250)}`
+    log(memorySeedFailure)
+    const profile = path.join(out, 'profile')
+    if (existsSync(profile)) {
+      renameSync(profile, path.join(out, `failed-memory-seed-${Date.now()}`))
+    }
+    return { memorySeedFailure }
+  }
+}
+
+export async function convertedMemoryRows(
+  ctx: RunContext,
+  observer: ConversionObserver
+): Promise<void> {
+  // Conversion completes before the baseline on this otherwise untouched copy.
+  if (
+    observer.databaseFile &&
+    !(await observer.allReady(seededChats(ctx.manifest), 2 * 60 * 60_000))
+  ) {
+    throw new Error('memory profile conversion did not finish within 2h')
+  }
+  await observer.stop()
+  await memoryRows(ctx)
+}

config/scripts/performance/native-chat/native-chat-4a-probes.ts

@@ -204,7 +204,10 @@ export async function hostWorkMark(app: ElectronApplication): Promise<() => Prom
 }
 
 /** All SQLite/journal files under the profile, so WAL growth and file growth read the same on both storage designs. */
-export function storageBytes(userDataDir: string): { db: number; wal: number; total: number } {
+export function storageBytes(
+  userDataDir: string,
+  includeHome = false
+): { db: number; wal: number; total: number } {
   let db = 0
   let wal = 0
   let total = 0
@@ -212,7 +215,7 @@ export function storageBytes(userDataDir: string): { db: number; wal: number; to
     for (const entry of readdirSync(dir, { withFileTypes: true })) {
       const file = path.join(dir, entry.name)
       if (entry.isDirectory()) {
-        if (entry.name !== 'home') {
+        if (includeHome || entry.name !== 'home') {
           visit(file)
         }
         continue

config/scripts/performance/native-chat/native-chat-4a-report.ts

@@ -116,6 +116,102 @@ function contextCell(row: Row | undefined): string {
     .join(', ')
 }
 
+function memoryValue(row: Row | undefined): number | null {
+  return row?.skipped
+    ? null
+    : (stat(row?.samples.map((sample) => sample.extra?.bytes) ?? [])?.p50 ?? null)
+}
+
+/** Positive reduction means the candidate uses fewer bytes; negative activation deltas have no relative denominator. */
+export function memoryComparison(base: ArmResult, next: ArmResult) {
+  const left = new Map(
+    base.rows.filter((row) => row.family === 'memory').map((row) => [key(row), row])
+  )
+  const right = new Map(
+    next.rows.filter((row) => row.family === 'memory').map((row) => [key(row), row])
+  )
+  return [...new Set([...left.keys(), ...right.keys()])].map((id) => {
+    const a = left.get(id)
+    const b = right.get(id)
+    if (a && b && !a.skipped && !b.skipped) {
+      for (const field of [
+        'seedRecipe',
+        'seedSha',
+        'chats',
+        'completedToolsPerChat',
+        'toolOutputBytes'
+      ]) {
+        if (a.context[field] === undefined || a.context[field] !== b.context[field]) {
+          throw new Error(
+            `memory comparison: arms disagree on or lack ${field}; use one identical seed`
+          )
+        }
+      }
+    }
+    const mainBytes = memoryValue(a)
+    const candidateBytes = memoryValue(b)
+    const reductionBytes =
+      mainBytes === null || candidateBytes === null ? null : mainBytes - candidateBytes
+    return {
+      operation: (a ?? b)!.operation,
+      phase: (a ?? b)!.variant,
+      mainBytes,
+      candidateBytes,
+      reductionBytes,
+      reductionPercent:
+        reductionBytes !== null && mainBytes !== null && mainBytes > 0
+          ? (reductionBytes / mainBytes) * 100
+          : null,
+      mainSkipped: a?.skipped ?? (mainBytes === null ? 'measurement missing' : null),
+      candidateSkipped: b?.skipped ?? (candidateBytes === null ? 'measurement missing' : null)
+    }
+  })
+}
+
+function memoryMarkdown(base: ArmResult, next?: ArmResult): string[] {
+  const compared = memoryComparison(base, next ?? base)
+  if (compared.length === 0) {
+    return []
+  }
+  const bytes = (value: number | null, skipped?: string | null): string =>
+    skipped
+      ? `skipped: ${skipped}`
+      : value === null
+        ? 'skipped: measurement missing'
+        : value.toFixed(0)
+  const lines = [
+    '',
+    'Main-process memory (bytes; median of the final stable 5-sample window). Each arm uses its own copy of one separate uniform seed. Conversion completes before the baseline. Every chat is opened through history and held subscribed until both active samples finish; no provider starts. The baseline can already include chats restored at startup, so compare absolute active bytes as well as activation deltas. These counts model the eight-hour completed-tool workload; this is not an eight-hour soak.',
+    '',
+    next
+      ? `| Counter | Phase | ${base.arm} bytes | ${next.arm} bytes | Reduction bytes | Reduction % |`
+      : `| Counter | Phase | ${base.arm} bytes |`,
+    next ? '| --- | --- | --- | --- | --- | --- |' : '| --- | --- | --- |'
+  ]
+  for (const row of compared) {
+    const cells = [row.operation, row.phase, bytes(row.mainBytes, row.mainSkipped)]
+    if (next) {
+      cells.push(
+        bytes(row.candidateBytes, row.candidateSkipped),
+        bytes(row.reductionBytes),
+        row.reductionPercent === null ? '—' : `${row.reductionPercent.toFixed(2)}%`
+      )
+    }
+    lines.push(`| ${cells.join(' | ')} |`)
+  }
+  lines.push(
+    '',
+    'RSS is process.memoryUsage().rss for the whole main process including workers, not macOS physical footprint. heapUsed/external/arrayBuffers cover the main V8 isolate. external includes arrayBuffers; do not sum them. Activation deltas subtract phase medians. GC runs only when main already exposes global.gc; unavailable or failed counters are explicitly skipped. Positive reduction means fewer candidate bytes. Seed time and logical file bytes (including the isolated provider home, not APFS allocated blocks) are recorded in each arm fixture and row Shape.'
+  )
+  for (const result of next ? [base, next] : [base]) {
+    const row = result.rows.find((value) => value.family === 'memory')
+    if (row) {
+      lines.push('', `${result.arm} memory Shape: ${contextCell(row)}`)
+    }
+  }
+  return lines
+}
+
 export function comparisonMarkdown(base: ArmResult, next: ArmResult): string {
   const nextRows = new Map(next.rows.map((row) => [key(row), row]))
   const baseRows = new Map(base.rows.map((row) => [key(row), row]))
@@ -131,6 +227,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string {
   ]
   const seen = new Set<string>()
   for (const row of ordered) {
+    if (row.family === 'memory') {
+      continue
+    }
     if (seen.has(key(row))) {
       continue
     }
@@ -151,6 +250,7 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string {
     '',
     WRITER_SPAN_DISCLOSURE
   )
+  lines.push(...memoryMarkdown(base, next))
   return `${lines.join('\n')}\n`
 }
 
@@ -164,11 +264,15 @@ export function armMarkdown(result: ArmResult): string {
     '| --- | --- | --- | --- | --- | --- |'
   ]
   for (const row of result.rows) {
+    if (row.family === 'memory') {
+      continue
+    }
     const summary = summarizeRow(row)
     lines.push(
       `| ${row.family} | ${row.operation} | ${caseCell(row)} | ${row.skipped ? `skipped: ${row.skipped}` : cell(primary(summary))} | ${cell(summary.hostMs)} | ${contextCell(row)} |`
     )
   }
+  lines.push(...memoryMarkdown(result))
   return `${lines.join('\n')}\n`
 }
 
@@ -197,7 +301,7 @@ export function writeComparison(
   }
   writeFileSync(
     path.join(out, `${name}.json`),
-    `${JSON.stringify({ workload, machine, arms: summaries }, null, 2)}\n`
+    `${JSON.stringify({ workload, machine, arms: summaries, memoryComparison: memoryComparison(base, next) }, null, 2)}\n`
   )
   writeFileSync(path.join(out, `${name}.md`), comparisonMarkdown(base, next))
   log(`wrote ${path.join(out, `${name}.md`)}`)

config/scripts/performance/native-chat/native-chat-4a-run-context.ts

@@ -1,6 +1,9 @@
 // What every 4a operation family receives, plus the one helper that turns a timed action into a row sample.
+import { existsSync, renameSync } from 'node:fs'
+import path from 'node:path'
 import type { Page } from '@stablyai/playwright-test'
-import type { Arm, Launch } from './native-chat-4a-arm'
+import { copyProfile, launchArm, stopArm, type Arm, type Launch } from './native-chat-4a-arm'
+import { ConversionObserver } from './native-chat-4a-sqlite-observer'
 import type { Manifest } from './native-chat-4a-fixture'
 import { hostWorkMark, rendererNow } from './native-chat-4a-probes'
 import type { Row, Sample } from './native-chat-4a-report'
@@ -64,3 +67,60 @@ export const resetTags = (): void => {
 }
 export const tagFor = (prefix: string): string =>
   `${prefix}-${String((unique += 1)).padStart(5, '0')}`
+
+// Each seed copy's first launch converts on the candidate; the observer starts before it, reading only.
+export function profileRunner(input: {
+  arm: Arm
+  out: string
+  stubCommand: string
+  workload: Workload
+  rows: Row[]
+  log: (line: string) => void
+}) {
+  return async (
+    seedDir: string,
+    manifest: Manifest,
+    phase: string,
+    body: (ctx: RunContext, observer: ConversionObserver) => Promise<void>,
+    workload = input.workload
+  ): Promise<void> => {
+    const { rows, log } = input
+    const profileDir = path.join(input.out, 'profile')
+    if (existsSync(profileDir)) {
+      renameSync(profileDir, path.join(input.out, `stale-profile-${Date.now()}`))
+    }
+    copyProfile(seedDir, profileDir)
+    resetTags()
+    const observer = new ConversionObserver(profileDir)
+    const open = () => launchArm(input.arm, profileDir, input.stubCommand)
+    const ctx = {
+      arm: input.arm,
+      launch: await open().catch(async (error: unknown) => {
+        await observer.stop()
+        throw error
+      }),
+      workload,
+      manifest,
+      rows,
+      log,
+      relaunch: async () => undefined
+    }
+    ctx.relaunch = async () => {
+      await stopArm(ctx.launch)
+      ctx.launch = await open()
+    }
+    try {
+      await body(ctx, observer)
+    } finally {
+      await observer.stop()
+      await stopArm(ctx.launch)
+      renameSync(
+        profileDir,
+        path.join(
+          input.out,
+          `after-${input.arm.name}-${phase}-${manifest.toolOutputBytes}-${input.arm.sha.slice(0, 8)}`
+        )
+      )
+    }
+  }
+}

config/scripts/performance/native-chat/native-chat-4a-session.ts

@@ -6,7 +6,10 @@ import type { Page } from '@stablyai/playwright-test'
 import { operationId, type Arm } from './native-chat-4a-arm'
 import { rendererTime, rpc, visibleChatState } from './native-chat-4a-probes'
 
-type HistoryItem = { itemId: string; body?: { kind?: string; role?: string; state?: string } }
+type HistoryItem = {
+  itemId: string
+  body?: { kind?: string; role?: string; state?: string; output?: { byteLength?: number } }
+}
 export type HistoryPage = {
   epoch: string
   fence?: number

config/scripts/performance/native-chat/native-chat-4a-workloads.ts

@@ -29,6 +29,9 @@ export type Workload = {
     queuedEvery: number
     fillerChats: number
     fillerTurns: number
+    /** Only the separate uniform memory recipe overrides the small fillers. */
+    fillerTools?: number
+    fillerToolOutputBytes?: number
   }
   /** Streams are long; `samples` per stream case, separate from the warm row count. */
   stream: { cases: StreamCase[]; chunkBytes: number; intervalMs: number; samples: number }
@@ -161,8 +164,8 @@ export function fillerTurn(chat: number, turn: number, workload: Workload): stri
   const op = {
     op: 'history',
     tag: `filler-${chat}-${turn}`,
-    tools: 2,
-    toolOutputBytes: 500,
+    tools: workload.history.fillerTools ?? 2,
+    toolOutputBytes: workload.history.fillerToolOutputBytes ?? 500,
     replyBytes: 400
   }
   return providerText.formatDirective(op, workload.history.promptBytes)