From d2235aa57caa171d7f0bf5414dbea295acd480fd Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:37:54 -0800 Subject: [PATCH 01/15] Add the 4a main-vs-candidate native chat comparison harness One entry (native-chat-4a-comparison.ts) seeds the 55k-history/310-chat profile once through the seed arm, clones it per arm at a fixed path, launches each built checkout in the background with an isolated home, and drives the same stub provider events through the host RPC surface both builds serve. Envelopes come from each arm's own source bundle. Rows cover send/queue, 1/8/16 MiB streams with a competing send, maintenance windows, open/reopen, history/detail, outline/fork, rewind, record operations, R3 retained-history admission and, reported separately, first-launch conversion (optional 20 GiB source). Output is one JSON and one markdown table keyed by both head SHAs. --- .../native-chat/native-chat-4a-arm.ts | 244 ++++++++++++++ .../native-chat/native-chat-4a-comparison.ts | 314 ++++++++++++++++++ .../native-chat/native-chat-4a-fixture.ts | 151 +++++++++ .../native-chat-4a-ops-lifecycle.ts | 313 +++++++++++++++++ .../native-chat/native-chat-4a-ops-queue.ts | 245 ++++++++++++++ .../native-chat/native-chat-4a-ops-reads.ts | 238 +++++++++++++ .../native-chat-4a-ops-retained.ts | 102 ++++++ .../native-chat/native-chat-4a-ops-stream.ts | 210 ++++++++++++ .../native-chat/native-chat-4a-probes.ts | 231 +++++++++++++ .../native-chat-4a-provider-stub.cjs | 310 +++++++++++++++++ .../native-chat-4a-provider-text.cjs | 78 +++++ .../native-chat/native-chat-4a-report.ts | 152 +++++++++ .../native-chat/native-chat-4a-run-context.ts | 66 ++++ .../native-chat/native-chat-4a-session.ts | 228 +++++++++++++ .../native-chat/native-chat-4a-workloads.ts | 216 ++++++++++++ 15 files changed, 3098 insertions(+) create mode 100644 config/scripts/performance/native-chat/native-chat-4a-arm.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-comparison.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-fixture.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-queue.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-retained.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-probes.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs create mode 100644 config/scripts/performance/native-chat/native-chat-4a-provider-text.cjs create mode 100644 config/scripts/performance/native-chat/native-chat-4a-report.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-run-context.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-session.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-workloads.ts diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm.ts b/config/scripts/performance/native-chat/native-chat-4a-arm.ts new file mode 100644 index 00000000000..c8381635a8c --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-arm.ts @@ -0,0 +1,244 @@ +// One 4a arm: a built checkout, its own envelope/fingerprint builders, and background launches it alone stops by recorded PID. +// Declares the window.__store and window.api globals these page.evaluate callbacks use. +import type {} from '../../../../tests/e2e/helpers/runtime-types' +import { execFileSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { chmodSync, constants, cpSync, existsSync, mkdirSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import path from 'node:path' +import { build } from 'esbuild' +import { + _electron as electron, + type ElectronApplication, + type Page +} from '@stablyai/playwright-test' +import { + createElectronHomeIsolation, + assertElectronResolvedIsolatedHome +} from '../../../../tests/e2e/helpers/electron-home-isolation' +import { getOrcaElectronLaunchArgs } from '../../../../tests/e2e/helpers/electron-launch-args' +import { closeElectronAppForE2E } from '../../../../tests/e2e/helpers/electron-process-shutdown' +import { getE2ECompletedOnboardingProfile } from '../../../../tests/e2e/helpers/e2e-completed-onboarding-profile' +import { buildFakeAgentCommandOverride } from '../../../../tests/e2e/helpers/fake-agent-command-override' + +const HARNESS_ROOT = path.resolve(import.meta.dirname, '../../../..') +const SYSTEM_DIRS = ['/usr/bin', '/bin', '/usr/sbin', '/sbin'] + +/** Envelope builders compiled from the arm's own source, so each host checks fingerprints it computes itself. */ +export type ArmProtocol = { + structuredAgentSessionPayloadFingerprint: (input: { + method: string + sessionId: string + fields: Record + }) => string + structuredAgentSessionCreateFingerprint: (input: { + sessionId: string + worktree: string + agent: string + forkFrom?: unknown + }) => string + createStructuredAgentSessionOperationId: (randomUuid: () => string) => string + createStructuredAgentSessionId: (agent: string, randomUuid: () => string) => string + structuredAgentSessionMessageSendMutation: (message: { + sessionId: string + clientOperationId: string + expectedRuntimeFence: number + body: { kind: 'message'; role: 'user'; blocks: { type: 'text'; text: string }[] } + delivery?: 'queue-if-active' + }) => { envelope: unknown; body: unknown; delivery?: 'queue-if-active' } + agentSessionSendBodyFingerprint: (sessionId: string, body: unknown) => string +} + +export type Arm = { + name: string + root: string + sha: string + electronPath: string + protocol: ArmProtocol +} + +export type Launch = { + app: ElectronApplication + page: Page + pid: number + /** `ps` start time recorded at launch; stop refuses a PID whose start time changed. */ + startedAt: string + userDataDir: string + launchedAtEpochMs: number + /** When the renderer reported its workspace session ready: launch-to-ready is metadata initialization. */ + readyEpochMs: number +} + +const git = (root: string, ...args: string[]): string => + execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim() + +export async function resolveArm(name: string, root: string, scratch: string): Promise { + const absolute = path.resolve(root) + if (!existsSync(path.join(absolute, 'out', 'main', 'index.js'))) { + throw new Error(`${name}: ${absolute} has no out/main/index.js; build that checkout first`) + } + const electronPath = createRequire(path.join(absolute, 'package.json'))('electron') as string + const outfile = path.join(scratch, `${name}-protocol.cjs`) + const shared = './src/shared/agent-session' + await build({ + stdin: { + contents: [ + `export * from '${shared}/lifecycle/structured-agent-session-mutation'`, + `export { createStructuredAgentSessionId } from '${shared}/structured-agent-session-create'`, + `export { structuredAgentSessionMessageSendMutation, agentSessionSendBodyFingerprint } from '${shared}/requests/structured-agent-session-send-mutation'` + ].join(';'), + resolveDir: absolute + }, + bundle: true, + platform: 'node', + format: 'cjs', + outfile, + logLevel: 'silent', + nodePaths: [path.join(HARNESS_ROOT, 'node_modules')] + }) + const protocol = createRequire(import.meta.url)(outfile) as ArmProtocol + return { name, root: absolute, sha: git(absolute, 'rev-parse', 'HEAD'), electronPath, protocol } +} + +/** The stub wrapper and its directory; the launch PATH holds only it and the system tool dirs. */ +export function writeProviderStub(dir: string): string { + mkdirSync(dir, { recursive: true }) + const command = path.join(dir, 'claude') + const quote = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'` + const stub = path.join(import.meta.dirname, 'native-chat-4a-provider-stub.cjs') + writeFileSync( + command, + `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(stub)} ${quote(path.join(dir, 'launches.jsonl'))} "$@"\n` + ) + chmodSync(command, 0o755) + return command +} + +function processStart(pid: number): string { + return execFileSync('ps', ['-o', 'lstart=', '-p', String(pid)], { encoding: 'utf8' }).trim() +} + +export async function launchArm( + arm: Arm, + userDataDir: string, + stubCommand: string +): Promise { + mkdirSync(userDataDir, { recursive: true }) + const profile = path.join(userDataDir, 'orca-data.json') + if (!existsSync(profile)) { + writeFileSync(profile, `${JSON.stringify(getE2ECompletedOnboardingProfile(), null, 2)}\n`) + } + const { ELECTRON_RUN_AS_NODE: _unused, ...inherited } = process.env + void _unused + const isolation = createElectronHomeIsolation({ + inheritedEnv: inherited, + launchEnv: { PATH: [path.dirname(stubCommand), ...SYSTEM_DIRS].join(path.delimiter) }, + extraEnv: {}, + userDataDir + }) + const launchedAtEpochMs = Date.now() + const app = await electron.launch({ + executablePath: arm.electronPath, + args: getOrcaElectronLaunchArgs(path.join(arm.root, 'out', 'main', 'index.js'), false), + env: { + ...isolation.env, + NODE_ENV: 'development', + ORCA_E2E_HEADLESS: '1', + ORCA_BACKGROUND_LAUNCH: '1', + ORCA_EVENT_LOOP_DELAY_PROBE: '1' + } + }) + const pid = app.process().pid + if (!pid) { + await closeElectronAppForE2E(app, { cleanupDaemonsForUserDataDir: userDataDir }) + throw new Error(`${arm.name}: launch recorded no PID`) + } + const startedAt = processStart(pid) + assertElectronResolvedIsolatedHome( + await app.evaluate(({ app: electronApp }) => electronApp.getPath('home')), + isolation + ) + const page = await app.firstWindow({ timeout: 120_000 }) + await page.waitForFunction( + () => window.__store?.getState().workspaceSessionReady === true, + null, + { timeout: 600_000, polling: 16 } + ) + return { app, page, pid, startedAt, userDataDir, launchedAtEpochMs, readyEpochMs: Date.now() } +} + +export async function stopArm(launch: Launch): Promise { + const current = (() => { + try { + return processStart(launch.pid) + } catch { + return null + } + })() + if (current !== null && current !== launch.startedAt) { + throw new Error( + `PID ${launch.pid} is no longer the process this harness launched; not stopping it` + ) + } + await closeElectronAppForE2E(launch.app, { cleanupDaemonsForUserDataDir: launch.userDataDir }) +} + +export type Workspace = { repoId: string; worktreeIds: [string, string] } + +/** Attach the seeded repo, trust it and route Claude to the stub, the way the e2e fixture prepares a profile. */ +export async function prepareWorkspace( + page: Page, + repoPath: string, + stubCommand: string +): Promise { + const override = buildFakeAgentCommandOverride(stubCommand) + return page.evaluate( + async ({ repoPath, override }) => { + const added = await window.api.repos.add({ path: repoPath }) + if ('error' in added) { + throw new Error(added.error) + } + const store = window.__store! + await store.getState().fetchRepos({ runtimeEnvironmentId: null, throwOnError: true }) + await store.getState().updateRepo(added.repo.id, { + externalWorktreeVisibility: 'show', + agentWorkspaceTrust: 'pre-trust' + }) + await store.getState().updateSettings({ + nativeChatProviders: { claude: true, codex: true }, + defaultTuiAgent: 'claude', + agentCmdOverrides: { claude: override } + }) + for (let attempt = 0; attempt < 60; attempt += 1) { + await store.getState().fetchWorktrees(added.repo.id) + const worktrees = store.getState().worktreesByRepo[added.repo.id] ?? [] + if (worktrees.length >= 2) { + const ordered = [...worktrees].sort( + (a, b) => Number(b.isMainWorktree) - Number(a.isMainWorktree) + ) + store.getState().setActiveWorktree(ordered[0]!.id) + return { + repoId: added.repo.id, + worktreeIds: [ordered[0]!.id, ordered[1]!.id] as [string, string] + } + } + await new Promise((resolve) => setTimeout(resolve, 500)) + } + throw new Error('seeded worktrees did not load') + }, + { repoPath, override } + ) +} + +/** Profiles hold absolute home paths (provider transcripts, config pins), so every arm runs at the same `to` path. */ +export function copyProfile(from: string, to: string): void { + cpSync(from, to, { + recursive: true, + errorOnExist: true, + force: false, + mode: constants.COPYFILE_FICLONE + }) +} + +export const operationId = (arm: Arm): string => + arm.protocol.createStructuredAgentSessionOperationId(randomUUID) diff --git a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts new file mode 100644 index 00000000000..cb0e3b2dc98 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts @@ -0,0 +1,314 @@ +// The 4a "no slower than main" comparison: one profile seeded once, every arm measured on its own copy, one JSON + one table. +// Dry run (no app): pnpm exec tsx config/scripts/performance/native-chat/native-chat-4a-comparison.ts --profile full --dry-run +// Full run: ... --arm main= --arm candidate= --profile full --out [--large-source] +// Re-table: ... --out --report-only +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + writeFileSync +} from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { parseArgs } from 'node:util' +import type { Arm } from './native-chat-4a-arm' +import type { Manifest } from './native-chat-4a-fixture' +import type { RunContext } from './native-chat-4a-run-context' +import { + armMarkdown, + comparisonMarkdown, + summarizeRow, + type ArmResult, + type Row +} from './native-chat-4a-report' +import { + conversionToolOutputBytes, + dryRunWorkload, + workloadFor, + type Workload +} from './native-chat-4a-workloads' + +const FAMILIES = [ + 'conversion', + 'send', + 'stream', + 'history', + 'outline', + 'records', + 'retained', + 'open', + 'rewind' +] as const +type Family = (typeof FAMILIES)[number] + +const { values } = parseArgs({ + options: { + arm: { type: 'string', multiple: true, default: [] }, + profile: { type: 'string', default: 'smoke' }, + out: { type: 'string' }, + 'seed-arm': { type: 'string' }, + families: { type: 'string', default: FAMILIES.join(',') }, + 'large-source': { type: 'boolean', default: false }, + 'conversion-source-bytes': { type: 'string' }, + 'dry-run': { type: 'boolean', default: false }, + 'report-only': { type: 'boolean', default: false } + } +}) + +const workload = workloadFor( + values.profile, + values['conversion-source-bytes'] === undefined + ? undefined + : Number(values['conversion-source-bytes']) +) +const log = (line: string): void => console.error(`[4a ${new Date().toISOString()}] ${line}`) + +function writeComparison(out: string, results: ArmResult[]): void { + const [base, next] = results + if (!base || !next) { + return + } + const name = `4a-${base.sha.slice(0, 8)}-vs-${next.sha.slice(0, 8)}` + const summaries = results.map((result) => ({ + ...result, + summaries: result.rows.map((row: Row) => ({ ...row, summary: summarizeRow(row) })) + })) + const machine = { + platform: process.platform, + release: os.release(), + cpu: os.cpus()[0]?.model, + cpus: os.cpus().length, + memoryBytes: os.totalmem() + } + writeFileSync( + path.join(out, `${name}.json`), + `${JSON.stringify({ workload, machine, arms: summaries }, null, 2)}\n` + ) + writeFileSync(path.join(out, `${name}.md`), comparisonMarkdown(base, next)) + log(`wrote ${path.join(out, `${name}.md`)}`) +} + +async function runArm(input: { + arm: Arm + out: string + seedDir: string + manifest: Manifest + stubCommand: string + families: Set + largeSource?: { seedDir: string; manifest: Manifest } +}): Promise { + const { copyProfile, launchArm, stopArm } = await import('./native-chat-4a-arm') + const { installVisibilityProbe } = await import('./native-chat-4a-probes') + const { resetTags } = await import('./native-chat-4a-run-context') + const lifecycle = await import('./native-chat-4a-ops-lifecycle') + const queue = await import('./native-chat-4a-ops-queue') + const retained = await import('./native-chat-4a-ops-retained') + const stream = await import('./native-chat-4a-ops-stream') + const reads = await import('./native-chat-4a-ops-reads') + const profileDir = path.join(input.out, 'profile') + const rows: Row[] = [] + const startedAt = new Date().toISOString() + + const onProfile = async ( + seedDir: string, + manifest: typeof input.manifest, + body: (ctx: RunContext) => Promise + ) => { + if (existsSync(profileDir)) { + renameSync(profileDir, path.join(input.out, `stale-profile-${Date.now()}`)) + } + copyProfile(seedDir, profileDir) + resetTags() + const open = async () => { + const launch = await launchArm(input.arm, profileDir, input.stubCommand) + await installVisibilityProbe(launch.page) + return launch + } + const ctx = { + arm: input.arm, + launch: await open(), + workload, + manifest, + rows, + log, + relaunch: async () => undefined + } + ctx.relaunch = async () => { + await stopArm(ctx.launch) + ctx.launch = await open() + } + try { + await body(ctx) + } finally { + await stopArm(ctx.launch) + renameSync( + profileDir, + path.join( + input.out, + `after-${input.arm.name}-${manifest.toolOutputBytes}-${input.arm.sha.slice(0, 8)}` + ) + ) + } + } + + const run = async (family: Family, body: () => Promise) => { + if (!input.families.has(family)) { + return + } + log(`${input.arm.name}: ${family}`) + try { + await body() + } catch (error) { + const reason = + `family failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300) + rows.push({ + family, + operation: '(family)', + variant: 'all', + samples: [], + context: {}, + skipped: reason + }) + log(`${input.arm.name}: ${family} ${reason}`) + } + } + + await onProfile(input.seedDir, input.manifest, async (ctx) => { + const seedFiles = lifecycle.seededDbFiles(input.seedDir) + // First: the seeded tail markers and unconverted state exist only before any other family writes. + await run('conversion', () => + lifecycle.firstLaunchRows( + ctx, + seedFiles, + `${workload.history.largeTurns}-turn/${workload.history.fillerChats + 1}-chat profile` + ) + ) + await run('send', () => queue.sendAndQueueRows(ctx)) + await run('stream', () => stream.streamingRows(ctx)) + await run('history', () => reads.historyRows(ctx)) + await run('outline', () => reads.outlineAndForkRows(ctx)) + await run('records', () => reads.recordRows(ctx)) + await run('retained', () => retained.retainedAdmissionRows(ctx)) + await run('open', () => lifecycle.openRows(ctx)) + await run('rewind', () => lifecycle.rewindRows(ctx)) + }) + if (input.largeSource && input.families.has('conversion')) { + const source = input.largeSource + const label = `${(workload.conversionSourceBytes / 1024 ** 3).toFixed(0)} GiB source` + await onProfile(source.seedDir, source.manifest, (ctx) => + run('conversion', () => + lifecycle.firstLaunchRows(ctx, lifecycle.seededDbFiles(source.seedDir), label) + ) + ) + } + return { + arm: input.arm.name, + sha: input.arm.sha, + startedAt, + finishedAt: new Date().toISOString(), + rows, + fixture: { + repoId: input.manifest.workspace.repoId, + worktreeIds: input.manifest.workspace.worktreeIds.join(' '), + seedArm: input.manifest.seedArm, + seedSha: input.manifest.seedSha, + largeItems: input.manifest.largeItems, + storageBytes: input.manifest.storageBytes, + seedMs: input.manifest.seedMs + } + } +} + +async function main(): Promise { + if (values['dry-run']) { + console.log(JSON.stringify(dryRunWorkload(workload), null, 2)) + return + } + const out = path.resolve(values.out ?? path.join(os.tmpdir(), `orca-4a-${workload.profile}`)) + mkdirSync(out, { recursive: true }) + if (values['report-only']) { + const results = readdirSync(out) + .filter((file) => /^arm-.*\.json$/.test(file)) + .map((file) => JSON.parse(readFileSync(path.join(out, file), 'utf8')) as ArmResult) + .sort((a, b) => a.startedAt.localeCompare(b.startedAt)) + writeComparison(out, results) + return + } + const families = new Set(values.families.split(',').map((family) => family.trim()) as Family[]) + const { resolveArm, writeProviderStub } = await import('./native-chat-4a-arm') + const { seedProfile, seedDirFor } = await import('./native-chat-4a-fixture') + const { createSeededTestRepo } = await import('../../../../tests/e2e/helpers/seeded-test-repo') + const scratch = path.join(out, 'scratch') + mkdirSync(scratch, { recursive: true }) + const arms: Arm[] = [] + for (const spec of values.arm) { + const [name, root] = spec.split('=') + if (!name || !root) { + throw new Error(`--arm takes name=checkout, got ${spec}`) + } + arms.push(await resolveArm(name, root, scratch)) + } + if (arms.length === 0) { + throw new Error('Pass at least one --arm name=; main first, then the candidate') + } + const stubCommand = writeProviderStub(path.join(out, 'stub')) + const repoFile = path.join(out, 'repo-path.txt') + const repoPath = existsSync(repoFile) + ? readFileSync(repoFile, 'utf8').trim() + : createSeededTestRepo({ publishPath: false }) + writeFileSync(repoFile, repoPath) + const seedArm = arms.find((arm) => arm.name === values['seed-arm']) ?? arms[0]! + const seed = (toolOutputBytes: number, seeded: Workload) => + seedProfile({ + arm: seedArm, + workload: seeded, + runDir: out, + profileDir: path.join(out, 'profile'), + repoPath, + stubCommand, + toolOutputBytes, + log + }) + const manifest = await seed(workload.history.toolOutputBytes, workload) + const large = + values['large-source'] && workload.conversionSourceBytes > 0 + ? await seed(conversionToolOutputBytes(workload), { + ...workload, + history: { ...workload.history, fillerChats: 3 } + }) + : null + const results: ArmResult[] = [] + for (const arm of arms) { + const result = await runArm({ + arm, + out, + seedDir: seedDirFor(out, manifest), + manifest, + stubCommand, + families, + ...(large ? { largeSource: { seedDir: seedDirFor(out, large), manifest: large } } : {}) + }) + writeFileSync( + path.join(out, `arm-${arm.name}-${arm.sha.slice(0, 12)}.json`), + `${JSON.stringify(result, null, 2)}\n` + ) + writeFileSync(path.join(out, `arm-${arm.name}-${arm.sha.slice(0, 12)}.md`), armMarkdown(result)) + results.push(result) + } + writeComparison(out, results) + const skipped = results.flatMap((result) => + result.rows + .filter((row) => row.skipped) + .map((row) => `${result.arm}: ${row.family}/${row.operation}/${row.variant}: ${row.skipped}`) + ) + if (skipped.length > 0) { + log(`rows without samples (not passes):\n${skipped.join('\n')}`) + } +} + +main().catch((error: unknown) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/config/scripts/performance/native-chat/native-chat-4a-fixture.ts b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts new file mode 100644 index 00000000000..eabedcc6236 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts @@ -0,0 +1,151 @@ +// Seed the retained-history profile once through the seed arm's own host, so every arm starts from the same logical data. +import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { + launchArm, + prepareWorkspace, + stopArm, + type Arm, + type Workspace +} from './native-chat-4a-arm' +import { rpc, storageBytes } from './native-chat-4a-probes' +import { createChat, history, send, waitForDurable, waitForIdle } from './native-chat-4a-session' +import { + fillerTurn, + largeHistoryTurns, + expectedLargeItems, + type Workload +} from './native-chat-4a-workloads' + +export type Manifest = { + seedArm: string + seedSha: string + workload: Workload['profile'] + toolOutputBytes: number + workspace: Workspace + largeSessionId: string + fillerSessionIds: string[] + largeItems: number + storageBytes: number + seedMs: number +} + +const SLEEP_FILLERS_EVERY = 20 + +/** Count the large chat's rows by paging back from the tail; refuses a fixture that seeded fewer rows than its recipe. */ +async function countItems( + page: Awaited>['page'], + sessionId: string +): Promise { + let count = 0 + let cursor: unknown + for (;;) { + const read = await history(page, sessionId, cursor ? 'before' : 'tail', 200, cursor) + count += read.page.items.length + if (!read.page.hasOlder) { + return count + } + cursor = read.page.window.nextCursor + } +} + +export async function seedProfile(input: { + arm: Arm + workload: Workload + runDir: string + profileDir: string + repoPath: string + stubCommand: string + toolOutputBytes: number + log: (line: string) => void +}): Promise { + const seedDir = path.join(input.runDir, `seed-${input.workload.profile}-${input.toolOutputBytes}`) + const manifestPath = `${seedDir}.json` + if (existsSync(manifestPath) && existsSync(seedDir)) { + return JSON.parse(readFileSync(manifestPath, 'utf8')) as Manifest + } + const started = Date.now() + const launch = await launchArm(input.arm, input.profileDir, input.stubCommand) + try { + const { page } = launch + const workspace = await prepareWorkspace(page, input.repoPath, input.stubCommand) + const large = await createChat(page, input.arm, workspace.worktreeIds[0]) + let turn = 0 + for (const next of largeHistoryTurns({ + ...input.workload, + history: { ...input.workload.history, toolOutputBytes: input.toolOutputBytes } + })) { + // A queued turn is sent while its predecessor still runs, leaving released queue history behind. + if (turn > 0 && !next.queued) { + await waitForDurable(page, large.sessionId, `4A-TAIL-large-${turn - 1}`) + } + await send(page, input.arm, large.sessionId, next.prompt) + turn += 1 + if (turn % 250 === 0) { + input.log(`seed: large chat ${turn}/${input.workload.history.largeTurns} turns`) + } + } + await waitForDurable(page, large.sessionId, `4A-TAIL-large-${turn - 1}`, 600_000) + await waitForIdle(page, large.sessionId, 600_000) + const fillerSessionIds: string[] = [] + const workspaceId = workspace.worktreeIds[1] + for (let chat = 0; chat < input.workload.history.fillerChats; chat += 1) { + const filler = await createChat(page, input.arm, workspaceId) + for ( + let fillerTurnIndex = 0; + fillerTurnIndex < input.workload.history.fillerTurns; + fillerTurnIndex += 1 + ) { + await send( + page, + input.arm, + filler.sessionId, + fillerTurn(chat, fillerTurnIndex, input.workload) + ) + await waitForDurable(page, filler.sessionId, `4A-TAIL-filler-${chat}-${fillerTurnIndex}`) + } + fillerSessionIds.push(filler.sessionId) + // Bound live provider processes: retained chats stay, their stubs stop. + if ((chat + 1) % SLEEP_FILLERS_EVERY === 0) { + await rpc(page, 'agentSession.sleepWorkspace', { workspaceId }) + input.log(`seed: ${chat + 1}/${input.workload.history.fillerChats} filler chats`) + } + } + const largeItems = await countItems(page, large.sessionId) + // Hosts may group a call with its result; fewer than prompt+reply per turn means turns were lost. + if (largeItems < input.workload.history.largeTurns * 2) { + throw new Error( + `seed: large chat holds ${largeItems} rows for ${input.workload.history.largeTurns} turns` + ) + } + if (largeItems < expectedLargeItems(input.workload)) { + input.log( + `seed: large chat holds ${largeItems} visible rows; recipe writes ${expectedLargeItems(input.workload)}` + ) + } + const manifest: Manifest = { + seedArm: input.arm.name, + seedSha: input.arm.sha, + workload: input.workload.profile, + toolOutputBytes: input.toolOutputBytes, + workspace, + largeSessionId: large.sessionId, + fillerSessionIds, + largeItems, + storageBytes: 0, + seedMs: 0 + } + await stopArm(launch) + manifest.storageBytes = storageBytes(input.profileDir).total + manifest.seedMs = Date.now() - started + renameSync(input.profileDir, seedDir) + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`) + return manifest + } catch (error) { + await stopArm(launch).catch(() => undefined) + throw error + } +} + +export const seedDirFor = (runDir: string, manifest: Manifest): string => + path.join(runDir, `seed-${manifest.workload}-${manifest.toolOutputBytes}`) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts new file mode 100644 index 00000000000..c395407b1ce --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts @@ -0,0 +1,313 @@ +// 4a process-lifecycle rows: first launch on the seeded profile (the candidate's one-time conversion), cold/warm open, and rewind. +import { readdirSync } from 'node:fs' +import path from 'node:path' +import { + markerSeenAt, + rendererNow, + rpc, + samplePressure, + storageBytes, + watchMarkers +} from './native-chat-4a-probes' +import type { Sample } from './native-chat-4a-report' +import { + addRow, + measured, + page, + sampled, + tagFor, + type RunContext +} from './native-chat-4a-run-context' +import { + history, + mutate, + openChatTab, + send, + userItemIds, + waitForControls, + waitForDurable, + waitForIdle +} from './native-chat-4a-session' +import { replyPrompt } from './native-chat-4a-workloads' + +type Chat = { sessionId: string; worktreeId: string } + +/** Open a chat and time it to its known tail marker visible plus controls known. */ +async function openTo(ctx: RunContext, chat: Chat, marker: string): Promise { + return measured(ctx, async () => { + await watchMarkers(page(ctx), [marker]) + const startedAt = await rendererNow(page(ctx)) + await openChatTab(page(ctx), chat.worktreeId, chat.sessionId) + const tailAt = await markerSeenAt(page(ctx), marker, 600_000) + await waitForControls(page(ctx), 600_000) + return { + e2eMs: (await rendererNow(page(ctx))) - startedAt, + extra: { tailVisibleMs: tailAt - startedAt } + } + }) +} + +/** Give a chat a known tail so open rows end at "the correct tail is visible". */ +async function markTail(ctx: RunContext, chat: Chat): Promise { + const tag = tagFor('tail') + await send(page(ctx), ctx.arm, chat.sessionId, replyPrompt(tag)) + await waitForDurable(page(ctx), chat.sessionId, `4A-TAIL-${tag}`) + await waitForIdle(page(ctx), chat.sessionId) + return `4A-TAIL-${tag}` +} + +const dbFiles = (dir: string): string[] => + readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter( + (entry) => + entry.isFile() && + /\.(db|sqlite)$/.test(entry.name) && + !entry.parentPath.includes(`${path.sep}home`) + ) + .map((entry) => path.relative(dir, path.join(entry.parentPath, entry.name))) + +/** + * The arm's first launch on the seeded profile. On the candidate this is the accepted one-time conversion; on main it is a + * cold open of the same data, so the rows line up but are reported apart from steady state. + */ +export async function firstLaunchRows( + ctx: RunContext, + seedDbFiles: string[], + label: string +): Promise { + const { workspace, largeSessionId, fillerSessionIds } = ctx.manifest + const context = { + profile: label, + largeItems: ctx.manifest.largeItems, + seededBytes: ctx.manifest.storageBytes, + chats: fillerSessionIds.length + 1 + } + const init = addRow(ctx, { + family: 'conversion', + operation: 'launch to session ready', + variant: label, + context + }) + init.samples.push({ e2eMs: ctx.launch.readyEpochMs - ctx.launch.launchedAtEpochMs }) + const stopPressure = samplePressure(ctx.launch.app, ctx.launch.userDataDir, 500) + const small: Chat = { sessionId: fillerSessionIds[0]!, worktreeId: workspace.worktreeIds[1] } + const smallOpen = addRow(ctx, { + family: 'conversion', + operation: 'first open, small chat', + variant: label, + context + }) + await sampled(ctx, smallOpen, 1, () => + openTo(ctx, small, `4A-TAIL-filler-0-${ctx.workload.history.fillerTurns - 1}`) + ) + const largeOpen = addRow(ctx, { + family: 'conversion', + operation: 'first open, large chat, with sends to the opened small chat', + variant: label, + context: { ...context, concurrent: 'sends to the already-opened small chat every 250 ms' } + }) + let opening = true + const concurrent: number[] = [] + const sends = (async () => { + while (opening) { + concurrent.push( + (await send(page(ctx), ctx.arm, small.sessionId, replyPrompt(tagFor('during')))).hostMs + ) + await new Promise((resolve) => setTimeout(resolve, 250)) + } + })() + await sampled(ctx, largeOpen, 1, () => + openTo( + ctx, + { sessionId: largeSessionId, worktreeId: workspace.worktreeIds[0] }, + `4A-TAIL-large-${ctx.workload.history.largeTurns - 1}` + ) + ) + opening = false + await sends + const during = addRow(ctx, { + family: 'conversion', + operation: 'small-chat send during large first open', + variant: label, + context + }) + during.samples.push(...concurrent.map((hostMs) => ({ hostMs }))) + const pressure = await stopPressure() + const remaining = dbFiles(ctx.launch.userDataDir) + const disk = addRow(ctx, { + family: 'conversion', + operation: 'disk', + variant: label, + context: { + retainedOriginalFiles: + seedDbFiles.filter((file) => remaining.includes(file)).join(' ') || 'none', + newFiles: remaining.filter((file) => !seedDbFiles.includes(file)).join(' ') || 'none' + } + }) + disk.samples.push({ + extra: { + peakTotalBytes: pressure.peakTotalBytes, + afterTotalBytes: storageBytes(ctx.launch.userDataDir).total, + peakMainRssBytes: pressure.peakMainRssBytes + } + }) + await waitForIdle(page(ctx), small.sessionId) +} + +export const seededDbFiles = dbFiles + +export async function openRows(ctx: RunContext): Promise { + const { workspace, largeSessionId, fillerSessionIds } = ctx.manifest + const large: Chat = { sessionId: largeSessionId, worktreeId: workspace.worktreeIds[0] } + const small: Chat = { sessionId: fillerSessionIds[0]!, worktreeId: workspace.worktreeIds[1] } + const marks = { large: await markTail(ctx, large), small: await markTail(ctx, small) } + const context = { largeItems: ctx.manifest.largeItems, chats: fillerSessionIds.length + 1 } + const ready = addRow(ctx, { + family: 'open', + operation: 'launch to session ready (metadata only)', + variant: 'converted profile', + context + }) + const coldLarge = addRow(ctx, { + family: 'open', + operation: 'cold process first open', + variant: 'large chat', + context + }) + const coldSmall = addRow(ctx, { + family: 'open', + operation: 'cold process first open', + variant: 'small chat', + context + }) + const coldOutline = addRow(ctx, { + family: 'outline', + operation: 'conversationOutline', + variant: 'first in process (cold)', + context + }) + // A third chat is left active, so the relaunch restores it and the measured chats open for the first time. + const neutral: Chat = { + sessionId: fillerSessionIds[2] ?? fillerSessionIds.at(-1)!, + worktreeId: workspace.worktreeIds[1] + } + for (let launch = 0; launch < ctx.workload.coldSamples; launch += 1) { + await openChatTab(page(ctx), neutral.worktreeId, neutral.sessionId) + await ctx.relaunch() + ready.samples.push({ e2eMs: ctx.launch.readyEpochMs - ctx.launch.launchedAtEpochMs }) + const [first, second, firstRow, secondRow] = + launch % 2 === 0 ? [large, small, coldLarge, coldSmall] : [small, large, coldSmall, coldLarge] + await sampled(ctx, firstRow, 1, () => + openTo(ctx, first, first === large ? marks.large : marks.small) + ) + await sampled(ctx, secondRow, 1, () => + openTo(ctx, second, second === large ? marks.large : marks.small) + ) + await sampled(ctx, coldOutline, 1, () => + measured(ctx, async () => ({ + hostMs: ( + await rpc(page(ctx), 'agentSession.conversationOutline', { sessionId: largeSessionId }) + ).hostMs + })) + ) + } + const unopened = addRow(ctx, { + family: 'open', + operation: 'warm process, unopened chat', + variant: 'small chat', + context + }) + const untouched = fillerSessionIds.slice(3, -1) + await sampled(ctx, unopened, Math.min(ctx.workload.samples, untouched.length), (index) => { + const chat = { sessionId: untouched[index]!, worktreeId: workspace.worktreeIds[1] } + const chatIndex = fillerSessionIds.indexOf(chat.sessionId) + return openTo(ctx, chat, `4A-TAIL-filler-${chatIndex}-${ctx.workload.history.fillerTurns - 1}`) + }) + for (const [variant, chat, marker] of [ + ['large chat', large, marks.large], + ['small chat', small, marks.small] + ] as const) { + const reopen = addRow(ctx, { family: 'open', operation: 'warm reopen', variant, context }) + await sampled(ctx, reopen, ctx.workload.samples, async () => { + await openChatTab( + page(ctx), + (chat === large ? small : large).worktreeId, + (chat === large ? small : large).sessionId + ) + return openTo(ctx, chat, marker) + }) + } +} + +/** Near-tip and deep rewinds on the large chat, with a competing send to another chat while the local writer cuts. */ +export async function rewindRows(ctx: RunContext): Promise { + const { workspace, largeSessionId, fillerSessionIds } = ctx.manifest + const other = fillerSessionIds[1] ?? fillerSessionIds[0]! + await openChatTab(page(ctx), workspace.worktreeIds[0], largeSessionId) + await waitForControls(page(ctx)) + const rewindOnce = async (itemId: string, goneMarker: string): Promise => { + const epoch = (await history(page(ctx), largeSessionId, 'tail', 1)).page.epoch + return measured(ctx, async () => { + const competingTag = tagFor('rewind-compete') + const [answer, competing] = await Promise.all([ + mutate(page(ctx), ctx.arm, 'agentSession.rewind', largeSessionId, { + itemId, + expectedEpoch: epoch + }), + send(page(ctx), ctx.arm, other, replyPrompt(competingTag)) + ]) + const goneAt = await page(ctx) + .waitForFunction( + (marker) => { + const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( + (candidate) => candidate.checkVisibility() + ) + return root && !root.textContent?.includes(marker) ? performance.now() : false + }, + goneMarker, + { timeout: 600_000, polling: 16 } + ) + .then(async (handle) => (await handle.jsonValue()) as number) + await waitForDurable(page(ctx), other, `4A-TAIL-${competingTag}`) + return { + hostMs: answer.hostMs, + e2eMs: goneAt - answer.startedAt, + extra: { competingSendHostMs: competing.hostMs } + } + }) + } + const endToEndOnly = 'preflight, provider success and local cut are one host span' + const nearTip = addRow(ctx, { + family: 'rewind', + operation: 'near-tip rewind', + variant: 'last turn', + context: { largeItems: ctx.manifest.largeItems }, + endToEndOnly + }) + await sampled(ctx, nearTip, ctx.workload.rewind.nearTipSamples, async () => { + const marker = await markTail(ctx, { + sessionId: largeSessionId, + worktreeId: workspace.worktreeIds[0] + }) + const [tip] = (await userItemIds(page(ctx), largeSessionId, 1)).slice(-1) + return rewindOnce(tip!, marker) + }) + const cut = ctx.workload.rewind.deepCutTurns + const deep = addRow(ctx, { + family: 'rewind', + operation: 'deep rewind', + variant: `${cut} turns from tip`, + context: { removedTurns: cut }, + endToEndOnly + }) + await sampled(ctx, deep, ctx.workload.rewind.deepSamples, async () => { + const ids = await userItemIds(page(ctx), largeSessionId, cut + 1) + const tail = JSON.stringify( + (await history(page(ctx), largeSessionId, 'tail', 3)).page.items + ).match(/4A-TAIL-[\w-]+/)?.[0] + if (ids.length <= cut || !tail) { + throw new Error(`large chat has ${ids.length} user turns left, deep cut needs ${cut + 1}`) + } + return rewindOnce(ids[0]!, tail) + }) +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-queue.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-queue.ts new file mode 100644 index 00000000000..7de40e92359 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-queue.ts @@ -0,0 +1,245 @@ +// 4a send and queue rows, plus R3's retained-history admission: a small live queue over growing released history. +import { operationId } from './native-chat-4a-arm' +import { rpc, markerSeenAt, watchMarkers } from './native-chat-4a-probes' +import { + addRow, + measured, + page, + sampled, + tagFor, + type RunContext +} from './native-chat-4a-run-context' +import { + history, + mutate, + openChatTab, + send, + waitForControls, + waitForIdle, + waitForWorking +} from './native-chat-4a-session' +import { holdPrompt, replyPrompt } from './native-chat-4a-workloads' +import providerText from './native-chat-4a-provider-text.cjs' + +/** Type into the composer, then click Send inside the page so the start mark is the click itself. */ +async function clickSend(ctx: RunContext, text: string): Promise { + const composer = page(ctx) + .locator('[data-native-chat-root="true"]:visible [role="textbox"]') + .first() + await composer.fill(text) + return page(ctx).evaluate(() => { + const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( + (candidate) => candidate.checkVisibility() + ) + const button = [...(root?.querySelectorAll('button') ?? [])].find( + (candidate) => + candidate.getAttribute('aria-label') === 'Send' || candidate.textContent?.trim() === 'Send' + ) + if (!button) { + throw new Error('Send button not found') + } + const startedAt = performance.now() + button.click() + return startedAt + }) +} + +export const sentText = (tag: string, padding = 0): string => + `${providerText.formatDirective({ op: 'reply', tag }, padding)} 4A-SENT-${tag}` + +export async function holdActive(ctx: RunContext, sessionId: string, ms: number): Promise { + const tag = tagFor('hold') + await send(page(ctx), ctx.arm, sessionId, holdPrompt(tag, ms)) + await waitForWorking(page(ctx), true) + return tag +} + +export async function queued(ctx: RunContext, sessionId: string) { + return (await history(page(ctx), sessionId, 'tail', 1)).page.queuedMessages ?? [] +} + +export async function sendAndQueueRows(ctx: RunContext): Promise { + const { workspace, fillerSessionIds, largeSessionId } = ctx.manifest + const small = fillerSessionIds[0]! + const n = ctx.workload.samples + await openChatTab(page(ctx), workspace.worktreeIds[1], small) + await waitForControls(page(ctx)) + + const idle = addRow(ctx, { + family: 'send', + operation: 'idle direct send', + variant: 'small chat, click', + context: { chats: fillerSessionIds.length + 1 } + }) + await sampled(ctx, idle, n, () => + measured(ctx, async () => { + const tag = tagFor('idle') + await watchMarkers(page(ctx), [`4A-SENT-${tag}`, `4A-TAIL-${tag}`]) + const clickedAt = await clickSend(ctx, sentText(tag)) + const sentAt = await markerSeenAt(page(ctx), `4A-SENT-${tag}`) + const handover = await waitForWorking(page(ctx), true).catch(() => Number.NaN) + const replyAt = await markerSeenAt(page(ctx), `4A-TAIL-${tag}`) + await waitForIdle(page(ctx), small) + return { + e2eMs: sentAt - clickedAt, + extra: { handoverMs: handover - clickedAt, replyVisibleMs: replyAt - clickedAt } + } + }) + ) + + const duplicate = addRow(ctx, { + family: 'send', + operation: 'duplicate retry', + variant: 'same operation id', + context: {} + }) + await sampled(ctx, duplicate, n, async () => { + const tag = tagFor('dup') + const params = ctx.arm.protocol.structuredAgentSessionMessageSendMutation({ + sessionId: small, + clientOperationId: operationId(ctx.arm), + expectedRuntimeFence: (await history(page(ctx), small, 'tail', 1)).page.fence ?? 1, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: replyPrompt(tag) }] }, + delivery: 'queue-if-active' + }) + const first = await rpc(page(ctx), 'agentSession.send', params) + const sample = await measured(ctx, async () => ({ + hostMs: (await rpc(page(ctx), 'agentSession.send', params)).hostMs + })) + await waitForIdle(page(ctx), small) + return { ...sample, extra: { firstHostMs: first.hostMs } } + }) + + await openChatTab(page(ctx), workspace.worktreeIds[0], largeSessionId) + await waitForControls(page(ctx)) + for (const populated of [0, 5]) { + const active = addRow(ctx, { + family: 'send', + operation: 'send while active', + variant: `large chat, ${populated} already queued`, + context: { + largeItems: ctx.manifest.largeItems, + queueLength: populated, + queuedBodyBytes: populated * 4096 + } + }) + await sampled(ctx, active, n, async () => { + await holdActive(ctx, largeSessionId, 60_000) + for (let index = 0; index < populated; index += 1) { + await send(page(ctx), ctx.arm, largeSessionId, sentText(tagFor('fill'), 4096)) + } + const tag = tagFor('active') + await watchMarkers(page(ctx), [`4A-SENT-${tag}`]) + const sample = await measured(ctx, async () => { + const answer = await send(page(ctx), ctx.arm, largeSessionId, sentText(tag)) + const shownAt = await markerSeenAt(page(ctx), `4A-SENT-${tag}`) + return { hostMs: answer.hostMs, e2eMs: shownAt - answer.startedAt } + }) + await mutate(page(ctx), ctx.arm, 'agentSession.cancel', largeSessionId, {}) + await drainQueue(ctx, largeSessionId) + return sample + }) + } + + for (const editHold of [false, true]) { + const claim = addRow(ctx, { + family: 'send', + operation: 'queue claim', + variant: editHold ? 'after edit hold release' : 'no edit hold', + context: { queueLength: 1 } + }) + await sampled(ctx, claim, n, async () => { + const hold = await holdActive(ctx, largeSessionId, 1_500) + const tag = tagFor('claim') + await send(page(ctx), ctx.arm, largeSessionId, sentText(tag)) + const [card] = await queued(ctx, largeSessionId) + const editId = tagFor('edit') + if (editHold && card) { + const expectedBodyFingerprint = ctx.arm.protocol.agentSessionSendBodyFingerprint( + largeSessionId, + card.body + ) + await rpc(page(ctx), 'agentSession.queuedMessageEditHold', { + sessionId: largeSessionId, + messageId: card.messageId, + editId, + action: 'acquire', + expectedBodyFingerprint + }) + } + await watchMarkers(page(ctx), [`4A-TAIL-${hold}`, `4A-TAIL-${tag}`]) + // With a hold, the claim can start only at release; without one, at the active turn's reply. + let freedAt = await markerSeenAt(page(ctx), `4A-TAIL-${hold}`) + if (editHold && card) { + const release = { + sessionId: largeSessionId, + messageId: card.messageId, + editId, + action: 'release' + } + freedAt = (await rpc(page(ctx), 'agentSession.queuedMessageEditHold', release)).startedAt + } + const repliedAt = await markerSeenAt(page(ctx), `4A-TAIL-${tag}`) + await waitForIdle(page(ctx), largeSessionId) + return { e2eMs: repliedAt - freedAt } + }) + } + + const edit = addRow(ctx, { + family: 'send', + operation: 'queued edit and move', + variant: '3 queued', + context: { queueLength: 3 } + }) + await sampled(ctx, edit, n, async () => { + await holdActive(ctx, largeSessionId, 60_000) + for (let index = 0; index < 3; index += 1) { + await send(page(ctx), ctx.arm, largeSessionId, sentText(tagFor('q'))) + } + const cards = await queued(ctx, largeSessionId) + const target = cards.at(-1)! + const tag = tagFor('edited') + await watchMarkers(page(ctx), [`4A-SENT-${tag}`]) + const sample = await measured(ctx, async () => { + const update = await mutate( + page(ctx), + ctx.arm, + 'agentSession.queuedMessageUpdate', + largeSessionId, + { + messageId: target.messageId, + expectedBodyFingerprint: ctx.arm.protocol.agentSessionSendBodyFingerprint( + largeSessionId, + target.body + ), + text: sentText(tag) + } + ) + const shownAt = await markerSeenAt(page(ctx), `4A-SENT-${tag}`) + const move = await mutate( + page(ctx), + ctx.arm, + 'agentSession.queuedMessageMove', + largeSessionId, + { messageId: target.messageId, beforeMessageId: cards[0]!.messageId } + ) + return { + hostMs: update.hostMs, + e2eMs: shownAt - update.startedAt, + extra: { moveHostMs: move.hostMs } + } + }) + await mutate(page(ctx), ctx.arm, 'agentSession.cancel', largeSessionId, {}) + await drainQueue(ctx, largeSessionId) + return sample + }) +} + +/** Stop pauses the queue; resume lets it drain so the next sample starts idle with an empty queue. */ +async function drainQueue(ctx: RunContext, sessionId: string): Promise { + await waitForWorking(page(ctx), false).catch(() => undefined) + if ((await queued(ctx, sessionId)).length > 0) { + await mutate(page(ctx), ctx.arm, 'agentSession.queuedMessagesResume', sessionId, {}) + } + await waitForIdle(page(ctx), sessionId, 300_000) +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts new file mode 100644 index 00000000000..47b3bb96bb3 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts @@ -0,0 +1,238 @@ +// 4a read rows on the retained large chat: history pages, item detail, outline, fork, and record operations on the many-chat profile. +import { randomUUID } from 'node:crypto' +import { operationId } from './native-chat-4a-arm' +import { rpc } from './native-chat-4a-probes' +import type { Sample } from './native-chat-4a-report' +import { + addRow, + measured, + page, + sampled, + tagFor, + type RunContext +} from './native-chat-4a-run-context' +import { createChat, history, mutate, userItemIds, waitForIdle } from './native-chat-4a-session' + +const bytesOf = (value: unknown): number => Buffer.byteLength(JSON.stringify(value)) + +type Item = { id: string; body?: { kind?: string } } + +async function read( + ctx: RunContext, + method: string, + params: unknown, + allowRefusal = false +): Promise { + return measured(ctx, async () => { + const answer = await rpc(page(ctx), method, params, allowRefusal ? { allowRefusal: true } : {}) + return { + hostMs: answer.hostMs, + extra: { bytes: bytesOf(answer.value), refused: Number(answer.refused) } + } + }) +} + +export async function historyRows(ctx: RunContext): Promise { + const sessionId = ctx.manifest.largeSessionId + const n = ctx.workload.samples + const context = { largeItems: ctx.manifest.largeItems } + const tail = await history(page(ctx), sessionId, 'tail', 200) + const older = await history(page(ctx), sessionId, 'before', 200, tail.page.window.nextCursor) + const catchUpFrom = older.page.window.nextCursor + const pages: [string, Record][] = [ + ['tail 40', { direction: 'tail', limit: 40 }], + ['tail 200 (same page repeated)', { direction: 'tail', limit: 200 }], + [ + 'one older page 200', + { direction: 'before', limit: 200, cursor: tail.page.window.nextCursor } + ], + ['ordinary after catch-up 200', { direction: 'after', limit: 200, cursor: catchUpFrom }], + [ + 'stale epoch reset catch-up', + { direction: 'after', limit: 200, cursor: { epoch: 'stale-4a-epoch', sequence: 1 } } + ] + ] + for (const [variant, params] of pages) { + const row = addRow(ctx, { + family: 'history', + operation: 'agentSession.history', + variant, + context + }) + await sampled(ctx, row, n, () => + read(ctx, 'agentSession.history', { sessionId, ...params }, true) + ) + } + addRow(ctx, { + family: 'history', + operation: 'sparse/interleaved subagent grouping', + variant: 'tail 200', + context, + skipped: + 'the 4a provider stub emits no subagent frames; covered by the Claude/Codex functional QA run' + }) + + // Huge originals come from the streaming rows that ran first; the smallest seeded tool item is the ordinary case. + const items = [...tail.page.items, ...older.page.items] as Item[] + const sized = items + .map((item) => ({ item, bytes: bytesOf(item) })) + .sort((a, b) => b.bytes - a.bytes) + const ordinary = sized.findLast((entry) => entry.item.body?.kind === 'tool-call') ?? sized.at(-1) + const largest = sized[0] + for (const [variant, entry] of [ + ['ordinary tool item', ordinary], + ['largest item on the tail pages', largest] + ] as const) { + const row = addRow(ctx, { + family: 'history', + operation: 'agentSession.item detail', + variant, + context: { wireHeadBytes: entry?.bytes ?? 0 } + }) + if (!entry) { + row.skipped = 'no item on the tail pages' + continue + } + await sampled(ctx, row, n, () => + read(ctx, 'agentSession.item', { sessionId, itemId: entry.item.id }) + ) + } +} + +export async function outlineAndForkRows(ctx: RunContext): Promise { + const sessionId = ctx.manifest.largeSessionId + const outline = addRow(ctx, { + family: 'outline', + operation: 'conversationOutline', + variant: 'repeated (warm)', + context: { largeItems: ctx.manifest.largeItems } + }) + await sampled(ctx, outline, ctx.workload.samples, () => + read(ctx, 'agentSession.conversationOutline', { sessionId }) + ) + + const [tip] = (await userItemIds(page(ctx), sessionId, 1)).slice(-1) + const fork = addRow(ctx, { + family: 'outline', + operation: 'tip fork (create forkFrom)', + variant: 'large chat tip', + context: { historicalSubmissions: ctx.workload.history.largeTurns }, + endToEndOnly: 'local validation and provider copy are one host span' + }) + if (!tip) { + fork.skipped = 'no user item on the large chat' + return + } + await sampled(ctx, fork, Math.min(ctx.workload.samples, 10), () => + measured(ctx, async () => { + const created = await createChat(page(ctx), ctx.arm, ctx.manifest.workspace.worktreeIds[0], { + sessionId, + throughItemId: tip + }) + const readyStarted = Date.now() + const first = await history(page(ctx), created.sessionId, 'tail', 40) + return { + hostMs: created.hostMs, + extra: { forkTailReadMs: Date.now() - readyStarted, forkTailItems: first.page.items.length } + } + }) + ) +} + +export async function recordRows(ctx: RunContext): Promise { + const { fillerSessionIds } = ctx.manifest + const chats = fillerSessionIds.length + 1 + const targets = [fillerSessionIds[0]!, fillerSessionIds.at(-1)!] + const rename = addRow(ctx, { + family: 'records', + operation: 'rename', + variant: 'point lookup, first and last chat', + context: { chats } + }) + await sampled(ctx, rename, ctx.workload.samples, (index) => + measured(ctx, async () => { + const name = tagFor('name') + const answer = await rpc(page(ctx), 'agentSession.rename', { + sessionId: targets[index % 2]!, + name + }) + const shownAt = await page(ctx) + .waitForFunction( + (name) => + [...document.querySelectorAll('[data-testid="sortable-tab"]')].some((tab) => + tab.textContent?.includes(name) + ) && performance.now(), + name, + { timeout: 60_000, polling: 16 } + ) + .then(async (handle) => (await handle.jsonValue()) as number) + .catch(() => Number.NaN) + return { + hostMs: answer.hostMs, + e2eMs: shownAt - answer.startedAt, + extra: { tabNotShown: Number(Number.isNaN(shownAt)) } + } + }) + ) + + const option = addRow(ctx, { + family: 'records', + operation: 'option update (setOption model)', + variant: 'idle small chat', + context: { chats } + }) + await sampled(ctx, option, ctx.workload.samples, () => + measured(ctx, async () => { + const answer = await mutate(page(ctx), ctx.arm, 'agentSession.setOption', targets[0]!, { + key: 'model', + value: 'default' + }) + await waitForIdle(page(ctx), targets[0]!) + return { hostMs: answer.hostMs } + }) + ) + + // Resuming a provider session another chat already adopted makes the host find that chat by provider ID. + const adopted = addRow(ctx, { + family: 'records', + operation: 'adopted-handle lookup (create resumeFrom)', + variant: 'provider id of last chat', + context: { chats } + }) + const provider = ( + await rpc<{ providerSession?: { id?: string } }>(page(ctx), 'agentSession.history', { + sessionId: targets[1]!, + direction: 'tail', + limit: 1 + }) + ).value.providerSession?.id + if (!provider) { + adopted.skipped = 'history named no provider session for the last chat' + return + } + await sampled(ctx, adopted, ctx.workload.samples, () => + measured(ctx, async () => { + const sessionId = ctx.arm.protocol.createStructuredAgentSessionId('claude', randomUUID) + const worktree = `id:${ctx.manifest.workspace.worktreeIds[1]}` + const resumeFrom = { providerSessionId: provider } + const fingerprint = ctx.arm.protocol.structuredAgentSessionPayloadFingerprint({ + method: 'agentSession.create', + sessionId, + fields: { worktree, agent: 'claude', resumeFrom } + }) + const envelope = { + sessionId, + clientOperationId: operationId(ctx.arm), + expectedRuntimeFence: null, + payloadFingerprint: fingerprint + } + const answer = await rpc( + page(ctx), + 'agentSession.create', + { envelope, worktree, agent: 'claude', resumeFrom }, + { allowRefusal: true } + ) + return { hostMs: answer.hostMs, extra: { refused: Number(answer.refused) } } + }) + ) +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-retained.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-retained.ts new file mode 100644 index 00000000000..41d8b3fa1ce --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-retained.ts @@ -0,0 +1,102 @@ +// R3's retained-history rows: queued-send admission and resume after a Stop pause, live queue small, released history growing. +import { markerSeenAt, watchMarkers } from './native-chat-4a-probes' +import { + addRow, + measured, + page, + sampled, + tagFor, + type RunContext +} from './native-chat-4a-run-context' +import { + mutate, + openChatTab, + send, + waitForControls, + waitForIdle, + waitForWorking +} from './native-chat-4a-session' +import { holdPrompt, replyPrompt } from './native-chat-4a-workloads' +import { holdActive, queued, sentText } from './native-chat-4a-ops-queue' + +/** R3: grow released queue history in batches, then measure admission with `liveQueue` waiting at each checkpoint. */ +export async function retainedAdmissionRows(ctx: RunContext): Promise { + const { workspace, fillerSessionIds, largeSessionId } = ctx.manifest + const chat = fillerSessionIds[2] ?? fillerSessionIds[0]! + const { checkpoints, liveQueue } = ctx.workload.retained + const targets: { sessionId: string; worktreeId: string; released: number | 'seeded' }[] = [ + { sessionId: largeSessionId, worktreeId: workspace.worktreeIds[0], released: 'seeded' }, + ...checkpoints.map((released) => ({ + sessionId: chat, + worktreeId: workspace.worktreeIds[1], + released + })) + ] + let released = 0 + for (const target of targets) { + if (typeof target.released === 'number') { + while (released < target.released) { + const batch = Math.min(10, target.released - released) + await send(page(ctx), ctx.arm, chat, holdPrompt(tagFor('grow'), 100)) + for (let index = 0; index < batch; index += 1) { + await send(page(ctx), ctx.arm, chat, replyPrompt(tagFor('released'))) + } + await waitForIdle(page(ctx), chat, 300_000) + released += batch + } + } + await openChatTab(page(ctx), target.worktreeId, target.sessionId) + await waitForControls(page(ctx)) + const context = { + releasedHistory: String(target.released), + liveQueue, + chat: target.sessionId === largeSessionId ? 'large (seeded queue history)' : 'small' + } + const admit = addRow(ctx, { + family: 'retained history', + operation: 'queued-send admission', + variant: `released=${target.released}`, + context + }) + const resume = addRow(ctx, { + family: 'retained history', + operation: 'resume after Stop pause', + variant: `released=${target.released}`, + context + }) + await sampled(ctx, admit, ctx.workload.samples, async () => { + await holdActive(ctx, target.sessionId, 60_000) + for (let index = 0; index < liveQueue - 1; index += 1) { + await send(page(ctx), ctx.arm, target.sessionId, replyPrompt(tagFor('live'))) + } + const tag = tagFor('admit') + await watchMarkers(page(ctx), [`4A-SENT-${tag}`]) + const sample = await measured(ctx, async () => { + const answer = await send(page(ctx), ctx.arm, target.sessionId, sentText(tag)) + return { + hostMs: answer.hostMs, + e2eMs: (await markerSeenAt(page(ctx), `4A-SENT-${tag}`)) - answer.startedAt + } + }) + await mutate(page(ctx), ctx.arm, 'agentSession.cancel', target.sessionId, {}) + await waitForWorking(page(ctx), false) + const last = (await queued(ctx, target.sessionId)).length + resume.samples.push( + await measured(ctx, async () => ({ + hostMs: ( + await mutate( + page(ctx), + ctx.arm, + 'agentSession.queuedMessagesResume', + target.sessionId, + {} + ) + ).hostMs, + extra: { pausedQueueLength: last } + })) + ) + await waitForIdle(page(ctx), target.sessionId, 300_000) + return sample + }) + } +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts new file mode 100644 index 00000000000..e472400b08d --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts @@ -0,0 +1,210 @@ +// 4a streaming rows (1/8/15+ MiB text, no-break and tool output) with a competing send, and the maintenance window after them. +import { + eventLoopLag, + markerSeenAt, + mutationGaps, + rpc, + samplePressure, + storageBytes, + watchMarkers +} from './native-chat-4a-probes' +import { stat, type Sample } from './native-chat-4a-report' +import { + addRow, + measured, + page, + sampled, + tagFor, + type RunContext +} from './native-chat-4a-run-context' +import { + history, + mutate, + openChatTab, + send, + waitForControls, + waitForDurable, + waitForIdle, + waitForWorking +} from './native-chat-4a-session' +import { replyPrompt, streamPrompt, type StreamCase } from './native-chat-4a-workloads' + +const MAINTENANCE_WINDOW_MS = 5_000 + +/** Wall-clock ms until the durable tail holds `marker`, polling slowly so the reader adds little load to either arm. */ +async function durableAfter( + ctx: RunContext, + sessionId: string, + marker: string, + sentEpochMs: number +): Promise { + await waitForDurable(page(ctx), sessionId, marker, 300_000, 250) + return Date.now() - sentEpochMs +} + +/** A small send to another chat while the stream runs: its acceptance span and when its reply is durable. */ +async function competingSend( + ctx: RunContext, + sessionId: string +): Promise<{ hostMs: number; durableMs: number }> { + const tag = tagFor('compete') + const sentAt = Date.now() + const answer = await send(page(ctx), ctx.arm, sessionId, replyPrompt(tag)) + await waitForDurable(page(ctx), sessionId, `4A-TAIL-${tag}`, 300_000, 25) + return { hostMs: answer.hostMs, durableMs: Date.now() - sentAt } +} + +async function streamSample( + ctx: RunContext, + shape: StreamCase, + stopAfterFirst: boolean +): Promise { + const { largeSessionId, fillerSessionIds } = ctx.manifest + const { launch } = ctx + // Earlier samples stay in the transcript, so every sample's markers are its own. + const stream = { ...shape, tag: tagFor(shape.tag) } + const other = fillerSessionIds[1] ?? fillerSessionIds[0]! + const before = storageBytes(launch.userDataDir) + const stopPressure = samplePressure(launch.app, launch.userDataDir) + const fromEpoch = Date.now() + const sample = await measured(ctx, async () => { + const first = `4A-FIRST-${stream.tag}` + await watchMarkers(page(ctx), [first]) + const sentEpoch = Date.now() + const answer = await send( + page(ctx), + ctx.arm, + largeSessionId, + streamPrompt(ctx.workload, stream) + ) + const firstAt = await markerSeenAt(page(ctx), first, 300_000) + const [competing, durableFirstMs] = await Promise.all([ + competingSend(ctx, other), + durableAfter(ctx, largeSessionId, first, sentEpoch) + ]) + let stopHostMs = Number.NaN + if (stopAfterFirst) { + stopHostMs = (await mutate(page(ctx), ctx.arm, 'agentSession.cancel', largeSessionId, {})) + .hostMs + } + const doneAt = await waitForWorking(page(ctx), false, 600_000) + const settled = await waitForIdle(page(ctx), largeSessionId, 600_000) + const durableEndMs = Date.now() - sentEpoch + const gaps = stat(await mutationGaps(page(ctx), firstAt, doneAt)) + return { + hostMs: answer.hostMs, + e2eMs: doneAt - answer.startedAt, + extra: { + firstTextMs: firstAt - answer.startedAt, + liveGapP95Ms: gaps?.p95 ?? Number.NaN, + liveGapMaxMs: gaps?.max ?? Number.NaN, + durableFirstMs, + durableSettledMs: durableEndMs, + competingHostMs: competing.hostMs, + competingDurableMs: competing.durableMs, + stopHostMs, + tailItems: settled.items.length + } + } + }) + const pressure = await stopPressure() + const after = storageBytes(launch.userDataDir) + const lag = eventLoopLag(launch.userDataDir, fromEpoch, Date.now()) + return { + ...sample, + extra: { + ...sample.extra, + walGrowthBytes: after.wal - before.wal, + fileGrowthBytes: after.total - before.total, + peakWalBytes: pressure.peakWalBytes, + peakMainRssBytes: pressure.peakMainRssBytes, + peakAppWorkingSetBytes: pressure.peakAppWorkingSetBytes, + eventLoopP99Ms: lag.p99Ms, + eventLoopMaxMs: lag.maxMs + } + } +} + +/** After a large stream, sends to another chat while retired payloads are pruned, pages reclaimed and the WAL checkpointed. */ +async function maintenanceWindow(ctx: RunContext): Promise { + const other = ctx.manifest.fillerSessionIds[1] ?? ctx.manifest.fillerSessionIds[0]! + const fromEpoch = Date.now() + const samples: Sample[] = [] + while (Date.now() - fromEpoch < MAINTENANCE_WINDOW_MS) { + const before = storageBytes(ctx.launch.userDataDir) + const sendAnswer = await send(page(ctx), ctx.arm, other, replyPrompt(tagFor('maint'))) + const read = await history(page(ctx), ctx.manifest.largeSessionId, 'tail', 40) + const after = storageBytes(ctx.launch.userDataDir) + samples.push({ + hostMs: sendAnswer.hostMs, + extra: { + largeTailReadMs: read.hostMs, + sinceStreamEndMs: Date.now() - fromEpoch, + walChangeBytes: after.wal - before.wal, + fileChangeBytes: after.total - before.total + } + }) + await new Promise((resolve) => setTimeout(resolve, 200)) + } + await waitForIdle(page(ctx), other) + const lag = eventLoopLag(ctx.launch.userDataDir, fromEpoch, Date.now()) + samples.at(-1)!.extra = { + ...samples.at(-1)!.extra, + windowEventLoopP99Ms: lag.p99Ms, + windowEventLoopMaxMs: lag.maxMs + } + return samples +} + +export async function streamingRows(ctx: RunContext): Promise { + const { workspace, largeSessionId, largeItems } = ctx.manifest + await openChatTab(page(ctx), workspace.worktreeIds[0], largeSessionId) + await waitForControls(page(ctx)) + const samples = ctx.workload.stream.samples + for (const stream of ctx.workload.stream.cases) { + const shape = stream.toolOutputBytes + ? 'tool output running→completed' + : stream.shape === 'nobreak' + ? 'no-break block' + : 'block-rich text' + const size = `${((stream.toolOutputBytes ?? stream.bytes) / 1024 / 1024).toFixed(2)} MiB` + const context = { + encodedBytes: stream.toolOutputBytes ?? stream.bytes, + chunkBytes: ctx.workload.stream.chunkBytes, + largeItems, + concurrent: 'small send in another chat after first text' + } + const row = addRow(ctx, { family: 'streaming', operation: shape, variant: size, context }) + await sampled(ctx, row, samples, () => streamSample(ctx, stream, false)) + if (!stream.toolOutputBytes) { + const stop = addRow(ctx, { + family: 'streaming', + operation: `${shape}, Stop after first text`, + variant: size, + context + }) + await sampled(ctx, stop, samples, () => streamSample(ctx, stream, true)) + } + const maintenance = addRow(ctx, { + family: 'maintenance', + operation: 'other-chat sends and large-chat reads after stream', + variant: size, + context: { + windowMs: MAINTENANCE_WINDOW_MS, + retiredPayloadBytes: stream.toolOutputBytes ?? stream.bytes + } + }) + try { + maintenance.samples.push(...(await maintenanceWindow(ctx))) + } catch (error) { + maintenance.skipped = + `failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300) + } + } + // A cheap read after all streams confirms the large chat still answers its tail. + await rpc(page(ctx), 'agentSession.history', { + sessionId: largeSessionId, + direction: 'tail', + limit: 1 + }) +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-probes.ts b/config/scripts/performance/native-chat/native-chat-4a-probes.ts new file mode 100644 index 00000000000..03b4550bf97 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-probes.ts @@ -0,0 +1,231 @@ +// Arm-neutral 4a probes: renderer-clock end points, host CPU/event-loop work, memory and on-disk storage, read from outside product code. +import { readdirSync, statSync } from 'node:fs' +import path from 'node:path' +import type { Performance as NodePerformance } from 'node:perf_hooks' +import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import { summarizeEventLoopDelay } from '../../../../tests/e2e/helpers/event-loop-delay-report' + +export type RpcAnswer = { ok: true; result: T } | { ok: false; error: unknown } + +/** One RPC from the renderer, timed on the renderer clock: the host span every row reports beside its visible end point. */ +export async function rpc( + page: Page, + method: string, + params: unknown, + options: { allowRefusal?: true } = {} +): Promise<{ value: T; refused: boolean; startedAt: number; hostMs: number }> { + const answer = await page.evaluate( + async ({ method, params }) => { + const startedAt = performance.now() + const reply = (await window.api.runtime.call({ method, params })) as RpcAnswer + return { reply, startedAt, hostMs: performance.now() - startedAt } + }, + { method, params } + ) + if (!answer.reply.ok && !options.allowRefusal) { + throw new Error(`${method} refused: ${JSON.stringify(answer.reply.error).slice(0, 400)}`) + } + const value = (answer.reply.ok ? answer.reply.result : answer.reply.error) as T + return { value, refused: !answer.reply.ok, startedAt: answer.startedAt, hostMs: answer.hostMs } +} + +/** Install once per page: records when text first appears in the chat root and every transcript mutation batch. */ +export async function installVisibilityProbe(page: Page): Promise { + await page.evaluate(() => { + const probe = window as unknown as { + __4a?: { seen: Map; mutations: number[]; watch: Set } + } + if (probe.__4a) { + return + } + const state = { + seen: new Map(), + mutations: [] as number[], + watch: new Set() + } + probe.__4a = state + const check = (): void => { + const now = performance.now() + state.mutations.push(now) + if (state.mutations.length > 200_000) { + state.mutations.splice(0, 100_000) + } + if (state.watch.size === 0) { + return + } + // Text still in the composer is not sent: count only occurrences outside it. + // Hidden tabs can stay mounted; only the chat the user sees counts. + const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( + (candidate) => candidate.checkVisibility() + ) + const text = root?.textContent ?? '' + const draft = root?.querySelector('[role="textbox"]')?.textContent ?? '' + const count = (value: string, marker: string): number => value.split(marker).length - 1 + for (const marker of state.watch) { + if (count(text, marker) > count(draft, marker)) { + state.seen.set(marker, now) + state.watch.delete(marker) + } + } + } + new MutationObserver(check).observe(document.body, { + subtree: true, + childList: true, + characterData: true + }) + }) +} + +export async function watchMarkers(page: Page, markers: string[]): Promise { + await page.evaluate((markers) => { + const state = (window as unknown as { __4a: { watch: Set; seen: Map } }) + .__4a + for (const marker of markers) { + state.seen.delete(marker) + state.watch.add(marker) + } + }, markers) +} + +/** Renderer-clock time a marker became visible, waiting up to `timeoutMs`. */ +export async function markerSeenAt( + page: Page, + marker: string, + timeoutMs = 120_000 +): Promise { + const handle = await page.waitForFunction( + (marker) => + (window as unknown as { __4a: { seen: Map } }).__4a.seen.get(marker) ?? false, + marker, + { timeout: timeoutMs, polling: 16 } + ) + return (await handle.jsonValue()) as number +} + +/** Gaps between visible transcript updates inside [from, to], the live-update cadence a reader sees. */ +export async function mutationGaps(page: Page, from: number, to: number): Promise { + return page.evaluate( + ({ from, to }) => { + const times = (window as unknown as { __4a: { mutations: number[] } }).__4a.mutations.filter( + (t) => t >= from && t <= to + ) + return times.slice(1).map((time, index) => time - times[index]!) + }, + { from, to } + ) +} + +export const rendererNow = (page: Page): Promise => page.evaluate(() => performance.now()) + +export type HostWork = { cpuMs: number; mainThreadActiveMs: number; offMainThreadCpuMs: number } + +/** Process CPU covers the main thread and its storage workers; main-thread ELU separates the two. */ +export async function hostWorkMark(app: ElectronApplication): Promise<() => Promise> { + // Main-process callbacks see Node's performance, not the DOM one this file is typed against. + const start = await app.evaluate(() => ({ + cpu: process.cpuUsage(), + elu: (performance as unknown as NodePerformance).eventLoopUtilization() + })) + return async () => { + const delta = await app.evaluate((_electron, start) => { + const node = performance as unknown as NodePerformance + return { + cpu: process.cpuUsage(start.cpu), + elu: node.eventLoopUtilization(node.eventLoopUtilization(), start.elu) + } + }, start) + const cpuMs = (delta.cpu.user + delta.cpu.system) / 1000 + return { + cpuMs, + mainThreadActiveMs: delta.elu.active, + offMainThreadCpuMs: Math.max(0, cpuMs - delta.elu.active) + } + } +} + +/** All SQLite/journal files under the profile, so WAL growth and file growth read the same on both storage designs. */ +export function storageBytes(userDataDir: string): { db: number; wal: number; total: number } { + let db = 0 + let wal = 0 + let total = 0 + const visit = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (entry.name !== 'home') { + visit(file) + } + continue + } + const size = statSync(file, { throwIfNoEntry: false })?.size ?? 0 + total += size + if (entry.name.endsWith('-wal')) { + wal += size + } else if (/\.(db|sqlite)$/.test(entry.name)) { + db += size + } + } + } + visit(userDataDir) + return { db, wal, total } +} + +export type Pressure = { + peakMainRssBytes: number + peakAppWorkingSetBytes: number + peakWalBytes: number + peakTotalBytes: number +} + +/** Sample memory and storage every `everyMs` until stopped; peaks only, so sampling cost stays flat. */ +export function samplePressure( + app: ElectronApplication, + userDataDir: string, + everyMs = 250 +): () => Promise { + const peak: Pressure = { + peakMainRssBytes: 0, + peakAppWorkingSetBytes: 0, + peakWalBytes: 0, + peakTotalBytes: 0 + } + let stopped = false + const loop = (async () => { + while (!stopped) { + const memory = await app + .evaluate(({ app: electronApp }) => ({ + rss: process.memoryUsage().rss, + workingSetKb: electronApp + .getAppMetrics() + .reduce((sum, metric) => sum + metric.memory.workingSetSize, 0) + })) + .catch(() => null) + if (memory) { + peak.peakMainRssBytes = Math.max(peak.peakMainRssBytes, memory.rss) + peak.peakAppWorkingSetBytes = Math.max( + peak.peakAppWorkingSetBytes, + memory.workingSetKb * 1024 + ) + } + const disk = storageBytes(userDataDir) + peak.peakWalBytes = Math.max(peak.peakWalBytes, disk.wal) + peak.peakTotalBytes = Math.max(peak.peakTotalBytes, disk.total) + await new Promise((resolve) => setTimeout(resolve, everyMs)) + } + })() + return async () => { + stopped = true + await loop + return peak + } +} + +/** Worst main-process event-loop window overlapping [fromEpochMs, toEpochMs], from the probe both arms already ship. */ +export function eventLoopLag( + userDataDir: string, + fromEpochMs: number, + toEpochMs: number +): { p99Ms: number; maxMs: number; windows: number } { + const summary = summarizeEventLoopDelay(userDataDir, 'main', fromEpochMs, toEpochMs) + return { p99Ms: summary.p99, maxMs: summary.max, windows: summary.windows } +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs new file mode 100644 index 00000000000..8155ded9d94 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs @@ -0,0 +1,310 @@ +// Claude stream-json stand-in for the 4a comparison; wrappers pass the launch log first. Speaks the same initialize/turn/interrupt +// protocol as tests/e2e/fixtures/structured-claude-stub.cjs and runs the prompt's [[4a {...}]] directive (see native-chat-4a-provider-text.cjs). +const { randomUUID } = require('node:crypto') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { parseDirective, prose, streamText } = require('./native-chat-4a-provider-text.cjs') + +const [launchLog, ...args] = process.argv.slice(2) +fs.appendFileSync(launchLog, `${JSON.stringify({ pid: process.pid, args })}\n`) + +if (args.includes('--version') || args.includes('-v')) { + process.stdout.write('2.1.280 (Claude Code)\n') + process.exit(0) +} + +function argValue(name) { + const joined = args.find((arg) => arg.startsWith(`${name}=`)) + if (joined) { + return joined.slice(name.length + 1) + } + const index = args.indexOf(name) + return index !== -1 ? args[index + 1] : undefined +} + +const sessionId = argValue('--session-id') ?? argValue('--resume') ?? randomUUID() +const claudeHome = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude') +const transcriptPath = path.join( + claudeHome, + 'projects', + process.cwd().replace(/[^a-zA-Z0-9]/g, '-'), + `${sessionId}.jsonl` +) +fs.mkdirSync(path.dirname(transcriptPath), { recursive: true }) +let parentUuid = null +if (fs.existsSync(transcriptPath)) { + const lines = fs.readFileSync(transcriptPath, 'utf8').trim().split('\n').filter(Boolean) + parentUuid = lines.length > 0 ? (JSON.parse(lines.at(-1)).uuid ?? null) : null +} + +// Fork and rewind read the provider's own transcript, so every user/assistant frame is also saved there. +function remember(frame) { + const row = { + parentUuid, + isSidechain: false, + type: frame.type, + message: frame.message, + uuid: frame.uuid, + timestamp: new Date().toISOString(), + sessionId, + cwd: process.cwd() + } + fs.appendFileSync(transcriptPath, `${JSON.stringify(row)}\n`) + parentUuid = frame.uuid +} + +let backlog = Promise.resolve() +function write(frame) { + const line = `${JSON.stringify(frame)}\n` + // Respect stdout backpressure so a 15 MiB stream measures the host, not an unbounded pipe buffer. + backlog = backlog.then( + () => + new Promise((resolve) => + process.stdout.write(line) ? resolve() : process.stdout.once('drain', resolve) + ) + ) + return backlog +} + +function framed(type, message, extra = {}) { + const frame = { + type, + uuid: randomUUID(), + session_id: sessionId, + parent_tool_use_id: null, + message, + ...extra + } + remember(frame) + return write(frame) +} + +const assistant = (content) => + framed('assistant', { id: randomUUID(), role: 'assistant', model: 'claude-fake', content }) +const streamEvent = (event) => + write({ + type: 'stream_event', + event, + session_id: sessionId, + parent_tool_use_id: null, + uuid: randomUUID() + }) +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +let current = null +const pending = [] +let turnCounter = 0 + +async function toolRun(turn, index, outputBytes, runMs) { + const id = `toolu_4a_${turn}_${index}` + await assistant([ + { type: 'tool_use', id, name: 'Bash', input: { command: `ls -la dir${index}` } } + ]) + if (runMs > 0) { + await write({ + type: 'system', + subtype: 'task_started', + task_id: `task-${turn}-${index}`, + tool_use_id: id, + description: 'run', + task_type: 'local_bash', + uuid: randomUUID(), + session_id: sessionId + }) + await sleep(runMs) + } + await framed('user', { + role: 'user', + content: [ + { tool_use_id: id, type: 'tool_result', content: prose(outputBytes, id), is_error: false } + ] + }) +} + +async function streamReply(op, signal) { + const text = streamText(op) + const messageId = randomUUID() + await streamEvent({ + type: 'message_start', + message: { + id: messageId, + type: 'message', + role: 'assistant', + model: 'claude-fake', + content: [] + } + }) + await streamEvent({ + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' } + }) + const chunk = op.chunkBytes ?? 16_384 + for (let offset = 0; offset < text.length && !signal.aborted; offset += chunk) { + await streamEvent({ + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: text.slice(offset, offset + chunk) } + }) + if (op.intervalMs) { + await sleep(op.intervalMs) + } + } + if (signal.aborted) { + return + } + await framed('assistant', { + id: messageId, + role: 'assistant', + model: 'claude-fake', + content: [{ type: 'text', text }] + }) + await streamEvent({ type: 'content_block_stop', index: 0 }) + await streamEvent({ type: 'message_stop' }) +} + +async function runOp(op, signal) { + const turn = (turnCounter += 1) + const tag = op.tag ?? `t${turn}` + if (op.op === 'history') { + for (let index = 1; index <= (op.tools ?? 8) && !signal.aborted; index += 1) { + await toolRun(turn, index, op.toolOutputBytes ?? 2000, 0) + } + await assistant([{ type: 'text', text: `4A-TAIL-${tag} ${prose(op.replyBytes ?? 1500, tag)}` }]) + return + } + if (op.op === 'stream') { + if (op.toolOutputBytes) { + await toolRun(turn, 0, op.toolOutputBytes, op.toolRunMs ?? 250) + } + await streamReply({ ...op, tag }, signal) + return + } + if (op.op === 'hold') { + const deadline = Date.now() + (op.ms ?? 60_000) + while (!signal.aborted && Date.now() < deadline) { + await sleep(10) + } + } + if (!signal.aborted) { + await assistant([{ type: 'text', text: `4A-TAIL-${tag} ${op.text ?? 'Stub reply.'}` }]) + } +} + +async function startTurn(frame) { + const controller = new AbortController() + current = controller + // The CLI's --replay-user-messages echo, which opens the turn. + remember({ type: 'user', uuid: frame.uuid ?? randomUUID(), message: frame.message }) + await write({ ...frame, isReplay: true }) + const content = frame.message?.content ?? frame.message + const prompt = Array.isArray(content) + ? content.map((block) => block?.text ?? '').join('\n') + : content + const op = parseDirective(prompt) ?? { op: 'reply' } + await runOp(op, controller.signal) + if (controller.signal.aborted) { + return + } + await write({ + type: 'result', + subtype: 'success', + uuid: randomUUID(), + session_id: sessionId, + is_error: false, + terminal_reason: 'completed', + result: 'done' + }) + current = null + const next = pending.shift() + if (next) { + void startTurn(next) + } +} + +function interruptTurn() { + if (!current) { + return + } + current.abort() + current = null + void write({ + type: 'result', + subtype: 'error_during_execution', + is_error: true, + session_id: sessionId, + parent_tool_use_id: null + }) + const next = pending.shift() + if (next) { + void startTurn(next) + } +} + +const initializeResponse = { + commands: [], + models: [{ value: 'default', displayName: 'Default', description: '' }], + account: {}, + output_style: 'default', + available_output_styles: ['default'] +} + +let buffer = '' +let initSent = false +process.stdin.setEncoding('utf8') +process.stdin.on('data', (chunk) => { + buffer += String(chunk) + let index + while ((index = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, index).trim() + buffer = buffer.slice(index + 1) + let frame + try { + frame = line ? JSON.parse(line) : null + } catch { + frame = null + } + if (frame?.type === 'user') { + if (current) { + pending.push(frame) + } else { + void startTurn(frame) + } + continue + } + if (frame?.type !== 'control_request') { + continue + } + const subtype = frame.request?.subtype + if (subtype === 'interrupt') { + interruptTurn() + } + void write({ + type: 'control_response', + response: { + subtype: 'success', + request_id: frame.request_id, + response: subtype === 'initialize' ? initializeResponse : {} + } + }) + if (subtype === 'initialize' && !initSent) { + initSent = true + void write({ + type: 'system', + subtype: 'init', + session_id: sessionId, + uuid: randomUUID(), + model: 'claude-fake', + cwd: process.cwd(), + tools: [], + mcp_servers: [], + permissionMode: 'default', + slash_commands: [], + apiKeySource: 'none', + output_style: 'default' + }) + } + } +}) +process.stdin.on('end', () => process.exit(0)) diff --git a/config/scripts/performance/native-chat/native-chat-4a-provider-text.cjs b/config/scripts/performance/native-chat/native-chat-4a-provider-text.cjs new file mode 100644 index 00000000000..ba74f8cdac1 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-provider-text.cjs @@ -0,0 +1,78 @@ +// Deterministic provider text and the prompt directive shared by the 4a stub and its driver, so both arms see identical bytes. +const { createHash } = require('node:crypto') + +const DIRECTIVE = /\[\[4a (\{[^\]]*\})\]\]/ + +/** A prompt that tells the stub what to emit; padding makes large prompt bodies without changing the op. */ +function formatDirective(op, paddingBytes = 0) { + const head = `[[4a ${JSON.stringify(op)}]]` + return paddingBytes > 0 ? `${head}\n${prose(paddingBytes, op.tag ?? 'pad')}` : head +} + +function parseDirective(text) { + const match = DIRECTIVE.exec(typeof text === 'string' ? text : JSON.stringify(text ?? '')) + if (!match) { + return null + } + try { + return JSON.parse(match[1]) + } catch { + return null + } +} + +function seededWords(seed) { + let state = 0 + for (const char of String(seed)) { + state = (Math.imul(state, 31) + char.charCodeAt(0)) | 0 + } + return () => { + state = (state + 0x6d2b79f5) | 0 + let t = Math.imul(state ^ (state >>> 15), 1 | state) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +const WORDS = ['journal', 'stream', 'outline', 'queue', 'render', 'commit', 'reader', 'writer'] + +/** Block-rich markdown: headings, lists, fenced code and prose, cut to exactly `bytes` ASCII bytes. */ +function prose(bytes, seed) { + const next = seededWords(seed) + const parts = [] + let length = 0 + for (let block = 0; length < bytes; block += 1) { + const words = Array.from({ length: 12 }, () => WORDS[Math.floor(next() * WORDS.length)]) + const shape = block % 4 + const text = + shape === 0 + ? `## Section ${block}\n\n${words.join(' ')}.\n\n` + : shape === 1 + ? `${words.map((word) => `- **${word}** item`).join('\n')}\n\n` + : shape === 2 + ? `\`\`\`ts\nconst ${words[0]}${block} = '${words.slice(1).join('-')}'\n\`\`\`\n\n` + : `${words.join(' ')} with \`${words[1]}\` and a [link](https://example.invalid/${block}).\n\n` + parts.push(text) + length += text.length + } + return parts.join('').slice(0, bytes) +} + +/** One unbroken line with no whitespace, the shape that defeats line-based chunking. */ +function noBreak(bytes, seed) { + const unit = createHash('sha256').update(String(seed)).digest('hex') + return unit.repeat(Math.ceil(bytes / unit.length)).slice(0, bytes) +} + +function streamText(op) { + const body = op.shape === 'nobreak' ? noBreak : prose + const head = `4A-FIRST-${op.tag} ` + const tail = ` 4A-END-${op.tag}` + return `${head}${body(Math.max(0, op.bytes - head.length - tail.length), op.tag)}${tail}` +} + +function digest(text) { + return createHash('sha256').update(text).digest('hex').slice(0, 16) +} + +module.exports = { formatDirective, parseDirective, prose, noBreak, streamText, digest } diff --git a/config/scripts/performance/native-chat/native-chat-4a-report.ts b/config/scripts/performance/native-chat/native-chat-4a-report.ts new file mode 100644 index 00000000000..44021dacc5c --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-report.ts @@ -0,0 +1,152 @@ +// 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's head SHA. +import type { HostWork } from './native-chat-4a-probes' + +export type Sample = { + /** Action to user-visible (or durably readable, when the row says so) result. */ + e2eMs?: number + /** The RPC's renderer round trip: host handling including storage. */ + hostMs?: number + work?: HostWork + /** Row-specific spans and sizes, e.g. firstTextMs, walBytes, bytes, items, queueLength. */ + extra?: Record +} + +export type Row = { + family: string + operation: string + /** e.g. cold, warm, 8MiB nobreak; rows match across arms by family/operation/variant. */ + variant: string + samples: Sample[] + /** Shape facts that hold for the whole row: bytes, items, queue length, concurrent work. */ + context: Record + /** Why a row has no samples on this arm, so absence is never read as a pass. */ + skipped?: string + /** Sub-spans this row cannot separate without a product hook; the table marks it "end-to-end only". */ + endToEndOnly?: string +} + +export type ArmResult = { + arm: string + sha: string + startedAt: string + finishedAt: string + rows: Row[] + fixture: Record +} + +export type Stat = { n: number; p50: number; p95: number; max: number } + +function nearestRank(sorted: number[], quantile: number): number { + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(quantile * sorted.length) - 1))]! +} + +export function stat(values: (number | undefined)[]): Stat | null { + const sorted = values + .filter((value): value is number => typeof value === 'number' && Number.isFinite(value)) + .sort((a, b) => a - b) + if (sorted.length === 0) { + return null + } + return { + n: sorted.length, + p50: nearestRank(sorted, 0.5), + p95: nearestRank(sorted, 0.95), + max: sorted.at(-1)! + } +} + +export function summarizeRow(row: Row) { + const extras = new Set(row.samples.flatMap((sample) => Object.keys(sample.extra ?? {}))) + return { + e2eMs: stat(row.samples.map((sample) => sample.e2eMs)), + hostMs: stat(row.samples.map((sample) => sample.hostMs)), + hostCpuMs: stat(row.samples.map((sample) => sample.work?.cpuMs)), + offMainThreadCpuMs: stat(row.samples.map((sample) => sample.work?.offMainThreadCpuMs)), + mainThreadActiveMs: stat(row.samples.map((sample) => sample.work?.mainThreadActiveMs)), + extra: Object.fromEntries( + [...extras].map((key) => [key, stat(row.samples.map((sample) => sample.extra?.[key]))]) + ) + } +} + +const key = (row: Pick): string => + `${row.family}\u0000${row.operation}\u0000${row.variant}` +const ms = (value: number): string => + value >= 100 ? value.toFixed(0) : value >= 10 ? value.toFixed(1) : value.toFixed(2) +const cell = (value: Stat | null): string => + value ? `${ms(value.p50)} / ${ms(value.p95)} / ${ms(value.max)} (n=${value.n})` : '—' + +function delta(base: Stat | null, next: Stat | null): string { + if (!base || !next) { + return '—' + } + const change = next.p95 - base.p95 + const percent = + base.p95 > 0 ? ` (${change >= 0 ? '+' : ''}${((change / base.p95) * 100).toFixed(0)}%)` : '' + return `${change >= 0 ? '+' : ''}${ms(change)}${percent}` +} + +const caseCell = (row: Row): string => + row.endToEndOnly ? `${row.variant} (end-to-end only: ${row.endToEndOnly})` : row.variant + +/** The row's primary latency: visible result where measured, else the host span. */ +const primary = (summary: ReturnType): Stat | null => + summary.e2eMs ?? summary.hostMs + +function contextCell(row: Row | undefined): string { + if (!row) { + return '' + } + return Object.entries(row.context) + .map(([name, value]) => `${name}=${value}`) + .join(', ') +} + +export function comparisonMarkdown(base: ArmResult, next: ArmResult): string { + const nextRows = new Map(next.rows.map((row) => [key(row), row])) + const baseRows = new Map(base.rows.map((row) => [key(row), row])) + const ordered = [...base.rows, ...next.rows.filter((row) => !baseRows.has(key(row)))] + const lines = [ + `4a comparison: ${base.arm} \`${base.sha.slice(0, 11)}\` vs ${next.arm} \`${next.sha.slice(0, 11)}\` (p50 / p95 / max ms, nearest rank).`, + '', + `| Family | Operation | Case | ${base.arm} | ${next.arm} | Δ p95 | Host span p95 ${base.arm} → ${next.arm} | Off-main CPU p95 ${base.arm} → ${next.arm} | Shape |`, + '| --- | --- | --- | --- | --- | --- | --- | --- | --- |' + ] + const seen = new Set() + for (const row of ordered) { + if (seen.has(key(row))) { + continue + } + seen.add(key(row)) + const left = baseRows.get(key(row)) + const right = nextRows.get(key(row)) + const a = left ? summarizeRow(left) : null + const b = right ? summarizeRow(right) : null + const span = (summary: typeof a, pick: 'hostMs' | 'offMainThreadCpuMs'): string => + summary?.[pick] ? ms(summary[pick]!.p95) : '—' + lines.push( + `| ${row.family} | ${row.operation} | ${caseCell(row)} | ${left?.skipped ? `skipped: ${left.skipped}` : cell(a && primary(a))} | ${right?.skipped ? `skipped: ${right.skipped}` : cell(b && primary(b))} | ${delta(a && primary(a), b && primary(b))} | ${span(a, 'hostMs')} → ${span(b, 'hostMs')} | ${span(a, 'offMainThreadCpuMs')} → ${span(b, 'offMainThreadCpuMs')} | ${contextCell(left ?? right)} |` + ) + } + lines.push( + '', + 'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.' + ) + return `${lines.join('\n')}\n` +} + +export function armMarkdown(result: ArmResult): string { + const lines = [ + `4a ${result.arm} \`${result.sha.slice(0, 11)}\` (p50 / p95 / max ms).`, + '', + '| Family | Operation | Case | Primary | Host span | Shape |', + '| --- | --- | --- | --- | --- | --- |' + ] + for (const row of result.rows) { + const summary = summarizeRow(row) + lines.push( + `| ${row.family} | ${row.operation} | ${caseCell(row)} | ${row.skipped ? `skipped: ${row.skipped}` : cell(primary(summary))} | ${cell(summary.hostMs)} | ${contextCell(row)} |` + ) + } + return `${lines.join('\n')}\n` +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-run-context.ts b/config/scripts/performance/native-chat/native-chat-4a-run-context.ts new file mode 100644 index 00000000000..999f11a2003 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-run-context.ts @@ -0,0 +1,66 @@ +// What every 4a operation family receives, plus the one helper that turns a timed action into a row sample. +import type { Page } from '@stablyai/playwright-test' +import type { Arm, Launch } from './native-chat-4a-arm' +import type { Manifest } from './native-chat-4a-fixture' +import { hostWorkMark, rendererNow } from './native-chat-4a-probes' +import type { Row, Sample } from './native-chat-4a-report' +import type { Workload } from './native-chat-4a-workloads' + +export type RunContext = { + arm: Arm + launch: Launch + workload: Workload + manifest: Manifest + rows: Row[] + log: (line: string) => void + /** Relaunch on the same profile (cold rows); replaces `launch`. */ + relaunch: () => Promise +} + +export const page = (ctx: RunContext): Page => ctx.launch.page + +export function addRow(ctx: RunContext, row: Omit & { samples?: Sample[] }): Row { + const full: Row = { samples: [], ...row } + ctx.rows.push(full) + return full +} + +/** Run `action` and record host work around it; `action` returns renderer-clock end points relative to its own start. */ +export async function measured( + ctx: RunContext, + action: (startedAt: number) => Promise> +): Promise { + const done = await hostWorkMark(ctx.launch.app) + const startedAt = await rendererNow(page(ctx)) + const sample = await action(startedAt) + return { ...sample, work: await done() } +} + +/** Run a row's samples, recording a failure as the row's skip reason instead of losing the whole family. */ +export async function sampled( + ctx: RunContext, + row: Row, + count: number, + take: (index: number) => Promise +): Promise { + try { + for (let index = 0; index < count; index += 1) { + row.samples.push(await take(index)) + } + } catch (error) { + row.skipped = + `failed after ${row.samples.length} samples: ${error instanceof Error ? error.message : String(error)}`.slice( + 0, + 300 + ) + ctx.log(`${row.family}/${row.operation}/${row.variant}: ${row.skipped}`) + } +} + +// Reset per arm so both arms send byte-identical prompts in the same order. +let unique = 0 +export const resetTags = (): void => { + unique = 0 +} +export const tagFor = (prefix: string): string => + `${prefix}-${String((unique += 1)).padStart(5, '0')}` diff --git a/config/scripts/performance/native-chat/native-chat-4a-session.ts b/config/scripts/performance/native-chat/native-chat-4a-session.ts new file mode 100644 index 00000000000..77b47d88666 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-session.ts @@ -0,0 +1,228 @@ +// Chat operations through the host RPC surface both arms serve, with envelopes built by each arm's own protocol bundle. +// Declares the window.__store and window.api globals these page.evaluate callbacks use. +import type {} from '../../../../tests/e2e/helpers/runtime-types' +import { randomUUID } from 'node:crypto' +import type { Page } from '@stablyai/playwright-test' +import { operationId, type Arm } from './native-chat-4a-arm' +import { rpc } from './native-chat-4a-probes' + +type HistoryItem = { id: string; body?: { kind?: string; role?: string; state?: string } } +export type HistoryPage = { + epoch: string + fence?: number + items: HistoryItem[] + window: { nextCursor: unknown } + hasOlder: boolean + queuedMessages?: { messageId: string; body: unknown; state: string }[] | null +} +type HistoryResult = { ok: boolean; page: HistoryPage; fence?: number } + +export async function history( + page: Page, + sessionId: string, + direction: 'tail' | 'before' | 'after', + limit: number, + cursor?: unknown +) { + const answer = await rpc(page, 'agentSession.history', { + sessionId, + direction, + limit, + ...(cursor ? { cursor } : {}) + }) + return { ...answer, page: answer.value.page } +} + +async function currentFence(page: Page, sessionId: string): Promise { + const { value } = await rpc(page, 'agentSession.history', { + sessionId, + direction: 'tail', + limit: 1 + }) + const fence = value.page.fence ?? value.fence + if (typeof fence !== 'number') { + throw new Error(`${sessionId}: history served no runtime fence`) + } + return fence +} + +export async function createChat( + page: Page, + arm: Arm, + worktreeId: string, + forkFrom?: { sessionId: string; throughItemId: string } +) { + const sessionId = arm.protocol.createStructuredAgentSessionId('claude', randomUUID) + const worktree = `id:${worktreeId}` + const params = { + envelope: { + sessionId, + clientOperationId: operationId(arm), + expectedRuntimeFence: null, + payloadFingerprint: arm.protocol.structuredAgentSessionCreateFingerprint({ + sessionId, + worktree, + agent: 'claude', + forkFrom + }) + }, + worktree, + agent: 'claude', + ...(forkFrom ? { forkFrom } : {}) + } + const answer = await rpc(page, 'agentSession.create', params) + return { sessionId, hostMs: answer.hostMs, startedAt: answer.startedAt } +} + +/** The renderer's generic write: fingerprint {method, sessionId, fields} at the fence the client last saw. */ +export async function mutate( + page: Page, + arm: Arm, + method: string, + sessionId: string, + fields: Record +) { + const envelope = { + sessionId, + clientOperationId: operationId(arm), + expectedRuntimeFence: await currentFence(page, sessionId), + payloadFingerprint: arm.protocol.structuredAgentSessionPayloadFingerprint({ + method, + sessionId, + fields + }) + } + return rpc(page, method, { envelope, ...fields }) +} + +export async function send( + page: Page, + arm: Arm, + sessionId: string, + text: string, + queueIfActive = true +) { + const params = arm.protocol.structuredAgentSessionMessageSendMutation({ + sessionId, + clientOperationId: operationId(arm), + expectedRuntimeFence: await currentFence(page, sessionId), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + ...(queueIfActive ? { delivery: 'queue-if-active' as const } : {}) + }) + return rpc<{ ok?: boolean; queued?: unknown }>(page, 'agentSession.send', params) +} + +/** Poll the durable tail until it holds `marker`: the reply was committed and is readable, independent of any view. */ +export async function waitForDurable( + page: Page, + sessionId: string, + marker: string, + timeoutMs = 120_000, + everyMs = 25 +) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const tail = await history(page, sessionId, 'tail', 3) + if (JSON.stringify(tail.page.items).includes(marker)) { + return + } + await new Promise((resolve) => setTimeout(resolve, everyMs)) + } + throw new Error(`${sessionId}: ${marker} was not durable within ${timeoutMs} ms`) +} + +/** Settled when no turn row in the tail is still running and nothing is queued. */ +export async function waitForIdle(page: Page, sessionId: string, timeoutMs = 120_000) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const tail = await history(page, sessionId, 'tail', 20) + const running = tail.page.items.some( + (item) => item.body?.kind === 'turn' && item.body.state === 'running' + ) + if (!running && (tail.page.queuedMessages ?? []).length === 0) { + return tail.page + } + await new Promise((resolve) => setTimeout(resolve, 25)) + } + throw new Error(`${sessionId}: still active after ${timeoutMs} ms`) +} + +export async function userItemIds(page: Page, sessionId: string, limit = 200): Promise { + const ids: string[] = [] + let cursor: unknown + for (;;) { + const read = await history(page, sessionId, cursor ? 'before' : 'tail', 200, cursor) + ids.unshift( + ...read.page.items + .filter((item) => item.body?.kind === 'message' && item.body.role === 'user') + .map((item) => item.id) + ) + if (!read.page.hasOlder || ids.length >= limit) { + return ids.slice(-limit) + } + cursor = read.page.window.nextCursor + } +} + +/** Activate the chat's tab through the tab bar, the way a user switches chats; never reveals or focuses the window. */ +export async function openChatTab( + page: Page, + worktreeId: string, + sessionId: string +): Promise { + const tabId = await page.waitForFunction( + ({ worktreeId, sessionId }) => + window.__store + ?.getState() + .unifiedTabsByWorktree[worktreeId]?.find((tab) => tab.entityId === sessionId)?.id ?? false, + { worktreeId, sessionId }, + { timeout: 60_000 } + ) + const id = (await tabId.jsonValue()) as string + await page.evaluate( + ({ worktreeId, id }) => { + const state = window.__store!.getState() + if (state.activeWorktreeId !== worktreeId) { + state.setActiveWorktree(worktreeId) + } + state.setActiveTab(id) + }, + { worktreeId, id } + ) +} + +/** Controls known: the composer is interactive and the working flag has been published for this chat. */ +export async function waitForControls(page: Page, timeoutMs = 60_000): Promise { + await page.waitForFunction( + () => { + const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( + (candidate) => candidate.checkVisibility() + ) + const composer = root?.querySelector('[role="textbox"]') + return Boolean(composer && root?.querySelector('[data-native-chat-working]')) + }, + null, + { timeout: timeoutMs, polling: 16 } + ) +} + +export async function waitForWorking( + page: Page, + working: boolean, + timeoutMs = 120_000 +): Promise { + const handle = await page.waitForFunction( + (working) => { + const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( + (candidate) => candidate.checkVisibility() + ) + const flag = root?.querySelector('[data-native-chat-working]') + return flag?.getAttribute('data-native-chat-working') === String(working) + ? performance.now() + : false + }, + working, + { timeout: timeoutMs, polling: 16 } + ) + return (await handle.jsonValue()) as number +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts new file mode 100644 index 00000000000..1961ce0a7bc --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts @@ -0,0 +1,216 @@ +// Deterministic 4a workloads: the retained-history profile, stream shapes and per-row sample counts, identical for both arms. +import providerText from './native-chat-4a-provider-text.cjs' + +export type StreamShape = 'blocks' | 'nobreak' + +export type StreamCase = { + tag: string + bytes: number + shape: StreamShape + /** Running-to-completed tool output emitted before the text, when set. */ + toolOutputBytes?: number +} + +export type Workload = { + profile: 'smoke' | 'full' + /** Warm repeated samples per row; cold rows take `coldSamples` relaunches. */ + samples: number + coldSamples: number + history: { + /** The large retained chat: one recipe turn is 1 prompt + `tools` tool items + 1 reply (+ its turn row). */ + largeTurns: number + tools: number + toolOutputBytes: number + promptBytes: number + replyBytes: number + /** Every Nth large-chat turn is sent while active, leaving released queue history for conversion. */ + queuedEvery: number + fillerChats: number + fillerTurns: number + } + /** Streams are long; `samples` per stream case, separate from the warm row count. */ + stream: { cases: StreamCase[]; chunkBytes: number; intervalMs: number; samples: number } + /** Released queued sends accumulated before each retained-history checkpoint; the live queue stays `liveQueue`. */ + retained: { checkpoints: number[]; liveQueue: number } + rewind: { nearTipSamples: number; deepCutTurns: number; deepSamples: number } + /** Generated only on the M4Air run; smoke keeps the 55k shape off this Mac. */ + conversionSourceBytes: number +} + +const MiB = 1024 * 1024 + +function streamCases(sizes: number[]): StreamCase[] { + return sizes.flatMap((bytes) => [ + { tag: `blocks-${bytes / MiB}m`, bytes, shape: 'blocks' as const }, + { tag: `nobreak-${bytes / MiB}m`, bytes, shape: 'nobreak' as const }, + { + tag: `tool-${bytes / MiB}m`, + bytes: 64 * 1024, + shape: 'blocks' as const, + toolOutputBytes: bytes + } + ]) +} + +export const WORKLOADS: Record = { + smoke: { + profile: 'smoke', + samples: 3, + coldSamples: 1, + history: { + largeTurns: 20, + tools: 8, + toolOutputBytes: 2000, + promptBytes: 300, + replyBytes: 1500, + queuedEvery: 10, + fillerChats: 3, + fillerTurns: 1 + }, + stream: { cases: streamCases([MiB / 4]), chunkBytes: 16_384, intervalMs: 1, samples: 1 }, + retained: { checkpoints: [0, 20], liveQueue: 2 }, + rewind: { nearTipSamples: 2, deepCutTurns: 5, deepSamples: 1 }, + conversionSourceBytes: 0 + }, + full: { + profile: 'full', + samples: 30, + coldSamples: 5, + // The established 5000-turn/55000-item recipe plus 309 small chats: the 55k-history/310-chat profile. + history: { + largeTurns: 5000, + tools: 8, + toolOutputBytes: 2000, + promptBytes: 300, + replyBytes: 1500, + queuedEvery: 10, + fillerChats: 309, + fillerTurns: 2 + }, + stream: { + cases: streamCases([MiB, 8 * MiB, 16 * MiB]), + chunkBytes: 16_384, + intervalMs: 2, + samples: 10 + }, + retained: { checkpoints: [0, 1_000, 10_000], liveQueue: 2 }, + rewind: { nearTipSamples: 10, deepCutTurns: 1000, deepSamples: 3 }, + conversionSourceBytes: 20 * 1024 * MiB + } +} + +export function workloadFor(profile: string, conversionSourceBytes?: number): Workload { + const workload = WORKLOADS[profile as Workload['profile']] + if (!workload) { + throw new Error(`Unknown profile ${profile}; expected ${Object.keys(WORKLOADS).join(', ')}`) + } + return conversionSourceBytes === undefined ? workload : { ...workload, conversionSourceBytes } +} + +export type HistoryTurn = { prompt: string; queued: boolean } + +/** Large-chat recipe turns in send order; each reply ends with a 4A-TAIL marker naming its turn. */ +export function* largeHistoryTurns(workload: Workload): Generator { + const { history } = workload + for (let turn = 0; turn < history.largeTurns; turn += 1) { + const op = { + op: 'history', + tag: `large-${turn}`, + tools: history.tools, + toolOutputBytes: history.toolOutputBytes, + replyBytes: history.replyBytes + } + yield { + prompt: providerText.formatDirective(op, history.promptBytes), + queued: history.queuedEvery > 0 && turn % history.queuedEvery === history.queuedEvery - 1 + } + } +} + +export function fillerTurn(chat: number, turn: number, workload: Workload): string { + const op = { + op: 'history', + tag: `filler-${chat}-${turn}`, + tools: 2, + toolOutputBytes: 500, + replyBytes: 400 + } + return providerText.formatDirective(op, workload.history.promptBytes) +} + +/** Expected large-chat rows, so a run can refuse a fixture that did not seed what it claims. */ +export function expectedLargeItems(workload: Workload): number { + return workload.history.largeTurns * (workload.history.tools + 3) +} + +/** Per-turn tool output that makes the conversion source reach `conversionSourceBytes` with the same turn count. */ +export function conversionToolOutputBytes(workload: Workload): number { + const { history } = workload + if (workload.conversionSourceBytes === 0) { + return history.toolOutputBytes + } + return Math.max( + history.toolOutputBytes, + Math.ceil(workload.conversionSourceBytes / (history.largeTurns * history.tools)) + ) +} + +export function streamPrompt(workload: Workload, stream: StreamCase): string { + return providerText.formatDirective({ + op: 'stream', + tag: stream.tag, + bytes: stream.bytes, + shape: stream.shape, + chunkBytes: workload.stream.chunkBytes, + intervalMs: workload.stream.intervalMs, + ...(stream.toolOutputBytes ? { toolOutputBytes: stream.toolOutputBytes, toolRunMs: 250 } : {}) + }) +} + +export const holdPrompt = (tag: string, ms: number): string => + providerText.formatDirective({ op: 'hold', tag, ms }) +export const replyPrompt = (tag: string): string => + providerText.formatDirective({ op: 'reply', tag }) + +/** Generate every byte both arms will receive and digest it; changing a generator changes the dry-run digest. */ +export function dryRunWorkload(workload: Workload): Record { + const started = performance.now() + let promptBytes = 0 + let queued = 0 + let turns = 0 + for (const turn of largeHistoryTurns(workload)) { + promptBytes += Buffer.byteLength(turn.prompt) + queued += Number(turn.queued) + turns += 1 + if (providerText.parseDirective(turn.prompt)?.op !== 'history') { + throw new Error('history directive did not round-trip') + } + } + const streams = workload.stream.cases.map((stream) => { + const op = providerText.parseDirective(streamPrompt(workload, stream)) + const text = providerText.streamText(op) + if (text.length !== stream.bytes || !text.startsWith(`4A-FIRST-${stream.tag}`)) { + throw new Error(`${stream.tag}: generated ${text.length} bytes, expected ${stream.bytes}`) + } + const newlines = text.split('\n').length - 1 + return { tag: stream.tag, bytes: text.length, newlines, digest: providerText.digest(text) } + }) + return { + profile: workload.profile, + largeTurns: turns, + expectedLargeItems: expectedLargeItems(workload), + queuedHistoryTurns: queued, + largePromptBytes: promptBytes, + largeToolOutputBytes: turns * workload.history.tools * workload.history.toolOutputBytes, + chats: 1 + workload.history.fillerChats, + fillerPromptDigest: providerText.digest(fillerTurn(0, 0, workload)), + streams, + retained: workload.retained, + rewind: workload.rewind, + conversion: { + sourceBytes: workload.conversionSourceBytes, + toolOutputBytesPerTool: conversionToolOutputBytes(workload) + }, + generatorMs: Math.round(performance.now() - started) + } +} From ef879135810343785a7a950a5d59e74b308fe3d8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:03:00 -0800 Subject: [PATCH 02/15] Fail the 4a harness launch fast on a non-e2e build and stop the app it launched --- .../native-chat/native-chat-4a-arm.ts | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm.ts b/config/scripts/performance/native-chat/native-chat-4a-arm.ts index c8381635a8c..aba2e023cd9 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-arm.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-arm.ts @@ -154,17 +154,30 @@ export async function launchArm( throw new Error(`${arm.name}: launch recorded no PID`) } const startedAt = processStart(pid) - assertElectronResolvedIsolatedHome( - await app.evaluate(({ app: electronApp }) => electronApp.getPath('home')), - isolation - ) - const page = await app.firstWindow({ timeout: 120_000 }) - await page.waitForFunction( - () => window.__store?.getState().workspaceSessionReady === true, - null, - { timeout: 600_000, polling: 16 } - ) - return { app, page, pid, startedAt, userDataDir, launchedAtEpochMs, readyEpochMs: Date.now() } + try { + assertElectronResolvedIsolatedHome( + await app.evaluate(({ app: electronApp }) => electronApp.getPath('home')), + isolation + ) + const page = await app.firstWindow({ timeout: 120_000 }) + // Only an e2e-mode build exposes the store; a desktop build would otherwise wait out the full budget. + await page + .waitForFunction(() => Boolean(window.__store), null, { timeout: 60_000 }) + .catch(() => { + throw new Error( + `${arm.name}: no window.__store; build ${arm.root} with VITE_EXPOSE_STORE=true electron-vite build --mode e2e` + ) + }) + await page.waitForFunction( + () => window.__store?.getState().workspaceSessionReady === true, + null, + { timeout: 600_000, polling: 16 } + ) + return { app, page, pid, startedAt, userDataDir, launchedAtEpochMs, readyEpochMs: Date.now() } + } catch (error) { + await closeElectronAppForE2E(app, { cleanupDaemonsForUserDataDir: userDataDir }) + throw error + } } export async function stopArm(launch: Launch): Promise { From 503d5f323c033b7371a328fc0feeb18a6caceccf Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:18:59 -0800 Subject: [PATCH 03/15] Shim tsx's __name in 4a harness pages and install the probe inside the guarded launch --- .../scripts/performance/native-chat/native-chat-4a-arm.ts | 6 ++++++ .../performance/native-chat/native-chat-4a-comparison.ts | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm.ts b/config/scripts/performance/native-chat/native-chat-4a-arm.ts index aba2e023cd9..cbba47b9723 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-arm.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-arm.ts @@ -20,6 +20,7 @@ import { getOrcaElectronLaunchArgs } from '../../../../tests/e2e/helpers/electro import { closeElectronAppForE2E } from '../../../../tests/e2e/helpers/electron-process-shutdown' import { getE2ECompletedOnboardingProfile } from '../../../../tests/e2e/helpers/e2e-completed-onboarding-profile' import { buildFakeAgentCommandOverride } from '../../../../tests/e2e/helpers/fake-agent-command-override' +import { installVisibilityProbe } from './native-chat-4a-probes' const HARNESS_ROOT = path.resolve(import.meta.dirname, '../../../..') const SYSTEM_DIRS = ['/usr/bin', '/bin', '/usr/sbin', '/sbin'] @@ -160,6 +161,10 @@ export async function launchArm( isolation ) const page = await app.firstWindow({ timeout: 120_000 }) + // tsx keeps function names by wrapping them in __name(), which page.evaluate callbacks carry into the renderer. + const nameShim = 'globalThis.__name ??= (fn) => fn' + await page.addInitScript(nameShim) + await page.evaluate(nameShim) // Only an e2e-mode build exposes the store; a desktop build would otherwise wait out the full budget. await page .waitForFunction(() => Boolean(window.__store), null, { timeout: 60_000 }) @@ -173,6 +178,7 @@ export async function launchArm( null, { timeout: 600_000, polling: 16 } ) + await installVisibilityProbe(page) return { app, page, pid, startedAt, userDataDir, launchedAtEpochMs, readyEpochMs: Date.now() } } catch (error) { await closeElectronAppForE2E(app, { cleanupDaemonsForUserDataDir: userDataDir }) diff --git a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts index cb0e3b2dc98..b847991f978 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts @@ -100,7 +100,6 @@ async function runArm(input: { largeSource?: { seedDir: string; manifest: Manifest } }): Promise { const { copyProfile, launchArm, stopArm } = await import('./native-chat-4a-arm') - const { installVisibilityProbe } = await import('./native-chat-4a-probes') const { resetTags } = await import('./native-chat-4a-run-context') const lifecycle = await import('./native-chat-4a-ops-lifecycle') const queue = await import('./native-chat-4a-ops-queue') @@ -121,11 +120,7 @@ async function runArm(input: { } copyProfile(seedDir, profileDir) resetTags() - const open = async () => { - const launch = await launchArm(input.arm, profileDir, input.stubCommand) - await installVisibilityProbe(launch.page) - return launch - } + const open = () => launchArm(input.arm, profileDir, input.stubCommand) const ctx = { arm: input.arm, launch: await open(), From 8d448e8d984dc91de59eb52fb53adcb6772e9161 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:30:47 -0800 Subject: [PATCH 04/15] Read the native chat working flag from the 4a harness's visible root itself --- .../performance/native-chat/native-chat-4a-session.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-session.ts b/config/scripts/performance/native-chat/native-chat-4a-session.ts index 77b47d88666..522a8741524 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-session.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-session.ts @@ -199,7 +199,8 @@ export async function waitForControls(page: Page, timeoutMs = 60_000): Promise candidate.checkVisibility() ) const composer = root?.querySelector('[role="textbox"]') - return Boolean(composer && root?.querySelector('[data-native-chat-working]')) + // The working flag is an attribute of the root itself. + return Boolean(composer && root?.hasAttribute('data-native-chat-working')) }, null, { timeout: timeoutMs, polling: 16 } @@ -216,8 +217,7 @@ export async function waitForWorking( const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( (candidate) => candidate.checkVisibility() ) - const flag = root?.querySelector('[data-native-chat-working]') - return flag?.getAttribute('data-native-chat-working') === String(working) + return root?.getAttribute('data-native-chat-working') === String(working) ? performance.now() : false }, From a65497e63a492605782ae7e3dfd925c551348181 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:38:27 -0800 Subject: [PATCH 05/15] Retry transient main-process evaluates in the 4a harness host-work probe --- .../native-chat/native-chat-4a-arm.ts | 5 +++- .../native-chat/native-chat-4a-probes.ts | 27 +++++++++++-------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm.ts b/config/scripts/performance/native-chat/native-chat-4a-arm.ts index cbba47b9723..0b6c5d3160c 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-arm.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-arm.ts @@ -17,6 +17,7 @@ import { assertElectronResolvedIsolatedHome } from '../../../../tests/e2e/helpers/electron-home-isolation' import { getOrcaElectronLaunchArgs } from '../../../../tests/e2e/helpers/electron-launch-args' +import { retryTransientMainEvaluate } from '../../../../tests/e2e/helpers/electron-main-evaluate-retry' import { closeElectronAppForE2E } from '../../../../tests/e2e/helpers/electron-process-shutdown' import { getE2ECompletedOnboardingProfile } from '../../../../tests/e2e/helpers/e2e-completed-onboarding-profile' import { buildFakeAgentCommandOverride } from '../../../../tests/e2e/helpers/fake-agent-command-override' @@ -157,7 +158,9 @@ export async function launchArm( const startedAt = processStart(pid) try { assertElectronResolvedIsolatedHome( - await app.evaluate(({ app: electronApp }) => electronApp.getPath('home')), + await retryTransientMainEvaluate(() => + app.evaluate(({ app: electronApp }) => electronApp.getPath('home')) + ), isolation ) const page = await app.firstWindow({ timeout: 120_000 }) diff --git a/config/scripts/performance/native-chat/native-chat-4a-probes.ts b/config/scripts/performance/native-chat/native-chat-4a-probes.ts index 03b4550bf97..cff44fe4ca9 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-probes.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-probes.ts @@ -3,6 +3,7 @@ import { readdirSync, statSync } from 'node:fs' import path from 'node:path' import type { Performance as NodePerformance } from 'node:perf_hooks' import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import { retryTransientMainEvaluate } from '../../../../tests/e2e/helpers/electron-main-evaluate-retry' import { summarizeEventLoopDelay } from '../../../../tests/e2e/helpers/event-loop-delay-report' export type RpcAnswer = { ok: true; result: T } | { ok: false; error: unknown } @@ -122,18 +123,22 @@ export type HostWork = { cpuMs: number; mainThreadActiveMs: number; offMainThrea /** Process CPU covers the main thread and its storage workers; main-thread ELU separates the two. */ export async function hostWorkMark(app: ElectronApplication): Promise<() => Promise> { // Main-process callbacks see Node's performance, not the DOM one this file is typed against. - const start = await app.evaluate(() => ({ - cpu: process.cpuUsage(), - elu: (performance as unknown as NodePerformance).eventLoopUtilization() - })) + const start = await retryTransientMainEvaluate(() => + app.evaluate(() => ({ + cpu: process.cpuUsage(), + elu: (performance as unknown as NodePerformance).eventLoopUtilization() + })) + ) return async () => { - const delta = await app.evaluate((_electron, start) => { - const node = performance as unknown as NodePerformance - return { - cpu: process.cpuUsage(start.cpu), - elu: node.eventLoopUtilization(node.eventLoopUtilization(), start.elu) - } - }, start) + const delta = await retryTransientMainEvaluate(() => + app.evaluate((_electron, start) => { + const node = performance as unknown as NodePerformance + return { + cpu: process.cpuUsage(start.cpu), + elu: node.eventLoopUtilization(node.eventLoopUtilization(), start.elu) + } + }, start) + ) const cpuMs = (delta.cpu.user + delta.cpu.system) / 1000 return { cpuMs, From fb4b83b4d2810c28fb88427fd484b9355d71aa47 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:48:02 -0800 Subject: [PATCH 06/15] Use the history page's itemId in the 4a harness and name what a visible-state wait timed out on --- .../native-chat/native-chat-4a-ops-reads.ts | 2 +- .../native-chat/native-chat-4a-probes.ts | 34 ++++++++++++++--- .../native-chat/native-chat-4a-session.ts | 37 ++++++++++++------- 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts index 47b3bb96bb3..8846cdf7b38 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts @@ -94,7 +94,7 @@ export async function historyRows(ctx: RunContext): Promise { continue } await sampled(ctx, row, n, () => - read(ctx, 'agentSession.item', { sessionId, itemId: entry.item.id }) + read(ctx, 'agentSession.item', { sessionId, itemId: entry.item.itemId }) ) } } diff --git a/config/scripts/performance/native-chat/native-chat-4a-probes.ts b/config/scripts/performance/native-chat/native-chat-4a-probes.ts index cff44fe4ca9..48bdd599775 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-probes.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-probes.ts @@ -94,15 +94,37 @@ export async function markerSeenAt( marker: string, timeoutMs = 120_000 ): Promise { - const handle = await page.waitForFunction( - (marker) => - (window as unknown as { __4a: { seen: Map } }).__4a.seen.get(marker) ?? false, - marker, - { timeout: timeoutMs, polling: 16 } - ) + const handle = await page + .waitForFunction( + (marker) => + (window as unknown as { __4a: { seen: Map } }).__4a.seen.get(marker) ?? + false, + marker, + { timeout: timeoutMs, polling: 16 } + ) + .catch(async (error: unknown) => { + throw new Error( + `${marker} not visible in ${timeoutMs} ms (${await visibleChatState(page)})`, + { + cause: error + } + ) + }) return (await handle.jsonValue()) as number } +/** One line on the chat the user sees, for timeout messages. */ +export function visibleChatState(page: Page): Promise { + return page + .evaluate(() => { + const roots = [...document.querySelectorAll('[data-native-chat-root="true"]')] + const root = roots.find((candidate) => candidate.checkVisibility()) + const text = root?.textContent ?? '' + return `${roots.length} roots, visible=${Boolean(root)}, working=${root?.getAttribute('data-native-chat-working')}, tail=${JSON.stringify(text.slice(-160))}` + }) + .catch((error: unknown) => `state unreadable: ${String(error)}`) +} + /** Gaps between visible transcript updates inside [from, to], the live-update cadence a reader sees. */ export async function mutationGaps(page: Page, from: number, to: number): Promise { return page.evaluate( diff --git a/config/scripts/performance/native-chat/native-chat-4a-session.ts b/config/scripts/performance/native-chat/native-chat-4a-session.ts index 522a8741524..c886b2d7bb2 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-session.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-session.ts @@ -4,9 +4,9 @@ import type {} from '../../../../tests/e2e/helpers/runtime-types' import { randomUUID } from 'node:crypto' import type { Page } from '@stablyai/playwright-test' import { operationId, type Arm } from './native-chat-4a-arm' -import { rpc } from './native-chat-4a-probes' +import { rpc, visibleChatState } from './native-chat-4a-probes' -type HistoryItem = { id: string; body?: { kind?: string; role?: string; state?: string } } +type HistoryItem = { itemId: string; body?: { kind?: string; role?: string; state?: string } } export type HistoryPage = { epoch: string fence?: number @@ -155,7 +155,7 @@ export async function userItemIds(page: Page, sessionId: string, limit = 200): P ids.unshift( ...read.page.items .filter((item) => item.body?.kind === 'message' && item.body.role === 'user') - .map((item) => item.id) + .map((item) => item.itemId) ) if (!read.page.hasOlder || ids.length >= limit) { return ids.slice(-limit) @@ -212,17 +212,26 @@ export async function waitForWorking( working: boolean, timeoutMs = 120_000 ): Promise { - const handle = await page.waitForFunction( - (working) => { - const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( - (candidate) => candidate.checkVisibility() + const handle = await page + .waitForFunction( + (working) => { + const root = [...document.querySelectorAll('[data-native-chat-root="true"]')].find( + (candidate) => candidate.checkVisibility() + ) + return root?.getAttribute('data-native-chat-working') === String(working) + ? performance.now() + : false + }, + working, + { timeout: timeoutMs, polling: 16 } + ) + .catch(async (error: unknown) => { + throw new Error( + `working=${working} not shown in ${timeoutMs} ms (${await visibleChatState(page)})`, + { + cause: error + } ) - return root?.getAttribute('data-native-chat-working') === String(working) - ? performance.now() - : false - }, - working, - { timeout: timeoutMs, polling: 16 } - ) + }) return (await handle.jsonValue()) as number } From 91225180577feaefa62ff8a50239f9599e0d5c94 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:52:13 -0800 Subject: [PATCH 07/15] Activate 4a harness chat tabs through their tab group and seed enough smoke filler chats for the unopened-chat row --- .../native-chat/native-chat-4a-ops-lifecycle.ts | 3 +++ .../native-chat/native-chat-4a-ops-reads.ts | 2 +- .../native-chat/native-chat-4a-session.ts | 15 ++++++++++++--- .../native-chat/native-chat-4a-workloads.ts | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts index c395407b1ce..7ea3de68e48 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts @@ -218,6 +218,9 @@ export async function openRows(ctx: RunContext): Promise { context }) const untouched = fillerSessionIds.slice(3, -1) + if (untouched.length === 0) { + unopened.skipped = 'profile has no filler chat left untouched by earlier families' + } await sampled(ctx, unopened, Math.min(ctx.workload.samples, untouched.length), (index) => { const chat = { sessionId: untouched[index]!, worktreeId: workspace.worktreeIds[1] } const chatIndex = fillerSessionIds.indexOf(chat.sessionId) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts index 8846cdf7b38..1b59de772a4 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts @@ -15,7 +15,7 @@ import { createChat, history, mutate, userItemIds, waitForIdle } from './native- const bytesOf = (value: unknown): number => Buffer.byteLength(JSON.stringify(value)) -type Item = { id: string; body?: { kind?: string } } +type Item = { itemId: string; body?: { kind?: string } } async function read( ctx: RunContext, diff --git a/config/scripts/performance/native-chat/native-chat-4a-session.ts b/config/scripts/performance/native-chat/native-chat-4a-session.ts index c886b2d7bb2..3bc5a0c0909 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-session.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-session.ts @@ -179,15 +179,24 @@ export async function openChatTab( { timeout: 60_000 } ) const id = (await tabId.jsonValue()) as string - await page.evaluate( + // Unified chat tabs activate through their group; a worktree switch can restore another tab first, so re-apply until it holds. + await page.waitForFunction( ({ worktreeId, id }) => { const state = window.__store!.getState() + const group = (state.groupsByWorktree[worktreeId] ?? []).find((candidate) => + candidate.tabOrder.includes(id) + ) + if (state.activeWorktreeId === worktreeId && group?.activeTabId === id) { + return true + } if (state.activeWorktreeId !== worktreeId) { state.setActiveWorktree(worktreeId) } - state.setActiveTab(id) + window.__store!.getState().activateTab(id, { worktreeId }) + return false }, - { worktreeId, id } + { worktreeId, id }, + { timeout: 30_000, polling: 100 } ) } diff --git a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts index 1961ce0a7bc..532480c824f 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts @@ -64,7 +64,7 @@ export const WORKLOADS: Record = { promptBytes: 300, replyBytes: 1500, queuedEvery: 10, - fillerChats: 3, + fillerChats: 6, fillerTurns: 1 }, stream: { cases: streamCases([MiB / 4]), chunkBytes: 16_384, intervalMs: 1, samples: 1 }, From 989d4bbfca5a777f31de33120b944b6bb1b75d84 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:07:53 -0800 Subject: [PATCH 08/15] Treat refusals carried inside a successful 4a harness RPC result as refusals --- .../native-chat/native-chat-4a-probes.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-probes.ts b/config/scripts/performance/native-chat/native-chat-4a-probes.ts index 48bdd599775..1c21792c1f7 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-probes.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-probes.ts @@ -23,11 +23,18 @@ export async function rpc( }, { method, params } ) - if (!answer.reply.ok && !options.allowRefusal) { - throw new Error(`${method} refused: ${JSON.stringify(answer.reply.error).slice(0, 400)}`) + // Some writes (create, fork) refuse inside a successful call: `{ ok: false, refusal }`. + const result = answer.reply.ok + ? (answer.reply.result as { ok?: unknown; refusal?: unknown }) + : null + const inResult = result?.ok === false && 'refusal' in result + const refused = !answer.reply.ok || inResult + if (refused && !options.allowRefusal) { + const reason = answer.reply.ok ? result?.refusal : answer.reply.error + throw new Error(`${method} refused: ${JSON.stringify(reason).slice(0, 400)}`) } const value = (answer.reply.ok ? answer.reply.result : answer.reply.error) as T - return { value, refused: !answer.reply.ok, startedAt: answer.startedAt, hostMs: answer.hostMs } + return { value, refused, startedAt: answer.startedAt, hostMs: answer.hostMs } } /** Install once per page: records when text first appears in the chat root and every transcript mutation batch. */ From cd48c33e9c77639b93e5e1cbb6e99a1215b02ec2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:09:08 -0800 Subject: [PATCH 09/15] Give the 4a stub's user echo the uuid its transcript row records --- .../native-chat/native-chat-4a-provider-stub.cjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs index 8155ded9d94..12de18656f5 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs +++ b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs @@ -196,8 +196,10 @@ async function startTurn(frame) { const controller = new AbortController() current = controller // The CLI's --replay-user-messages echo, which opens the turn. - remember({ type: 'user', uuid: frame.uuid ?? randomUUID(), message: frame.message }) - await write({ ...frame, isReplay: true }) + // One uuid for the echo and the transcript row, so fork and rewind can find this prompt. + const echo = { ...frame, uuid: frame.uuid ?? randomUUID(), session_id: sessionId, isReplay: true } + remember({ type: 'user', uuid: echo.uuid, message: frame.message }) + await write(echo) const content = frame.message?.content ?? frame.message const prompt = Array.isArray(content) ? content.map((block) => block?.text ?? '').join('\n') From 7768407d7fa4be569effdcff0d9f5847315a7e6a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:11:41 -0800 Subject: [PATCH 10/15] Chain the 4a stub's resumed transcript to the last message, past a fork copy's title row --- .../native-chat/native-chat-4a-provider-stub.cjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs index 12de18656f5..524bad5a412 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs +++ b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs @@ -35,7 +35,11 @@ fs.mkdirSync(path.dirname(transcriptPath), { recursive: true }) let parentUuid = null if (fs.existsSync(transcriptPath)) { const lines = fs.readFileSync(transcriptPath, 'utf8').trim().split('\n').filter(Boolean) - parentUuid = lines.length > 0 ? (JSON.parse(lines.at(-1)).uuid ?? null) : null + // Chain to the last message, as the CLI does: a fork copy ends with a parentless custom-title row. + const last = lines + .map((line) => JSON.parse(line)) + .findLast((row) => (row.type === 'user' || row.type === 'assistant') && !row.isSidechain) + parentUuid = last?.uuid ?? null } // Fork and rewind read the provider's own transcript, so every user/assistant frame is also saved there. From d3d106e4d4fcdd83a606c1548120486daa80d74b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:32:20 -0800 Subject: [PATCH 11/15] Fork the 4a harness's large chat through its latest response, not its latest prompt --- .../native-chat/native-chat-4a-ops-lifecycle.ts | 6 +++--- .../native-chat/native-chat-4a-ops-reads.ts | 6 +++--- .../performance/native-chat/native-chat-4a-session.ts | 10 ++++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts index 7ea3de68e48..b9f54755bc6 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts @@ -23,7 +23,7 @@ import { mutate, openChatTab, send, - userItemIds, + messageItemIds, waitForControls, waitForDurable, waitForIdle @@ -292,7 +292,7 @@ export async function rewindRows(ctx: RunContext): Promise { sessionId: largeSessionId, worktreeId: workspace.worktreeIds[0] }) - const [tip] = (await userItemIds(page(ctx), largeSessionId, 1)).slice(-1) + const [tip] = (await messageItemIds(page(ctx), largeSessionId, 'user', 1)).slice(-1) return rewindOnce(tip!, marker) }) const cut = ctx.workload.rewind.deepCutTurns @@ -304,7 +304,7 @@ export async function rewindRows(ctx: RunContext): Promise { endToEndOnly }) await sampled(ctx, deep, ctx.workload.rewind.deepSamples, async () => { - const ids = await userItemIds(page(ctx), largeSessionId, cut + 1) + const ids = await messageItemIds(page(ctx), largeSessionId, 'user', cut + 1) const tail = JSON.stringify( (await history(page(ctx), largeSessionId, 'tail', 3)).page.items ).match(/4A-TAIL-[\w-]+/)?.[0] diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts index 1b59de772a4..14cace375d3 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts @@ -11,7 +11,7 @@ import { tagFor, type RunContext } from './native-chat-4a-run-context' -import { createChat, history, mutate, userItemIds, waitForIdle } from './native-chat-4a-session' +import { createChat, history, messageItemIds, mutate, waitForIdle } from './native-chat-4a-session' const bytesOf = (value: unknown): number => Buffer.byteLength(JSON.stringify(value)) @@ -111,7 +111,7 @@ export async function outlineAndForkRows(ctx: RunContext): Promise { read(ctx, 'agentSession.conversationOutline', { sessionId }) ) - const [tip] = (await userItemIds(page(ctx), sessionId, 1)).slice(-1) + const [tip] = (await messageItemIds(page(ctx), sessionId, 'assistant', 1)).slice(-1) const fork = addRow(ctx, { family: 'outline', operation: 'tip fork (create forkFrom)', @@ -120,7 +120,7 @@ export async function outlineAndForkRows(ctx: RunContext): Promise { endToEndOnly: 'local validation and provider copy are one host span' }) if (!tip) { - fork.skipped = 'no user item on the large chat' + fork.skipped = 'no response on the large chat' return } await sampled(ctx, fork, Math.min(ctx.workload.samples, 10), () => diff --git a/config/scripts/performance/native-chat/native-chat-4a-session.ts b/config/scripts/performance/native-chat/native-chat-4a-session.ts index 3bc5a0c0909..241054f0930 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-session.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-session.ts @@ -147,14 +147,20 @@ export async function waitForIdle(page: Page, sessionId: string, timeoutMs = 120 throw new Error(`${sessionId}: still active after ${timeoutMs} ms`) } -export async function userItemIds(page: Page, sessionId: string, limit = 200): Promise { +/** Message item ids of one role, oldest first: rewind cuts at a user prompt, fork copies through a response. */ +export async function messageItemIds( + page: Page, + sessionId: string, + role: 'user' | 'assistant', + limit = 200 +): Promise { const ids: string[] = [] let cursor: unknown for (;;) { const read = await history(page, sessionId, cursor ? 'before' : 'tail', 200, cursor) ids.unshift( ...read.page.items - .filter((item) => item.body?.kind === 'message' && item.body.role === 'user') + .filter((item) => item.body?.kind === 'message' && item.body.role === role) .map((item) => item.itemId) ) if (!read.page.hasOlder || ids.length >= limit) { From 8302b87a15bf3c550499052a4b2a2ec929bfbee7 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:51:09 -0800 Subject: [PATCH 12/15] Cover the 4a conversion, history-shape and maintenance-debt cases through outside observation Conversion: seed a medium chat (310 chats on the full profile) and refuse a cached seed from another recipe. Observe the candidate's existing epoch-installed ready boundary with a read-only poll of journal_chat_state. An idle seed copy yields launch to every chat ready and the per-size ready-mark intervals; the families copy verifies an unconverted first open and waits for every chat ready before any steady-state family. History/detail: the stub emits ordinary Claude subagent frames (interleaved and sparse-root corpora) and a shapes turn (huge completed tool originals, one above the detail reply cap, an oversized ordinary message). Each named shape is verified through the real history/detail route before its samples count; diff items stay explicit missing evidence because the Claude adapter writes none. Maintenance: write and rewind large completed rows on a disposable seeded chat under a held outside reader, then measure other-chat sends during the hold, after release and while freed pages are reclaimed, from read-only page/WAL state. Retired-epoch pruning is skipped unless such rows are observed. --- .../native-chat/native-chat-4a-comparison.ts | 95 +++--- .../native-chat/native-chat-4a-fixture.ts | 132 +++++--- .../native-chat-4a-ops-conversion.ts | 310 ++++++++++++++++++ .../native-chat-4a-ops-lifecycle.ts | 132 +------- .../native-chat-4a-ops-maintenance.ts | 215 ++++++++++++ .../native-chat/native-chat-4a-ops-reads.ts | 12 +- .../native-chat/native-chat-4a-ops-shapes.ts | 270 +++++++++++++++ .../native-chat/native-chat-4a-ops-stream.ts | 7 +- .../native-chat-4a-provider-stub.cjs | 132 +++++++- .../native-chat/native-chat-4a-report.ts | 37 ++- .../native-chat-4a-sqlite-observer.ts | 228 +++++++++++++ .../native-chat/native-chat-4a-workloads.ts | 89 ++++- 12 files changed, 1411 insertions(+), 248 deletions(-) create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-conversion.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-maintenance.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts create mode 100644 config/scripts/performance/native-chat/native-chat-4a-sqlite-observer.ts diff --git a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts index b847991f978..3a65cdf5203 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts @@ -16,13 +16,7 @@ import { parseArgs } from 'node:util' import type { Arm } from './native-chat-4a-arm' import type { Manifest } from './native-chat-4a-fixture' import type { RunContext } from './native-chat-4a-run-context' -import { - armMarkdown, - comparisonMarkdown, - summarizeRow, - type ArmResult, - type Row -} from './native-chat-4a-report' +import { armMarkdown, writeComparison, type ArmResult, type Row } from './native-chat-4a-report' import { conversionToolOutputBytes, dryRunWorkload, @@ -65,30 +59,8 @@ const workload = workloadFor( ) const log = (line: string): void => console.error(`[4a ${new Date().toISOString()}] ${line}`) -function writeComparison(out: string, results: ArmResult[]): void { - const [base, next] = results - if (!base || !next) { - return - } - const name = `4a-${base.sha.slice(0, 8)}-vs-${next.sha.slice(0, 8)}` - const summaries = results.map((result) => ({ - ...result, - summaries: result.rows.map((row: Row) => ({ ...row, summary: summarizeRow(row) })) - })) - const machine = { - platform: process.platform, - release: os.release(), - cpu: os.cpus()[0]?.model, - cpus: os.cpus().length, - memoryBytes: os.totalmem() - } - writeFileSync( - path.join(out, `${name}.json`), - `${JSON.stringify({ workload, machine, arms: summaries }, null, 2)}\n` - ) - writeFileSync(path.join(out, `${name}.md`), comparisonMarkdown(base, next)) - log(`wrote ${path.join(out, `${name}.md`)}`) -} +const seededChatCount = (manifest: Manifest): number => + 1 + Number(manifest.mediumSessionId !== null) + manifest.fillerSessionIds.length async function runArm(input: { arm: Arm @@ -102,28 +74,37 @@ async function runArm(input: { const { copyProfile, launchArm, stopArm } = await import('./native-chat-4a-arm') const { resetTags } = await import('./native-chat-4a-run-context') const lifecycle = await import('./native-chat-4a-ops-lifecycle') + const conversion = await import('./native-chat-4a-ops-conversion') + const { ConversionObserver } = await import('./native-chat-4a-sqlite-observer') const queue = await import('./native-chat-4a-ops-queue') const retained = await import('./native-chat-4a-ops-retained') const stream = await import('./native-chat-4a-ops-stream') const reads = await import('./native-chat-4a-ops-reads') + const shapes = await import('./native-chat-4a-ops-shapes') const profileDir = path.join(input.out, 'profile') const rows: Row[] = [] const startedAt = new Date().toISOString() + // Each seed copy's first launch converts on the candidate; the observer starts before it, reading only. const onProfile = async ( seedDir: string, manifest: typeof input.manifest, - body: (ctx: RunContext) => Promise + phase: string, + body: (ctx: RunContext, observer: InstanceType) => Promise ) => { if (existsSync(profileDir)) { renameSync(profileDir, path.join(input.out, `stale-profile-${Date.now()}`)) } copyProfile(seedDir, profileDir) resetTags() + const observer = new ConversionObserver(profileDir) const open = () => launchArm(input.arm, profileDir, input.stubCommand) const ctx = { arm: input.arm, - launch: await open(), + launch: await open().catch(async (error: unknown) => { + await observer.stop() + throw error + }), workload, manifest, rows, @@ -135,14 +116,15 @@ async function runArm(input: { ctx.launch = await open() } try { - await body(ctx) + await body(ctx, observer) } finally { + await observer.stop() await stopArm(ctx.launch) renameSync( profileDir, path.join( input.out, - `after-${input.arm.name}-${manifest.toolOutputBytes}-${input.arm.sha.slice(0, 8)}` + `after-${input.arm.name}-${phase}-${manifest.toolOutputBytes}-${input.arm.sha.slice(0, 8)}` ) ) } @@ -170,19 +152,25 @@ async function runArm(input: { } } - await onProfile(input.seedDir, input.manifest, async (ctx) => { - const seedFiles = lifecycle.seededDbFiles(input.seedDir) - // First: the seeded tail markers and unconverted state exist only before any other family writes. - await run('conversion', () => - lifecycle.firstLaunchRows( - ctx, - seedFiles, - `${workload.history.largeTurns}-turn/${workload.history.fillerChats + 1}-chat profile` - ) + const label = `${workload.history.largeTurns}-turn/${seededChatCount(input.manifest)}-chat profile` + if (input.families.has('conversion')) { + // A copy of its own, left idle, so the sweep's whole duration and per-chat intervals carry no foreground work. + await onProfile(input.seedDir, input.manifest, 'sweep', (ctx, observer) => + run('conversion', () => conversion.sweepRows(ctx, observer, label)) ) + } + await onProfile(input.seedDir, input.manifest, 'families', async (ctx, observer) => { + const seedFiles = conversion.seededDbFiles(input.seedDir) + // First: the seeded tail markers and unconverted state exist only before any other family writes. + await run('conversion', () => conversion.firstLaunchRows(ctx, seedFiles, label, observer)) + await conversion.awaitSweep(ctx, observer, label) await run('send', () => queue.sendAndQueueRows(ctx)) await run('stream', () => stream.streamingRows(ctx)) - await run('history', () => reads.historyRows(ctx)) + await run('history', async () => { + await reads.historyRows(ctx) + await shapes.subagentRows(ctx) + await shapes.itemShapeRows(ctx) + }) await run('outline', () => reads.outlineAndForkRows(ctx)) await run('records', () => reads.recordRows(ctx)) await run('retained', () => retained.retainedAdmissionRows(ctx)) @@ -191,10 +179,15 @@ async function runArm(input: { }) if (input.largeSource && input.families.has('conversion')) { const source = input.largeSource - const label = `${(workload.conversionSourceBytes / 1024 ** 3).toFixed(0)} GiB source` - await onProfile(source.seedDir, source.manifest, (ctx) => + const sourceLabel = `${(workload.conversionSourceBytes / 1024 ** 3).toFixed(0)} GiB source` + await onProfile(source.seedDir, source.manifest, 'large-source', (ctx, observer) => run('conversion', () => - lifecycle.firstLaunchRows(ctx, lifecycle.seededDbFiles(source.seedDir), label) + conversion.firstLaunchRows( + ctx, + conversion.seededDbFiles(source.seedDir), + sourceLabel, + observer + ) ) ) } @@ -228,7 +221,7 @@ async function main(): Promise { .filter((file) => /^arm-.*\.json$/.test(file)) .map((file) => JSON.parse(readFileSync(path.join(out, file), 'utf8')) as ArmResult) .sort((a, b) => a.startedAt.localeCompare(b.startedAt)) - writeComparison(out, results) + writeComparison(out, workload, results, log) return } const families = new Set(values.families.split(',').map((family) => family.trim()) as Family[]) @@ -271,7 +264,7 @@ async function main(): Promise { values['large-source'] && workload.conversionSourceBytes > 0 ? await seed(conversionToolOutputBytes(workload), { ...workload, - history: { ...workload.history, fillerChats: 3 } + history: { ...workload.history, fillerChats: 5, mediumTurns: 0 } }) : null const results: ArmResult[] = [] @@ -292,7 +285,7 @@ async function main(): Promise { writeFileSync(path.join(out, `arm-${arm.name}-${arm.sha.slice(0, 12)}.md`), armMarkdown(result)) results.push(result) } - writeComparison(out, results) + writeComparison(out, workload, results, log) const skipped = results.flatMap((result) => result.rows .filter((row) => row.skipped) diff --git a/config/scripts/performance/native-chat/native-chat-4a-fixture.ts b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts index eabedcc6236..d21de95034a 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-fixture.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts @@ -1,6 +1,8 @@ // Seed the retained-history profile once through the seed arm's own host, so every arm starts from the same logical data. +import { createHash } from 'node:crypto' import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs' import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' import { launchArm, prepareWorkspace, @@ -10,22 +12,22 @@ import { } from './native-chat-4a-arm' import { rpc, storageBytes } from './native-chat-4a-probes' import { createChat, history, send, waitForDurable, waitForIdle } from './native-chat-4a-session' -import { - fillerTurn, - largeHistoryTurns, - expectedLargeItems, - type Workload -} from './native-chat-4a-workloads' +import { expectedItems, fillerTurn, historyTurns, type Workload } from './native-chat-4a-workloads' export type Manifest = { seedArm: string seedSha: string workload: Workload['profile'] + /** Digest of the seeded history recipe; a cached seed from another recipe is refused, never reused. */ + recipe: string toolOutputBytes: number workspace: Workspace largeSessionId: string + /** Null when the recipe seeds no medium chat (the large-source profile). */ + mediumSessionId: string | null fillerSessionIds: string[] largeItems: number + mediumItems: number storageBytes: number seedMs: number } @@ -33,10 +35,7 @@ export type Manifest = { const SLEEP_FILLERS_EVERY = 20 /** Count the large chat's rows by paging back from the tail; refuses a fixture that seeded fewer rows than its recipe. */ -async function countItems( - page: Awaited>['page'], - sessionId: string -): Promise { +async function countItems(page: Page, sessionId: string): Promise { let count = 0 let cursor: unknown for (;;) { @@ -49,6 +48,49 @@ async function countItems( } } +const recipeOf = (workload: Workload, toolOutputBytes: number): string => + createHash('sha256') + .update(JSON.stringify({ history: workload.history, toolOutputBytes })) + .digest('hex') + .slice(0, 16) + +/** Send a recipe chat's turns; a queued turn goes while its predecessor still runs, leaving released queue history behind. */ +async function seedRecipeChat( + page: Page, + arm: Arm, + worktreeId: string, + workload: Workload, + chat: 'large' | 'medium', + log: (line: string) => void +): Promise<{ sessionId: string; items: number }> { + const created = await createChat(page, arm, worktreeId) + const turns = chat === 'large' ? workload.history.largeTurns : workload.history.mediumTurns + let turn = 0 + for (const next of historyTurns(workload, chat)) { + if (turn > 0 && !next.queued) { + await waitForDurable(page, created.sessionId, `4A-TAIL-${chat}-${turn - 1}`) + } + await send(page, arm, created.sessionId, next.prompt) + turn += 1 + if (turn % 250 === 0) { + log(`seed: ${chat} chat ${turn}/${turns} turns`) + } + } + await waitForDurable(page, created.sessionId, `4A-TAIL-${chat}-${turn - 1}`, 600_000) + await waitForIdle(page, created.sessionId, 600_000) + const items = await countItems(page, created.sessionId) + // Hosts may group a call with its result; fewer than prompt+reply per turn means turns were lost. + if (items < turns * 2) { + throw new Error(`seed: ${chat} chat holds ${items} rows for ${turns} turns`) + } + if (items < expectedItems(turns, workload)) { + log( + `seed: ${chat} chat holds ${items} visible rows; recipe writes ${expectedItems(turns, workload)}` + ) + } + return { sessionId: created.sessionId, items } +} + export async function seedProfile(input: { arm: Arm workload: Workload @@ -61,32 +103,44 @@ export async function seedProfile(input: { }): Promise { const seedDir = path.join(input.runDir, `seed-${input.workload.profile}-${input.toolOutputBytes}`) const manifestPath = `${seedDir}.json` + const recipe = recipeOf(input.workload, input.toolOutputBytes) if (existsSync(manifestPath) && existsSync(seedDir)) { - return JSON.parse(readFileSync(manifestPath, 'utf8')) as Manifest + const cached = JSON.parse(readFileSync(manifestPath, 'utf8')) as Partial + if (cached.recipe !== recipe) { + throw new Error( + `${manifestPath} was seeded from another recipe (${cached.recipe ?? 'none recorded'}, now ${recipe}); move it and ${seedDir} aside or use a new --out` + ) + } + return cached as Manifest } const started = Date.now() const launch = await launchArm(input.arm, input.profileDir, input.stubCommand) try { const { page } = launch const workspace = await prepareWorkspace(page, input.repoPath, input.stubCommand) - const large = await createChat(page, input.arm, workspace.worktreeIds[0]) - let turn = 0 - for (const next of largeHistoryTurns({ + const recipeWorkload = { ...input.workload, history: { ...input.workload.history, toolOutputBytes: input.toolOutputBytes } - })) { - // A queued turn is sent while its predecessor still runs, leaving released queue history behind. - if (turn > 0 && !next.queued) { - await waitForDurable(page, large.sessionId, `4A-TAIL-large-${turn - 1}`) - } - await send(page, input.arm, large.sessionId, next.prompt) - turn += 1 - if (turn % 250 === 0) { - input.log(`seed: large chat ${turn}/${input.workload.history.largeTurns} turns`) - } } - await waitForDurable(page, large.sessionId, `4A-TAIL-large-${turn - 1}`, 600_000) - await waitForIdle(page, large.sessionId, 600_000) + const large = await seedRecipeChat( + page, + input.arm, + workspace.worktreeIds[0], + recipeWorkload, + 'large', + input.log + ) + const medium = + input.workload.history.mediumTurns > 0 + ? await seedRecipeChat( + page, + input.arm, + workspace.worktreeIds[0], + recipeWorkload, + 'medium', + input.log + ) + : null const fillerSessionIds: string[] = [] const workspaceId = workspace.worktreeIds[1] for (let chat = 0; chat < input.workload.history.fillerChats; chat += 1) { @@ -111,27 +165,18 @@ export async function seedProfile(input: { input.log(`seed: ${chat + 1}/${input.workload.history.fillerChats} filler chats`) } } - const largeItems = await countItems(page, large.sessionId) - // Hosts may group a call with its result; fewer than prompt+reply per turn means turns were lost. - if (largeItems < input.workload.history.largeTurns * 2) { - throw new Error( - `seed: large chat holds ${largeItems} rows for ${input.workload.history.largeTurns} turns` - ) - } - if (largeItems < expectedLargeItems(input.workload)) { - input.log( - `seed: large chat holds ${largeItems} visible rows; recipe writes ${expectedLargeItems(input.workload)}` - ) - } const manifest: Manifest = { seedArm: input.arm.name, seedSha: input.arm.sha, workload: input.workload.profile, + recipe, toolOutputBytes: input.toolOutputBytes, workspace, largeSessionId: large.sessionId, + mediumSessionId: medium?.sessionId ?? null, fillerSessionIds, - largeItems, + largeItems: large.items, + mediumItems: medium?.items ?? 0, storageBytes: 0, seedMs: 0 } @@ -149,3 +194,12 @@ export async function seedProfile(input: { export const seedDirFor = (runDir: string, manifest: Manifest): string => path.join(runDir, `seed-${manifest.workload}-${manifest.toolOutputBytes}`) + +/** The untouched filler the background sweep reaches last (it converts in ascending session_id order). */ +export function unconvertedOpenTarget(manifest: Manifest): string | null { + return manifest.fillerSessionIds.slice(3, -1).sort().at(-1) ?? null +} + +/** Every chat the seed made, in the sweep's order. */ +export const seededChats = (manifest: Manifest): string[] => + [manifest.largeSessionId, manifest.mediumSessionId ?? [], manifest.fillerSessionIds].flat().sort() diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-conversion.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-conversion.ts new file mode 100644 index 00000000000..b4b80ebfaf5 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-conversion.ts @@ -0,0 +1,310 @@ +// 4a one-time conversion rows, reported apart from steady state. Ready is the candidate's existing epoch-installed boundary, +// observed from outside through a read-only poll of journal_chat_state (see native-chat-4a-sqlite-observer.ts). +import path from 'node:path' +import { seededChats, unconvertedOpenTarget } from './native-chat-4a-fixture' +import { openTo, type Chat } from './native-chat-4a-ops-lifecycle' +import { samplePressure, storageBytes } from './native-chat-4a-probes' +import type { Row, Sample } from './native-chat-4a-report' +import { addRow, page, sampled, tagFor, type RunContext } from './native-chat-4a-run-context' +import { send, waitForIdle } from './native-chat-4a-session' +import { + CHAT_STATE_CODE, + seededDbFiles, + type ChatState, + type ConversionObserver +} from './native-chat-4a-sqlite-observer' +import { replyPrompt } from './native-chat-4a-workloads' + +const SWEEP_TIMEOUT_MS = 2 * 60 * 60_000 +const NO_DOMAIN_DATABASE = + 'this arm created no agent-session-domain.db by session ready: nothing converts' + +export { seededDbFiles } from './native-chat-4a-sqlite-observer' + +function observation(observer: ConversionObserver): Record { + return { + observedDatabase: observer.databaseFile + ? path.basename(observer.databaseFile) + : 'none (no agent-session-domain.db)', + readyBoundary: 'journal_chat_state.epoch installed, first poll that sees it', + observationResolutionMs: observer.resolutionMs + } +} + +/** Open a chat for the first time, reading its conversion state before the open and once the open has been requested. */ +async function firstOpen( + ctx: RunContext, + observer: ConversionObserver, + chat: Chat, + marker: string +): Promise<{ sample: Sample; atOpen: ChatState }> { + const before = observer.state(chat.sessionId) + let atOpen: ChatState = before + const requestedAt = Date.now() + const sample = await openTo(ctx, chat, marker, () => { + // Ready marks never revert, so a pending read here proves the chat was unconverted when the open began. + atOpen = observer.state(chat.sessionId) + return { + stateBeforeOpen: CHAT_STATE_CODE[before], + stateAtOpen: CHAT_STATE_CODE[atOpen] + } + }) + observer.state(chat.sessionId) + const readyAt = observer.readyAt.get(chat.sessionId) + sample.extra = { + ...sample.extra, + readyMarkMs: readyAt === undefined || before === 'ready' ? Number.NaN : readyAt - requestedAt + } + return { sample, atOpen } +} + +/** + * The arm's first launch on the seeded profile. On the candidate this is the accepted one-time conversion; on main it is a + * cold open of the same data, so the rows line up but are reported apart from steady state. + */ +export async function firstLaunchRows( + ctx: RunContext, + seedDbFiles: string[], + label: string, + observer: ConversionObserver +): Promise { + const { workspace, largeSessionId, fillerSessionIds } = ctx.manifest + const context = { + profile: label, + largeItems: ctx.manifest.largeItems, + seededBytes: ctx.manifest.storageBytes, + chats: seededChats(ctx.manifest).length, + ...observation(observer) + } + const init = addRow(ctx, { + family: 'conversion', + operation: 'launch to session ready', + variant: label, + context + }) + init.samples.push({ e2eMs: ctx.launch.readyEpochMs - ctx.launch.launchedAtEpochMs }) + const stopPressure = samplePressure(ctx.launch.app, ctx.launch.userDataDir, 500) + const target = unconvertedOpenTarget(ctx.manifest) + const unconverted = addRow(ctx, { + family: 'conversion', + operation: 'first open, unconverted chat (last in sweep order)', + variant: label, + context: { ...context, stateCodes: 'NaN no domain db, 0 pending, 1 importing, 2 ready' } + }) + if (!target) { + unconverted.skipped = 'profile has no untouched filler chat to open unconverted' + } else { + const index = fillerSessionIds.indexOf(target) + await sampled(ctx, unconverted, 1, async () => { + const { sample, atOpen } = await firstOpen( + ctx, + observer, + { sessionId: target, worktreeId: workspace.worktreeIds[1] }, + `4A-TAIL-filler-${index}-${ctx.workload.history.fillerTurns - 1}` + ) + if (atOpen === 'ready' || atOpen === 'uncatalogued') { + addRow(ctx, { + family: 'conversion', + operation: `first open, chat ${atOpen === 'ready' ? 'converted' : 'uncatalogued'} before open`, + variant: label, + context, + samples: [sample] + }) + throw new Error(`target was ${atOpen} when the open began; not an unconverted open`) + } + return sample + }) + } + const small: Chat = { sessionId: fillerSessionIds[0]!, worktreeId: workspace.worktreeIds[1] } + const smallOpen = addRow(ctx, { + family: 'conversion', + operation: 'first open, small chat', + variant: label, + context + }) + const openOnce = (chat: Chat, marker: string) => async () => + (await firstOpen(ctx, observer, chat, marker)).sample + await sampled( + ctx, + smallOpen, + 1, + openOnce(small, `4A-TAIL-filler-0-${ctx.workload.history.fillerTurns - 1}`) + ) + const largeOpen = addRow(ctx, { + family: 'conversion', + operation: 'first open, large chat, with sends to the opened small chat', + variant: label, + context: { ...context, concurrent: 'sends to the already-opened small chat every 250 ms' } + }) + let opening = true + const concurrent: number[] = [] + const sends = (async () => { + while (opening) { + concurrent.push( + (await send(page(ctx), ctx.arm, small.sessionId, replyPrompt(tagFor('during')))).hostMs + ) + await new Promise((resolve) => setTimeout(resolve, 250)) + } + })() + await sampled( + ctx, + largeOpen, + 1, + openOnce( + { sessionId: largeSessionId, worktreeId: workspace.worktreeIds[0] }, + `4A-TAIL-large-${ctx.workload.history.largeTurns - 1}` + ) + ) + opening = false + await sends + const during = addRow(ctx, { + family: 'conversion', + operation: 'small-chat send during large first open', + variant: label, + context + }) + during.samples.push(...concurrent.map((hostMs) => ({ hostMs }))) + const pressure = await stopPressure() + const remaining = seededDbFiles(ctx.launch.userDataDir) + const disk = addRow(ctx, { + family: 'conversion', + operation: 'disk', + variant: label, + context: { + retainedOriginalFiles: + seedDbFiles.filter((file) => remaining.includes(file)).join(' ') || 'none', + newFiles: remaining.filter((file) => !seedDbFiles.includes(file)).join(' ') || 'none' + } + }) + disk.samples.push({ + extra: { + peakTotalBytes: pressure.peakTotalBytes, + afterTotalBytes: storageBytes(ctx.launch.userDataDir).total, + peakMainRssBytes: pressure.peakMainRssBytes + } + }) + await waitForIdle(page(ctx), small.sessionId) +} + +/** Steady-state families start only once every seeded chat is ready, so import never mixes into their samples. */ +export async function awaitSweep( + ctx: RunContext, + observer: ConversionObserver, + label: string +): Promise { + // Polling stops here, so steady-state families carry no observer reads. + try { + if (!observer.databaseFile) { + return + } + const started = Date.now() + const done = await observer.allReady(seededChats(ctx.manifest), SWEEP_TIMEOUT_MS) + const row = addRow(ctx, { + family: 'conversion', + operation: 'wait for every chat ready before steady-state families', + variant: label, + context: observation(observer) + }) + if (!done) { + row.skipped = `not every seeded chat was ready after ${SWEEP_TIMEOUT_MS} ms; later families overlap conversion` + ctx.log(`${ctx.arm.name}: ${row.skipped}`) + return + } + row.samples.push({ e2eMs: Date.now() - started }) + } finally { + await observer.stop() + } +} + +/** Idle launch on its own seed copy: launch to the last ready mark, and each chat's ready-mark interval in observed order. */ +export async function sweepRows( + ctx: RunContext, + observer: ConversionObserver, + label: string +): Promise { + const { manifest, launch } = ctx + const chats = seededChats(manifest) + const context = { + profile: label, + chats: chats.length, + activity: 'none: the app stays idle until the last ready mark', + endPoints: + "previous chat's ready mark → this chat's ready mark in observed order; includes the sweep's idle-boundary waits", + ...observation(observer) + } + const row = (operation: string, extra: Record = {}): Row => + addRow(ctx, { + family: 'conversion', + operation, + variant: label, + context: { ...context, ...extra } + }) + const whole = row('background sweep, launch to every chat ready') + const sized = [ + { name: 'small chats', ids: manifest.fillerSessionIds, items: 'filler recipe' }, + { + name: 'medium chat', + ids: manifest.mediumSessionId ? [manifest.mediumSessionId] : [], + items: manifest.mediumItems + }, + { + name: `${manifest.largeItems}-item chat`, + ids: [manifest.largeSessionId], + items: manifest.largeItems + } + ] + const intervalRows = sized.map((size) => ({ + ...size, + row: row(`background sweep ready-mark interval, ${size.name}`, { items: size.items }) + })) + if (!observer.databaseFile) { + for (const each of [whole, ...intervalRows.map((entry) => entry.row)]) { + each.skipped = NO_DOMAIN_DATABASE + } + return + } + const done = await observer.allReady(chats, SWEEP_TIMEOUT_MS) + const readyAt = (id: string): number => observer.readyAt.get(id)! + const ordered = chats + .filter((id) => observer.readyAt.has(id)) + .sort((a, b) => readyAt(a) - readyAt(b) || (a < b ? -1 : 1)) + const interval = new Map() + let previous = observer.databaseSeenAt + for (const id of ordered) { + interval.set(id, readyAt(id) - previous) + previous = readyAt(id) + } + if (!done) { + whole.skipped = `${ordered.length} of ${chats.length} chats ready after ${SWEEP_TIMEOUT_MS} ms` + } else { + whole.samples.push({ + e2eMs: readyAt(ordered.at(-1)!) - launch.launchedAtEpochMs, + extra: { + sessionReadyMs: launch.readyEpochMs - launch.launchedAtEpochMs, + storeSeenMs: observer.databaseSeenAt - launch.launchedAtEpochMs, + readyBySessionReady: ordered.filter((id) => readyAt(id) <= launch.readyEpochMs).length, + readyOutOfSweepOrder: ordered.filter((id, index) => index > 0 && id < ordered[index - 1]!) + .length, + catalogRows: observer.catalogRows + } + }) + } + for (const { ids, row: sizeRow, name } of intervalRows) { + const spans = ids.flatMap((id) => (interval.has(id) ? [interval.get(id)!] : [])) + if (ids.length === 0) { + sizeRow.skipped = `profile seeds no ${name}` + continue + } + if (spans.length < ids.length) { + sizeRow.skipped = `${ids.length - spans.length} of ${ids.length} ${name} not seen ready` + continue + } + sizeRow.samples.push(...spans.map((e2eMs) => ({ e2eMs }))) + const total = spans.reduce((sum, span) => sum + span, 0) + sizeRow.context = { + ...sizeRow.context, + totalSpanMs: total, + meanMs: spans.length ? Math.round((total / spans.length) * 100) / 100 : Number.NaN + } + } +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts index b9f54755bc6..db41444fdde 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts @@ -1,14 +1,6 @@ -// 4a process-lifecycle rows: first launch on the seeded profile (the candidate's one-time conversion), cold/warm open, and rewind. -import { readdirSync } from 'node:fs' -import path from 'node:path' -import { - markerSeenAt, - rendererNow, - rpc, - samplePressure, - storageBytes, - watchMarkers -} from './native-chat-4a-probes' +// 4a process-lifecycle rows: cold/warm open and rewind; first launch on the seeded profile lives in the conversion module. +import { unconvertedOpenTarget } from './native-chat-4a-fixture' +import { markerSeenAt, rendererNow, rpc, watchMarkers } from './native-chat-4a-probes' import type { Sample } from './native-chat-4a-report' import { addRow, @@ -30,19 +22,25 @@ import { } from './native-chat-4a-session' import { replyPrompt } from './native-chat-4a-workloads' -type Chat = { sessionId: string; worktreeId: string } +export type Chat = { sessionId: string; worktreeId: string } -/** Open a chat and time it to its known tail marker visible plus controls known. */ -async function openTo(ctx: RunContext, chat: Chat, marker: string): Promise { +/** Open a chat and time it to its known tail marker visible plus controls known; `issued` runs once the open is requested. */ +export async function openTo( + ctx: RunContext, + chat: Chat, + marker: string, + issued?: () => Record +): Promise { return measured(ctx, async () => { await watchMarkers(page(ctx), [marker]) const startedAt = await rendererNow(page(ctx)) await openChatTab(page(ctx), chat.worktreeId, chat.sessionId) + const observed = issued?.() ?? {} const tailAt = await markerSeenAt(page(ctx), marker, 600_000) await waitForControls(page(ctx), 600_000) return { e2eMs: (await rendererNow(page(ctx))) - startedAt, - extra: { tailVisibleMs: tailAt - startedAt } + extra: { tailVisibleMs: tailAt - startedAt, ...observed } } }) } @@ -56,106 +54,6 @@ async function markTail(ctx: RunContext, chat: Chat): Promise { return `4A-TAIL-${tag}` } -const dbFiles = (dir: string): string[] => - readdirSync(dir, { recursive: true, withFileTypes: true }) - .filter( - (entry) => - entry.isFile() && - /\.(db|sqlite)$/.test(entry.name) && - !entry.parentPath.includes(`${path.sep}home`) - ) - .map((entry) => path.relative(dir, path.join(entry.parentPath, entry.name))) - -/** - * The arm's first launch on the seeded profile. On the candidate this is the accepted one-time conversion; on main it is a - * cold open of the same data, so the rows line up but are reported apart from steady state. - */ -export async function firstLaunchRows( - ctx: RunContext, - seedDbFiles: string[], - label: string -): Promise { - const { workspace, largeSessionId, fillerSessionIds } = ctx.manifest - const context = { - profile: label, - largeItems: ctx.manifest.largeItems, - seededBytes: ctx.manifest.storageBytes, - chats: fillerSessionIds.length + 1 - } - const init = addRow(ctx, { - family: 'conversion', - operation: 'launch to session ready', - variant: label, - context - }) - init.samples.push({ e2eMs: ctx.launch.readyEpochMs - ctx.launch.launchedAtEpochMs }) - const stopPressure = samplePressure(ctx.launch.app, ctx.launch.userDataDir, 500) - const small: Chat = { sessionId: fillerSessionIds[0]!, worktreeId: workspace.worktreeIds[1] } - const smallOpen = addRow(ctx, { - family: 'conversion', - operation: 'first open, small chat', - variant: label, - context - }) - await sampled(ctx, smallOpen, 1, () => - openTo(ctx, small, `4A-TAIL-filler-0-${ctx.workload.history.fillerTurns - 1}`) - ) - const largeOpen = addRow(ctx, { - family: 'conversion', - operation: 'first open, large chat, with sends to the opened small chat', - variant: label, - context: { ...context, concurrent: 'sends to the already-opened small chat every 250 ms' } - }) - let opening = true - const concurrent: number[] = [] - const sends = (async () => { - while (opening) { - concurrent.push( - (await send(page(ctx), ctx.arm, small.sessionId, replyPrompt(tagFor('during')))).hostMs - ) - await new Promise((resolve) => setTimeout(resolve, 250)) - } - })() - await sampled(ctx, largeOpen, 1, () => - openTo( - ctx, - { sessionId: largeSessionId, worktreeId: workspace.worktreeIds[0] }, - `4A-TAIL-large-${ctx.workload.history.largeTurns - 1}` - ) - ) - opening = false - await sends - const during = addRow(ctx, { - family: 'conversion', - operation: 'small-chat send during large first open', - variant: label, - context - }) - during.samples.push(...concurrent.map((hostMs) => ({ hostMs }))) - const pressure = await stopPressure() - const remaining = dbFiles(ctx.launch.userDataDir) - const disk = addRow(ctx, { - family: 'conversion', - operation: 'disk', - variant: label, - context: { - retainedOriginalFiles: - seedDbFiles.filter((file) => remaining.includes(file)).join(' ') || 'none', - newFiles: remaining.filter((file) => !seedDbFiles.includes(file)).join(' ') || 'none' - } - }) - disk.samples.push({ - extra: { - peakTotalBytes: pressure.peakTotalBytes, - afterTotalBytes: storageBytes(ctx.launch.userDataDir).total, - peakMainRssBytes: pressure.peakMainRssBytes - } - }) - await waitForIdle(page(ctx), small.sessionId) -} - -export const seededDbFiles = dbFiles - export async function openRows(ctx: RunContext): Promise { const { workspace, largeSessionId, fillerSessionIds } = ctx.manifest const large: Chat = { sessionId: largeSessionId, worktreeId: workspace.worktreeIds[0] } @@ -217,7 +115,9 @@ export async function openRows(ctx: RunContext): Promise { variant: 'small chat', context }) - const untouched = fillerSessionIds.slice(3, -1) + // The conversion family's unconverted-open target is already open. + const target = unconvertedOpenTarget(ctx.manifest) + const untouched = fillerSessionIds.slice(3, -1).filter((id) => id !== target) if (untouched.length === 0) { unopened.skipped = 'profile has no filler chat left untouched by earlier families' } diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-maintenance.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-maintenance.ts new file mode 100644 index 00000000000..6c4d939d3dd --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-maintenance.ts @@ -0,0 +1,215 @@ +// 4a maintenance rows on debt the run creates and observes: large completed rows written then rewound on a disposable seeded +// chat while an outside read-only reader holds a snapshot, its release, and freed pages watched back down, each while another +// chat sends. Page, free-list and WAL state come from read-only SQLite reads; nothing asks the app for timing. +import path from 'node:path' +import type { Sample } from './native-chat-4a-report' +import { addRow, page, tagFor, type RunContext } from './native-chat-4a-run-context' +import { + history, + messageItemIds, + mutate, + openChatTab, + send, + waitForControls, + waitForDurable, + waitForIdle +} from './native-chat-4a-session' +import { chatDatabasePath, holdReader, pageState } from './native-chat-4a-sqlite-observer' +import { debtPrompt, replyPrompt } from './native-chat-4a-workloads' + +// JOURNAL_WAL_BACKPRESSURE_FRAMES on the candidate (journal-checkpoint-owner.ts): writes wait above this many WAL frames. +const BACKPRESSURE_FRAMES = 32_768 +const RELEASE_WATCH_MS = 30_000 +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** Sends to `other` until `done()`, each sample its host span plus the store's WAL bytes and free pages right after. */ +async function sendsUntil( + ctx: RunContext, + other: string, + file: string, + done: () => boolean, + everyMs: number +): Promise { + const started = Date.now() + const samples: Sample[] = [] + while (!done()) { + const answer = await send(page(ctx), ctx.arm, other, replyPrompt(tagFor('maint'))) + const state = pageState(file) + samples.push({ + hostMs: answer.hostMs, + extra: { + sinceStartMs: Date.now() - started, + walBytes: state.walBytes, + freelistPages: state.freelistPages + } + }) + await sleep(everyMs) + } + await waitForIdle(page(ctx), other) + return samples +} + +const peak = (samples: Sample[], key: string): number => + Math.max(0, ...samples.map((sample) => sample.extra?.[key] ?? 0)) + +export async function maintenanceDebtRows(ctx: RunContext): Promise { + const { fillerSessionIds, workspace } = ctx.manifest + const { debtTools, debtToolBytes, holdMs, reclaimWaitMs } = ctx.workload.maintenance + // The neutral chat, which later families only use as an active tab; its debt turn is rewound away again. + const debt = fillerSessionIds[2] ?? fillerSessionIds.at(-1)! + const other = fillerSessionIds[1] ?? fillerSessionIds[0]! + const file = chatDatabasePath(ctx.launch.userDataDir) + const variant = `${debtTools} × ${(debtToolBytes / 1024 / 1024).toFixed(2)} MiB completed rows` + const context = { + observedDatabase: file ? path.basename(file) : 'none', + holdMs, + concurrent: 'small sends to another chat' + } + const row = (operation: string, extra: Record = {}) => + addRow(ctx, { family: 'maintenance', operation, variant, context: { ...context, ...extra } }) + const removal = row('large-row removal (rewind) under a held outside reader', { + endPoints: 'rewind request → host answer; includes any WAL admission wait' + }) + const hold = row('other-chat sends while an outside reader holds a snapshot', { + backpressureFrames: BACKPRESSURE_FRAMES + }) + const release = row('other-chat sends after the outside reader releases', { + watchMs: RELEASE_WATCH_MS + }) + const reclaim = row('other-chat sends while freed pages are reclaimed', { + watchMs: reclaimWaitMs, + reclaimOwner: 'periodic maintenance each minute, at most 1024 single-page units per run' + }) + const retired = row('retired-epoch prune') + const all = [removal, hold, release, reclaim, retired] + if (!file) { + for (const each of all) { + each.skipped = 'no chat SQLite database found in the profile' + } + return + } + try { + await openChatTab(page(ctx), workspace.worktreeIds[1], debt) + await waitForControls(page(ctx)) + const baseline = pageState(file, true) + const tag = tagFor('debt') + await send(page(ctx), ctx.arm, debt, debtPrompt(ctx.workload, tag)) + await waitForDurable(page(ctx), debt, `4A-TAIL-${tag}`, 600_000, 250) + await waitForIdle(page(ctx), debt, 600_000) + const written = pageState(file) + const [cutAt] = (await messageItemIds(page(ctx), debt, 'user', 1)).slice(-1) + const epoch = (await history(page(ctx), debt, 'tail', 1)).page.epoch + + // The reader is released by a timer, whatever the app or the sends are doing. + const releaseReader = holdReader(file) + let releasedAt = Number.NaN + const timer = setTimeout(() => { + releaseReader() + releasedAt = Date.now() + }, holdMs) + let removalHostMs = Number.NaN + try { + // Settled into a value so a refusal during the hold is never an unhandled rejection. + const rewound = mutate(page(ctx), ctx.arm, 'agentSession.rewind', debt, { + itemId: cutAt, + expectedEpoch: epoch + }).then( + (answer) => ({ answer }), + (error: unknown) => ({ error }) + ) + hold.samples.push( + ...(await sendsUntil(ctx, other, file, () => !Number.isNaN(releasedAt), 250)) + ) + const outcome = await rewound + if ('error' in outcome) { + throw outcome.error + } + removalHostMs = outcome.answer.hostMs + } finally { + clearTimeout(timer) + releaseReader() + releasedAt = Number.isNaN(releasedAt) ? Date.now() : releasedAt + } + const peakWal = peak(hold.samples, 'walBytes') + const walFrames = Math.max(0, Math.floor((peakWal - 32) / (baseline.pageSize + 24))) + hold.context = { + ...hold.context, + peakWalBytes: peakWal, + peakWalFrames: walFrames, + reachedBackpressure: Number(walFrames >= BACKPRESSURE_FRAMES) + } + + const atRelease = pageState(file) + let shrunkAt = Number.NaN + release.samples.push( + ...(await sendsUntil( + ctx, + other, + file, + () => { + if (Number.isNaN(shrunkAt) && pageState(file).walBytes < atRelease.walBytes) { + shrunkAt = Date.now() + } + return !Number.isNaN(shrunkAt) || Date.now() - releasedAt > RELEASE_WATCH_MS + }, + 250 + )) + ) + release.context = { ...release.context, walShrinkMs: shrunkAt - releasedAt } + + const removed = pageState(file, true) + removal.samples.push({ + hostMs: removalHostMs, + extra: { + writtenBytes: (written.pageCount - baseline.pageCount) * baseline.pageSize, + freedPages: removed.freelistPages - baseline.freelistPages + } + }) + + const reclaimStarted = Date.now() + let backAt = Number.NaN + reclaim.samples.push( + ...(await sendsUntil( + ctx, + other, + file, + () => { + if (Number.isNaN(backAt) && pageState(file).freelistPages <= baseline.freelistPages) { + backAt = Date.now() + } + return !Number.isNaN(backAt) || Date.now() - reclaimStarted > reclaimWaitMs + }, + 1_000 + )) + ) + const end = pageState(file, true) + reclaim.context = { + ...reclaim.context, + freelistBaseline: baseline.freelistPages, + freelistAtStart: removed.freelistPages, + freelistAtEnd: end.freelistPages, + reclaimedPages: removed.freelistPages - end.freelistPages, + backToBaselineMs: backAt - reclaimStarted + } + + const retiredCounts = [baseline, removed, end].map((state) => state.retiredEpochRows) + retired.context = { ...retired.context, retiredEpochRows: retiredCounts.join(' → ') } + if (retiredCounts.every((count) => !count)) { + retired.skipped = + 'missing evidence: no row under a retired epoch was observed; ordinary rewind/removal deletes rows in place, so retired-epoch pruning was not exercised' + } else { + retired.samples.push({ + extra: { + retiredAtStart: removed.retiredEpochRows ?? 0, + retiredAtEnd: end.retiredEpochRows ?? 0 + } + }) + } + } catch (error) { + const reason = `failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300) + for (const each of all.filter((entry) => entry.samples.length === 0 && !entry.skipped)) { + each.skipped = reason + } + ctx.log(`maintenance debt rows: ${reason}`) + } +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts index 14cace375d3..f3ec51ca9da 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts @@ -17,7 +17,7 @@ const bytesOf = (value: unknown): number => Buffer.byteLength(JSON.stringify(val type Item = { itemId: string; body?: { kind?: string } } -async function read( +export async function read( ctx: RunContext, method: string, params: unknown, @@ -63,16 +63,8 @@ export async function historyRows(ctx: RunContext): Promise { read(ctx, 'agentSession.history', { sessionId, ...params }, true) ) } - addRow(ctx, { - family: 'history', - operation: 'sparse/interleaved subagent grouping', - variant: 'tail 200', - context, - skipped: - 'the 4a provider stub emits no subagent frames; covered by the Claude/Codex functional QA run' - }) - // Huge originals come from the streaming rows that ran first; the smallest seeded tool item is the ordinary case. + // Whatever the streaming rows left on the tail; named shapes are verified rows of their own (native-chat-4a-ops-shapes.ts). const items = [...tail.page.items, ...older.page.items] as Item[] const sized = items .map((item) => ({ item, bytes: bytesOf(item) })) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts new file mode 100644 index 00000000000..0578bba0fd9 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts @@ -0,0 +1,270 @@ +// 4a history/detail rows for named shapes: sparse and interleaved subagent grouping, huge completed tool originals, an +// original above the detail reply cap and an oversized ordinary message. Each shape is verified on the real route first. +import { read } from './native-chat-4a-ops-reads' +import { rpc, samplePressure } from './native-chat-4a-probes' +import type { Row } from './native-chat-4a-report' +import { addRow, page, sampled, tagFor, type RunContext } from './native-chat-4a-run-context' +import { createChat, history, send, waitForDurable, waitForIdle } from './native-chat-4a-session' +import { shapesPrompt, subagentPrompt, type Workload } from './native-chat-4a-workloads' + +// JOURNAL_ITEM_DETAIL_MAX_BYTES on the candidate (journal-domain-history-metadata.ts): 4 MiB outbound JSON less 64 KiB. +const DETAIL_REPLY_CAP_BYTES = 4 * 1024 * 1024 - 64 * 1024 +const MiB = (bytes: number): string => `${(bytes / 1024 / 1024).toFixed(2)} MiB` + +type WireItem = { + itemId: string + agentId?: string + body?: { + kind?: string + presentation?: string + output?: { head?: string; byteLength?: number } + blocks?: { text?: string }[] + } +} +type Page = Awaited>['page'] + +/** One chat holding the turns' corpus, every turn durable and settled before any sample. */ +async function corpusChat(ctx: RunContext, prompts: string[]): Promise { + const chat = await createChat(page(ctx), ctx.arm, ctx.manifest.workspace.worktreeIds[1]) + for (const prompt of prompts) { + const tag = /"tag":"([^"]+)"/.exec(prompt)![1] + await send(page(ctx), ctx.arm, chat.sessionId, prompt) + await waitForDurable(page(ctx), chat.sessionId, `4A-TAIL-${tag}`, 600_000, 250) + await waitForIdle(page(ctx), chat.sessionId, 600_000) + } + return chat.sessionId +} + +const itemsOf = (read: Page): WireItem[] => read.items as WireItem[] +const byProviderId = (items: WireItem[], id: string): WireItem | undefined => + items.find((item) => JSON.stringify(item).includes(`"${id}"`)) + +/** Fill a row only when `verify` names no mismatch; the observed shape stays in the row either way. */ +async function verifiedRow( + ctx: RunContext, + row: Row, + observed: Record, + mismatch: string | null, + take: () => ReturnType +): Promise { + row.context = { ...row.context, ...observed } + if (mismatch) { + row.skipped = `shape not produced on this arm: ${mismatch}` + ctx.log(`${row.family}/${row.operation}/${row.variant}: ${row.skipped}`) + return + } + await sampled(ctx, row, ctx.workload.samples, take) +} + +export async function subagentRows(ctx: RunContext): Promise { + const { subagents } = ctx.workload + const corpora = [ + { + mode: 'interleaved' as const, + variant: `tail 200 (${subagents.agents} parallel agents × ${subagents.childrenPerAgent} children × ${subagents.interleavedTurns} turns)`, + turns: subagents.interleavedTurns + }, + { + mode: 'sparse' as const, + variant: `tail 200 (${subagents.sparseChildren} children under one agent, near-empty root)`, + turns: 1 + } + ] + for (const corpus of corpora) { + const sessionId = await corpusChat( + ctx, + Array.from({ length: corpus.turns }, () => + subagentPrompt(ctx.workload, tagFor(`sub-${corpus.mode}`), corpus.mode) + ) + ) + const tail = await history(page(ctx), sessionId, 'tail', 200) + const items = itemsOf(tail.page) + const agentIds = items.map((item) => item.agentId ?? null) + const subagentItems = agentIds.filter((id) => id !== null) + const switches = subagentItems.filter( + (id, index) => index > 0 && id !== subagentItems[index - 1] + ) + const observed = { + sessionId, + pageItems: items.length, + subagentItems: subagentItems.length, + ownItems: items.length - subagentItems.length, + distinctAgents: new Set(subagentItems).size, + agentSwitches: switches.length + } + const mismatch = + subagentItems.length === 0 + ? 'the tail page carries no item with agentId' + : corpus.mode === 'interleaved' && observed.distinctAgents < subagents.agents + ? `${observed.distinctAgents} distinct agents on the page` + : corpus.mode === 'interleaved' && switches.length < subagents.childrenPerAgent + ? `${switches.length} agent switches; children did not interleave` + : corpus.mode === 'sparse' && observed.ownItems > subagentItems.length / 10 + ? `${observed.ownItems} own rows against ${subagentItems.length} subagent rows` + : null + const tailRow = addRow(ctx, { + family: 'history', + operation: `subagent grouping, ${corpus.mode}`, + variant: corpus.variant, + context: {} + }) + await verifiedRow(ctx, tailRow, observed, mismatch, () => + read(ctx, 'agentSession.history', { sessionId, direction: 'tail', limit: 200 }) + ) + const older = addRow(ctx, { + family: 'history', + operation: `subagent grouping, ${corpus.mode}`, + variant: 'one older page 200', + context: {} + }) + await verifiedRow( + ctx, + older, + { hasOlder: Number(tail.page.hasOlder) }, + mismatch ?? (tail.page.hasOlder ? null : 'the corpus fits one page'), + () => + read(ctx, 'agentSession.history', { + sessionId, + direction: 'before', + limit: 200, + cursor: tail.page.window.nextCursor + }) + ) + } +} + +/** Item detail with the main process's peak RSS while it answers, since worker-to-host bytes are not observable outside. */ +const detail = (ctx: RunContext, sessionId: string, itemId: string) => async () => { + const stop = samplePressure(ctx.launch.app, ctx.launch.userDataDir, 50) + const sample = await read(ctx, 'agentSession.item', { sessionId, itemId }) + const pressure = await stop() + return { ...sample, extra: { ...sample.extra, peakMainRssBytes: pressure.peakMainRssBytes } } +} + +async function detailOf(ctx: RunContext, sessionId: string, itemId: string): Promise { + return (await rpc<{ item: WireItem }>(page(ctx), 'agentSession.item', { sessionId, itemId })) + .value.item +} + +export async function itemShapeRows(ctx: RunContext): Promise { + const shapes: Workload['shapes'] = ctx.workload.shapes + const tag = tagFor('shapes') + const sessionId = await corpusChat(ctx, [shapesPrompt(ctx.workload, tag)]) + const tail = await history(page(ctx), sessionId, 'tail', 40) + const items = itemsOf(tail.page) + const huge = Array.from({ length: shapes.hugeTools }, (_, index) => + byProviderId(items, `toolu_4a_${tag}_huge_${index}`) + ) + const cap = byProviderId(items, `toolu_4a_${tag}_cap`) + const headBytes = (item: WireItem | undefined): number => + Buffer.byteLength(item?.body?.output?.head ?? '') + const hugeMismatch = huge.some((item) => !item) + ? `${huge.filter(Boolean).length} of ${shapes.hugeTools} huge tool items on the tail page` + : huge.find((item) => item!.body?.output?.byteLength !== shapes.hugeToolBytes) + ? 'a huge tool item does not report its whole original byteLength' + : huge.find((item) => headBytes(item) >= shapes.hugeToolBytes) + ? 'a huge tool item carries its whole original on the page' + : null + const hugeRow = addRow(ctx, { + family: 'history', + operation: 'huge completed tool originals, heads on one page', + variant: `tail 40 (${shapes.hugeTools} × ${MiB(shapes.hugeToolBytes)} originals + ${MiB(shapes.capToolBytes)})`, + context: { sessionId } + }) + await verifiedRow( + ctx, + hugeRow, + { + originalBytes: shapes.hugeTools * shapes.hugeToolBytes + shapes.capToolBytes, + largestHeadBytes: Math.max(...huge.map(headBytes), headBytes(cap)) + }, + hugeMismatch, + () => read(ctx, 'agentSession.history', { sessionId, direction: 'tail', limit: 40 }) + ) + + for (const [variant, item, bytes] of [ + ['original above the reply cap', cap, shapes.capToolBytes], + ['huge original under the reply cap', huge[0], shapes.hugeToolBytes] + ] as const) { + const row = addRow(ctx, { + family: 'history', + operation: 'agentSession.item detail', + variant, + context: { originalBytes: bytes, replyCapBytes: DETAIL_REPLY_CAP_BYTES } + }) + const full = item ? await detailOf(ctx, sessionId, item.itemId) : undefined + const above = bytes > DETAIL_REPLY_CAP_BYTES + const mismatch = !full + ? 'item not found on the tail page' + : full.body?.output?.byteLength !== bytes + ? `detail reports ${full.body?.output?.byteLength} original bytes` + : above && headBytes(full) >= bytes + ? 'detail returned the whole original above the cap' + : !above && headBytes(full) !== bytes + ? 'detail shortened an original under the cap' + : null + await verifiedRow(ctx, row, { detailHeadBytes: headBytes(full) }, mismatch, () => + detail(ctx, sessionId, item!.itemId)() + ) + } + + // The message's group exceeds every page budget, so the page that reaches it carries only its placeholder. + let cursor: unknown + let carrying: { cursor: unknown; item: WireItem } | null = null + for (let pages = 0; pages < 4 && !carrying; pages += 1) { + const at = await history(page(ctx), sessionId, cursor ? 'before' : 'tail', 40, cursor) + const placeholder = itemsOf(at.page).find( + (item) => item.body?.presentation === 'history-item-too-large' + ) + carrying = placeholder ? { cursor, item: placeholder } : null + if (!at.page.hasOlder) { + break + } + cursor = at.page.window.nextCursor + } + const message = carrying ? await detailOf(ctx, sessionId, carrying.item.itemId) : undefined + const text = message?.body?.blocks?.map((block) => block.text ?? '').join('') ?? '' + const messageMismatch = !carrying + ? 'no history-item-too-large placeholder within 4 pages' + : !text.startsWith(`4A-BIG-${tag} `) + ? 'the placeholder is not the seeded message' + : Buffer.byteLength(text) !== shapes.messageBytes + `4A-BIG-${tag} `.length + ? `detail returned ${Buffer.byteLength(text)} message bytes` + : null + const placeholderRow = addRow(ctx, { + family: 'history', + operation: 'oversized ordinary message placeholder', + variant: `page reaching a ${MiB(shapes.messageBytes)} message`, + context: { messageBytes: shapes.messageBytes } + }) + await verifiedRow( + ctx, + placeholderRow, + { detailTextBytes: Buffer.byteLength(text) }, + messageMismatch, + () => + read(ctx, 'agentSession.history', { + sessionId, + direction: carrying!.cursor ? 'before' : 'tail', + limit: 40, + ...(carrying!.cursor ? { cursor: carrying!.cursor } : {}) + }) + ) + const messageDetail = addRow(ctx, { + family: 'history', + operation: 'agentSession.item detail', + variant: 'oversized ordinary message (under the reply cap)', + context: { messageBytes: shapes.messageBytes, replyCapBytes: DETAIL_REPLY_CAP_BYTES } + }) + await verifiedRow(ctx, messageDetail, {}, messageMismatch, () => + detail(ctx, sessionId, carrying!.item.itemId)() + ) + addRow(ctx, { + family: 'history', + operation: 'completed diff originals', + variant: 'tail 40', + context: {}, + skipped: + 'missing evidence: the 4a stub speaks Claude stream-json, and the Claude adapter writes no diff items (only the Codex and ACP adapters do)' + }) +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts index e472400b08d..084e70c306d 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts @@ -8,6 +8,7 @@ import { storageBytes, watchMarkers } from './native-chat-4a-probes' +import { maintenanceDebtRows } from './native-chat-4a-ops-maintenance' import { stat, type Sample } from './native-chat-4a-report' import { addRow, @@ -125,7 +126,7 @@ async function streamSample( } } -/** After a large stream, sends to another chat while retired payloads are pruned, pages reclaimed and the WAL checkpointed. */ +/** After a large stream, sends to another chat and large-chat reads while whatever maintenance follows runs; no debt is asserted. */ async function maintenanceWindow(ctx: RunContext): Promise { const other = ctx.manifest.fillerSessionIds[1] ?? ctx.manifest.fillerSessionIds[0]! const fromEpoch = Date.now() @@ -191,7 +192,8 @@ export async function streamingRows(ctx: RunContext): Promise { variant: size, context: { windowMs: MAINTENANCE_WINDOW_MS, - retiredPayloadBytes: stream.toolOutputBytes ?? stream.bytes + // The stream's final body stays current; retired-data debt is measured in the maintenance module. + precedingStreamBytes: stream.toolOutputBytes ?? stream.bytes } }) try { @@ -201,6 +203,7 @@ export async function streamingRows(ctx: RunContext): Promise { `failed: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300) } } + await maintenanceDebtRows(ctx) // A cheap read after all streams confirms the large chat still answers its tail. await rpc(page(ctx), 'agentSession.history', { sessionId: largeSessionId, diff --git a/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs index 524bad5a412..8d9013de26a 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs +++ b/config/scripts/performance/native-chat/native-chat-4a-provider-stub.cjs @@ -43,10 +43,12 @@ if (fs.existsSync(transcriptPath)) { } // Fork and rewind read the provider's own transcript, so every user/assistant frame is also saved there. +// Subagent frames are sidechain rows, as the CLI records them; the main chain skips them. function remember(frame) { + const sidechain = Boolean(frame.parent_tool_use_id) const row = { parentUuid, - isSidechain: false, + isSidechain: sidechain, type: frame.type, message: frame.message, uuid: frame.uuid, @@ -55,7 +57,9 @@ function remember(frame) { cwd: process.cwd() } fs.appendFileSync(transcriptPath, `${JSON.stringify(row)}\n`) - parentUuid = frame.uuid + if (!sidechain) { + parentUuid = frame.uuid + } } let backlog = Promise.resolve() @@ -84,8 +88,10 @@ function framed(type, message, extra = {}) { return write(frame) } -const assistant = (content) => - framed('assistant', { id: randomUUID(), role: 'assistant', model: 'claude-fake', content }) +const assistant = (content, extra) => + framed('assistant', { id: randomUUID(), role: 'assistant', model: 'claude-fake', content }, extra) +const system = (subtype, fields) => + write({ type: 'system', subtype, ...fields, uuid: randomUUID(), session_id: sessionId }) const streamEvent = (event) => write({ type: 'stream_event', @@ -100,11 +106,11 @@ let current = null const pending = [] let turnCounter = 0 -async function toolRun(turn, index, outputBytes, runMs) { - const id = `toolu_4a_${turn}_${index}` - await assistant([ - { type: 'tool_use', id, name: 'Bash', input: { command: `ls -la dir${index}` } } - ]) +async function toolRun(turn, index, outputBytes, runMs, id = `toolu_4a_${turn}_${index}`, extra) { + await assistant( + [{ type: 'tool_use', id, name: 'Bash', input: { command: `ls -la dir${index}` } }], + extra + ) if (runMs > 0) { await write({ type: 'system', @@ -118,12 +124,103 @@ async function toolRun(turn, index, outputBytes, runMs) { }) await sleep(runMs) } - await framed('user', { - role: 'user', - content: [ - { tool_use_id: id, type: 'tool_result', content: prose(outputBytes, id), is_error: false } - ] - }) + await framed( + 'user', + { + role: 'user', + content: [ + { tool_use_id: id, type: 'tool_result', content: prose(outputBytes, id), is_error: false } + ] + }, + extra + ) +} + +/** Shapes for history/detail: an oversized ordinary message, huge completed tool originals, one above the detail cap. */ +async function shapes(op, signal) { + await assistant([{ type: 'text', text: `4A-BIG-${op.tag} ${prose(op.messageBytes, op.tag)}` }]) + for (let index = 0; index < op.hugeTools && !signal.aborted; index += 1) { + await toolRun(0, index, op.hugeToolBytes, 0, `toolu_4a_${op.tag}_huge_${index}`) + } + await toolRun(0, 0, op.capToolBytes, 0, `toolu_4a_${op.tag}_cap`) +} + +/** Foreground Agent calls in the CLI's frame shapes; children of parallel agents alternate, one agent's children run sparse. */ +async function subagents(op, signal) { + const agents = Array.from({ length: op.agents }, (_, index) => ({ + toolId: `toolu_4a_${op.tag}_agent_${index}`, + taskId: `task4a${op.tag.replace(/\W/g, '')}${index}`, + prompt: `4A-SUBAGENT-${op.tag}-${index} list the files` + })) + await assistant( + agents.map((agent) => ({ + type: 'tool_use', + id: agent.toolId, + name: 'Agent', + input: { description: 'List files', subagent_type: 'general-purpose', prompt: agent.prompt } + })) + ) + for (const agent of agents) { + await system('task_started', { + task_id: agent.taskId, + tool_use_id: agent.toolId, + description: 'List files', + subagent_type: 'general-purpose', + is_backgrounded: false, + spawn_depth: 1, + task_type: 'local_agent', + prompt: agent.prompt + }) + await framed( + 'user', + { role: 'user', content: [{ type: 'text', text: agent.prompt }] }, + { + parent_tool_use_id: agent.toolId, + subagent_type: 'general-purpose', + task_description: 'List files' + } + ) + } + for (let child = 0; child < op.children && !signal.aborted; child += 1) { + for (const agent of agents) { + await toolRun(0, child, 200, 0, `${agent.toolId}_child_${child}`, { + parent_tool_use_id: agent.toolId, + subagent_type: 'general-purpose', + task_description: 'List files' + }) + } + } + for (const agent of agents) { + const summary = `4A-SUBAGENT-DONE-${op.tag} ${agent.taskId}` + await system('task_notification', { + task_id: agent.taskId, + tool_use_id: agent.toolId, + status: 'completed', + summary + }) + await framed( + 'user', + { + role: 'user', + content: [ + { + tool_use_id: agent.toolId, + type: 'tool_result', + content: [{ type: 'text', text: summary }] + } + ] + }, + { + tool_use_result: { + status: 'completed', + prompt: agent.prompt, + agentId: agent.taskId, + agentType: 'general-purpose', + content: [{ type: 'text', text: summary }] + } + } + ) + } } async function streamReply(op, signal) { @@ -178,6 +275,11 @@ async function runOp(op, signal) { await assistant([{ type: 'text', text: `4A-TAIL-${tag} ${prose(op.replyBytes ?? 1500, tag)}` }]) return } + if (op.op === 'shapes' || op.op === 'subagents') { + await (op.op === 'shapes' ? shapes : subagents)(op, signal) + await assistant([{ type: 'text', text: `4A-TAIL-${tag} Stub reply.` }]) + return + } if (op.op === 'stream') { if (op.toolOutputBytes) { await toolRun(turn, 0, op.toolOutputBytes, op.toolRunMs ?? 250) diff --git a/config/scripts/performance/native-chat/native-chat-4a-report.ts b/config/scripts/performance/native-chat/native-chat-4a-report.ts index 44021dacc5c..f9efc93ac6d 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-report.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-report.ts @@ -1,5 +1,9 @@ // 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's head SHA. +import { writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' import type { HostWork } from './native-chat-4a-probes' +import type { Workload } from './native-chat-4a-workloads' export type Sample = { /** Action to user-visible (or durably readable, when the row says so) result. */ @@ -130,7 +134,7 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string { } lines.push( '', - 'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.' + 'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; steady-state families start only after every chat is seen ready. Rows whose Shape names `observedDatabase` end at an outside read-only SQLite observation and carry its resolution. Per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.' ) return `${lines.join('\n')}\n` } @@ -150,3 +154,34 @@ export function armMarkdown(result: ArmResult): string { } return `${lines.join('\n')}\n` } + +/** The comparison JSON (every sample and summary, the workload and machine) beside its table. */ +export function writeComparison( + out: string, + workload: Workload, + results: ArmResult[], + log: (line: string) => void +): void { + const [base, next] = results + if (!base || !next) { + return + } + const name = `4a-${base.sha.slice(0, 8)}-vs-${next.sha.slice(0, 8)}` + const summaries = results.map((result) => ({ + ...result, + summaries: result.rows.map((row: Row) => ({ ...row, summary: summarizeRow(row) })) + })) + const machine = { + platform: process.platform, + release: os.release(), + cpu: os.cpus()[0]?.model, + cpus: os.cpus().length, + memoryBytes: os.totalmem() + } + writeFileSync( + path.join(out, `${name}.json`), + `${JSON.stringify({ workload, machine, arms: summaries }, null, 2)}\n` + ) + writeFileSync(path.join(out, `${name}.md`), comparisonMarkdown(base, next)) + log(`wrote ${path.join(out, `${name}.md`)}`) +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-sqlite-observer.ts b/config/scripts/performance/native-chat/native-chat-4a-sqlite-observer.ts new file mode 100644 index 00000000000..7e73610eb2a --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-sqlite-observer.ts @@ -0,0 +1,228 @@ +// Read-only outside observation of an arm's chat SQLite database: per-chat ready marks, page/WAL state and a held reader snapshot. +// Every connection opens with readOnly; nothing here writes to the profile or asks the app for timing. +import { existsSync, readdirSync, statSync } from 'node:fs' +import path from 'node:path' +import { DatabaseSync } from 'node:sqlite' + +const DOMAIN_DB = 'agent-session-domain.db' +const JOURNAL_DB = 'agent-session-journal.db' + +function findDatabase(userDataDir: string, name: string): string | null { + for (const entry of readdirSync(userDataDir, { recursive: true, withFileTypes: true })) { + if (entry.isFile() && entry.name === name && !entry.parentPath.includes(`${path.sep}home`)) { + return path.join(entry.parentPath, entry.name) + } + } + return null +} + +/** Every SQLite file in the profile outside the isolated home, relative to it: which originals a conversion retained. */ +export const seededDbFiles = (dir: string): string[] => + readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter( + (entry) => + entry.isFile() && + /\.(db|sqlite)$/.test(entry.name) && + !entry.parentPath.includes(`${path.sep}home`) + ) + .map((entry) => path.relative(dir, path.join(entry.parentPath, entry.name))) + +/** The store chat rows live in on this arm: the candidate's converted store when present, else the journal store both arms keep. */ +export const chatDatabasePath = (userDataDir: string): string | null => + findDatabase(userDataDir, DOMAIN_DB) ?? findDatabase(userDataDir, JOURNAL_DB) + +const openReadOnly = (file: string): DatabaseSync => + new DatabaseSync(file, { readOnly: true, timeout: 2_000 }) + +const hasTable = (db: DatabaseSync, name: string): boolean => + db.prepare("SELECT 1 FROM sqlite_schema WHERE type='table' AND name=?").get(name) !== undefined + +export type PageState = { + pageSize: number + pageCount: number + freelistPages: number + walBytes: number + /** Rows under an epoch other than their chat's ready epoch; null on a store without epochs. */ + retiredEpochRows: number | null +} + +/** One short read-only look at page, free-list and WAL state; `retired` adds a scan for retired-epoch rows. */ +export function pageState(file: string, retired = false): PageState { + const db = openReadOnly(file) + try { + const pragma = (name: string): number => + Number(Object.values(db.prepare(`PRAGMA ${name}`).get() ?? {})[0] ?? Number.NaN) + const epochs = hasTable(db, 'journal_chat_state') && hasTable(db, 'journal_items') + return { + pageSize: pragma('page_size'), + pageCount: pragma('page_count'), + freelistPages: pragma('freelist_count'), + walBytes: statSync(`${file}-wal`, { throwIfNoEntry: false })?.size ?? 0, + retiredEpochRows: + retired && epochs + ? Number( + db + .prepare(`SELECT count(*) AS n FROM journal_items item JOIN journal_chat_state head + ON head.session_id=item.session_id WHERE head.epoch IS NOT NULL AND item.epoch<>head.epoch`) + .get()?.n + ) + : null + } + } finally { + db.close() + } +} + +/** Hold one read snapshot, as a long reader would, until the returned release runs. */ +export function holdReader(file: string): () => void { + const db = openReadOnly(file) + db.exec('BEGIN') + db.prepare('SELECT count(*) FROM sqlite_schema').get() + let held = true + return () => { + if (held) { + held = false + db.exec('COMMIT') + db.close() + } + } +} + +/** Where a chat stands on the candidate's epoch-installed ready boundary. */ +export type ChatState = 'no-domain-database' | 'uncatalogued' | 'pending' | 'importing' | 'ready' +export const CHAT_STATE_CODE: Record = { + 'no-domain-database': Number.NaN, + uncatalogued: -1, + pending: 0, + importing: 1, + ready: 2 +} + +/** + * Polls `journal_chat_state` for epochs installed since the last poll; a chat's ready time is the first poll that sees it, + * so every ready mark is an upper bound with `resolutionMs` resolution. Start it before the launch it observes. + */ +export class ConversionObserver { + readonly readyAt = new Map() + databaseSeenAt = Number.NaN + catalogRows = 0 + private db: DatabaseSync | null = null + /** The converted store is created beside the seeded journal store, in the same state directory. */ + private readonly file: string | null + private slowestPollMs = 0 + private stopped = false + private readonly loop: Promise + + constructor( + userDataDir: string, + readonly everyMs = 25 + ) { + const journal = findDatabase(userDataDir, JOURNAL_DB) + this.file = journal ? path.join(path.dirname(journal), DOMAIN_DB) : null + this.loop = this.run() + } + + /** The converted store once it exists; null on an arm that never creates one. */ + get databaseFile(): string | null { + return this.db ? this.file : null + } + + get resolutionMs(): number { + return this.everyMs + Math.ceil(this.slowestPollMs) + } + + private connection(): DatabaseSync | null { + if (!this.db) { + if (!this.file || !existsSync(this.file)) { + return null + } + try { + const db = openReadOnly(this.file) + if (!hasTable(db, 'journal_chat_state')) { + db.close() + return null + } + this.db = db + this.databaseSeenAt = Date.now() + } catch { + return null + } + } + return this.db + } + + private poll(): void { + const db = this.connection() + if (!db) { + return + } + const started = performance.now() + try { + const ready = db + .prepare('SELECT session_id FROM journal_chat_state WHERE epoch IS NOT NULL') + .all() + const now = Date.now() + for (const row of ready) { + const id = String(row.session_id) + if (!this.readyAt.has(id)) { + this.readyAt.set(id, now) + } + } + this.catalogRows = Number(db.prepare('SELECT count(*) AS n FROM journal_chat_state').get()?.n) + } catch { + // A busy or briefly locked store is retried on the next poll. + } + this.slowestPollMs = Math.max(this.slowestPollMs, performance.now() - started) + } + + private async run(): Promise { + while (!this.stopped) { + this.poll() + // Unref'd: an observer left running never keeps the harness process alive. + await new Promise((resolve) => setTimeout(resolve, this.everyMs).unref()) + } + } + + /** Read one chat's state now, outside the poll. */ + state(sessionId: string): ChatState { + const db = this.connection() + if (!db) { + return 'no-domain-database' + } + const head = db + .prepare('SELECT epoch FROM journal_chat_state WHERE session_id=?') + .get(sessionId) + if (!head) { + return 'uncatalogued' + } + if (head.epoch !== null) { + this.readyAt.set(sessionId, this.readyAt.get(sessionId) ?? Date.now()) + return 'ready' + } + return db.prepare('SELECT 1 FROM journal_items WHERE session_id=? LIMIT 1').get(sessionId) + ? 'importing' + : 'pending' + } + + /** Resolve once every listed chat was seen ready (true), or false at the deadline or without a domain database. */ + async allReady(sessionIds: readonly string[], timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (!this.connection()) { + return false + } + if (sessionIds.every((id) => this.readyAt.has(id))) { + return true + } + await new Promise((resolve) => setTimeout(resolve, this.everyMs)) + } + return false + } + + async stop(): Promise { + this.stopped = true + await this.loop + this.db?.close() + this.db = null + } +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts index 532480c824f..de021cb3056 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts @@ -19,6 +19,8 @@ export type Workload = { history: { /** The large retained chat: one recipe turn is 1 prompt + `tools` tool items + 1 reply (+ its turn row). */ largeTurns: number + /** The medium chat's turns, same recipe; 0 seeds no medium chat. */ + mediumTurns: number tools: number toolOutputBytes: number promptBytes: number @@ -35,6 +37,17 @@ export type Workload = { rewind: { nearTipSamples: number; deepCutTurns: number; deepSamples: number } /** Generated only on the M4Air run; smoke keeps the 55k shape off this Mac. */ conversionSourceBytes: number + /** Huge completed tool originals whose heads share one page, one original above the detail reply cap, one oversized message. */ + shapes: { hugeTools: number; hugeToolBytes: number; capToolBytes: number; messageBytes: number } + /** Interleaved: parallel subagents whose child rows alternate, per turn. Sparse: one subagent's rows under a near-empty root. */ + subagents: { + interleavedTurns: number + agents: number + childrenPerAgent: number + sparseChildren: number + } + /** Large completed rows written then rewound on a disposable chat, an outside reader held for `holdMs`, reclaim watched for `reclaimWaitMs`. */ + maintenance: { debtTools: number; debtToolBytes: number; holdMs: number; reclaimWaitMs: number } } const MiB = 1024 * 1024 @@ -59,6 +72,7 @@ export const WORKLOADS: Record = { coldSamples: 1, history: { largeTurns: 20, + mediumTurns: 5, tools: 8, toolOutputBytes: 2000, promptBytes: 300, @@ -70,21 +84,25 @@ export const WORKLOADS: Record = { stream: { cases: streamCases([MiB / 4]), chunkBytes: 16_384, intervalMs: 1, samples: 1 }, retained: { checkpoints: [0, 20], liveQueue: 2 }, rewind: { nearTipSamples: 2, deepCutTurns: 5, deepSamples: 1 }, - conversionSourceBytes: 0 + conversionSourceBytes: 0, + shapes: { hugeTools: 3, hugeToolBytes: MiB, capToolBytes: 5 * MiB, messageBytes: 3 * MiB }, + subagents: { interleavedTurns: 2, agents: 2, childrenPerAgent: 30, sparseChildren: 250 }, + maintenance: { debtTools: 4, debtToolBytes: 2 * MiB, holdMs: 10_000, reclaimWaitMs: 150_000 } }, full: { profile: 'full', samples: 30, coldSamples: 5, - // The established 5000-turn/55000-item recipe plus 309 small chats: the 55k-history/310-chat profile. + // The established 5000-turn/55000-item recipe, a 500-turn medium chat and 308 small chats: the 55k-history/310-chat profile. history: { largeTurns: 5000, + mediumTurns: 500, tools: 8, toolOutputBytes: 2000, promptBytes: 300, replyBytes: 1500, queuedEvery: 10, - fillerChats: 309, + fillerChats: 308, fillerTurns: 2 }, stream: { @@ -95,7 +113,11 @@ export const WORKLOADS: Record = { }, retained: { checkpoints: [0, 1_000, 10_000], liveQueue: 2 }, rewind: { nearTipSamples: 10, deepCutTurns: 1000, deepSamples: 3 }, - conversionSourceBytes: 20 * 1024 * MiB + conversionSourceBytes: 20 * 1024 * MiB, + shapes: { hugeTools: 4, hugeToolBytes: 3 * MiB, capToolBytes: 16 * MiB, messageBytes: 3 * MiB }, + subagents: { interleavedTurns: 20, agents: 2, childrenPerAgent: 40, sparseChildren: 400 }, + // 160 MiB of rows rewound under a held reader passes the candidate's 32,768-frame WAL admission threshold. + maintenance: { debtTools: 10, debtToolBytes: 16 * MiB, holdMs: 60_000, reclaimWaitMs: 600_000 } } } @@ -109,13 +131,17 @@ export function workloadFor(profile: string, conversionSourceBytes?: number): Wo export type HistoryTurn = { prompt: string; queued: boolean } -/** Large-chat recipe turns in send order; each reply ends with a 4A-TAIL marker naming its turn. */ -export function* largeHistoryTurns(workload: Workload): Generator { +/** A recipe chat's turns in send order; each reply ends with a 4A-TAIL marker naming its chat and turn. */ +export function* historyTurns( + workload: Workload, + chat: 'large' | 'medium' = 'large' +): Generator { const { history } = workload - for (let turn = 0; turn < history.largeTurns; turn += 1) { + const turns = chat === 'large' ? history.largeTurns : history.mediumTurns + for (let turn = 0; turn < turns; turn += 1) { const op = { op: 'history', - tag: `large-${turn}`, + tag: `${chat}-${turn}`, tools: history.tools, toolOutputBytes: history.toolOutputBytes, replyBytes: history.replyBytes @@ -138,9 +164,9 @@ export function fillerTurn(chat: number, turn: number, workload: Workload): stri return providerText.formatDirective(op, workload.history.promptBytes) } -/** Expected large-chat rows, so a run can refuse a fixture that did not seed what it claims. */ -export function expectedLargeItems(workload: Workload): number { - return workload.history.largeTurns * (workload.history.tools + 3) +/** Expected recipe-chat rows, so a run can refuse a fixture that did not seed what it claims. */ +export function expectedItems(turns: number, workload: Workload): number { + return turns * (workload.history.tools + 3) } /** Per-turn tool output that makes the conversion source reach `conversionSourceBytes` with the same turn count. */ @@ -167,6 +193,29 @@ export function streamPrompt(workload: Workload, stream: StreamCase): string { }) } +export const shapesPrompt = (workload: Workload, tag: string): string => + providerText.formatDirective({ op: 'shapes', tag, ...workload.shapes }) +export const subagentPrompt = ( + workload: Workload, + tag: string, + mode: 'interleaved' | 'sparse' +): string => { + const { agents, childrenPerAgent, sparseChildren } = workload.subagents + return providerText.formatDirective({ + op: 'subagents', + tag, + agents: mode === 'sparse' ? 1 : agents, + children: mode === 'sparse' ? sparseChildren : childrenPerAgent + }) +} +export const debtPrompt = (workload: Workload, tag: string): string => + providerText.formatDirective({ + op: 'history', + tag, + tools: workload.maintenance.debtTools, + toolOutputBytes: workload.maintenance.debtToolBytes, + replyBytes: 400 + }) export const holdPrompt = (tag: string, ms: number): string => providerText.formatDirective({ op: 'hold', tag, ms }) export const replyPrompt = (tag: string): string => @@ -178,7 +227,7 @@ export function dryRunWorkload(workload: Workload): Record { let promptBytes = 0 let queued = 0 let turns = 0 - for (const turn of largeHistoryTurns(workload)) { + for (const turn of historyTurns(workload)) { promptBytes += Buffer.byteLength(turn.prompt) queued += Number(turn.queued) turns += 1 @@ -198,13 +247,25 @@ export function dryRunWorkload(workload: Workload): Record { return { profile: workload.profile, largeTurns: turns, - expectedLargeItems: expectedLargeItems(workload), + expectedLargeItems: expectedItems(workload.history.largeTurns, workload), + expectedMediumItems: expectedItems(workload.history.mediumTurns, workload), queuedHistoryTurns: queued, largePromptBytes: promptBytes, largeToolOutputBytes: turns * workload.history.tools * workload.history.toolOutputBytes, - chats: 1 + workload.history.fillerChats, + chats: 1 + Number(workload.history.mediumTurns > 0) + workload.history.fillerChats, + mediumPromptDigest: providerText.digest( + [...historyTurns(workload, 'medium')].map((turn) => turn.prompt).join('\n') + ), fillerPromptDigest: providerText.digest(fillerTurn(0, 0, workload)), streams, + shapesPromptDigest: providerText.digest(shapesPrompt(workload, 'shapes')), + subagentPromptDigests: (['interleaved', 'sparse'] as const).map((mode) => + providerText.digest(subagentPrompt(workload, 'subagents', mode)) + ), + debtPromptDigest: providerText.digest(debtPrompt(workload, 'debt')), + shapes: workload.shapes, + subagents: workload.subagents, + maintenance: workload.maintenance, retained: workload.retained, rewind: workload.rewind, conversion: { From bf869bc42162fa53f7e4b3ab837a771a6e9ba678 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:37:32 -0800 Subject: [PATCH 13/15] Replace casts and empty type imports in the 4a harness scripts with annotations and guards Renderer times and tab ids from waitForFunction handles go through typeof guards, the visibility probe is a declared Window global, the runtime call keeps its own typed response, and cached manifests, Electron paths, profiles and families are narrowed by guards. The empty `import type {}` blocks name AppStore and use it. Three casts remain, each with a line-specific SAFETY: the generic RPC result and Node's performance inside main-process callbacks. --- .../native-chat/native-chat-4a-arm.ts | 19 +++-- .../native-chat/native-chat-4a-comparison.ts | 10 ++- .../native-chat/native-chat-4a-fixture.ts | 15 +++- .../native-chat-4a-ops-lifecycle.ts | 4 +- .../native-chat/native-chat-4a-ops-reads.ts | 6 +- .../native-chat/native-chat-4a-ops-shapes.ts | 2 +- .../native-chat/native-chat-4a-probes.ts | 82 ++++++++++++------- .../native-chat/native-chat-4a-session.ts | 18 ++-- .../native-chat/native-chat-4a-workloads.ts | 2 +- 9 files changed, 101 insertions(+), 57 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm.ts b/config/scripts/performance/native-chat/native-chat-4a-arm.ts index 0b6c5d3160c..09feb7e4eba 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-arm.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-arm.ts @@ -1,6 +1,6 @@ // One 4a arm: a built checkout, its own envelope/fingerprint builders, and background launches it alone stops by recorded PID. -// Declares the window.__store and window.api globals these page.evaluate callbacks use. -import type {} from '../../../../tests/e2e/helpers/runtime-types' +// Its AppStore import also declares the window.__store and window.api globals these page.evaluate callbacks use. +import type { AppStore } from '../../../../tests/e2e/helpers/runtime-types' import { execFileSync } from 'node:child_process' import { randomUUID } from 'node:crypto' import { chmodSync, constants, cpSync, existsSync, mkdirSync, writeFileSync } from 'node:fs' @@ -79,7 +79,10 @@ export async function resolveArm(name: string, root: string, scratch: string): P if (!existsSync(path.join(absolute, 'out', 'main', 'index.js'))) { throw new Error(`${name}: ${absolute} has no out/main/index.js; build that checkout first`) } - const electronPath = createRequire(path.join(absolute, 'package.json'))('electron') as string + const electronPath: unknown = createRequire(path.join(absolute, 'package.json'))('electron') + if (typeof electronPath !== 'string') { + throw new Error(`${name}: the electron package in ${absolute} names no executable`) + } const outfile = path.join(scratch, `${name}-protocol.cjs`) const shared = './src/shared/agent-session' await build({ @@ -98,7 +101,7 @@ export async function resolveArm(name: string, root: string, scratch: string): P logLevel: 'silent', nodePaths: [path.join(HARNESS_ROOT, 'node_modules')] }) - const protocol = createRequire(import.meta.url)(outfile) as ArmProtocol + const protocol: ArmProtocol = createRequire(import.meta.url)(outfile) return { name, root: absolute, sha: git(absolute, 'rev-parse', 'HEAD'), electronPath, protocol } } @@ -220,7 +223,7 @@ export async function prepareWorkspace( if ('error' in added) { throw new Error(added.error) } - const store = window.__store! + const store: AppStore = window.__store! await store.getState().fetchRepos({ runtimeEnvironmentId: null, throwOnError: true }) await store.getState().updateRepo(added.repo.id, { externalWorktreeVisibility: 'show', @@ -239,10 +242,8 @@ export async function prepareWorkspace( (a, b) => Number(b.isMainWorktree) - Number(a.isMainWorktree) ) store.getState().setActiveWorktree(ordered[0]!.id) - return { - repoId: added.repo.id, - worktreeIds: [ordered[0]!.id, ordered[1]!.id] as [string, string] - } + const worktreeIds: [string, string] = [ordered[0]!.id, ordered[1]!.id] + return { repoId: added.repo.id, worktreeIds } } await new Promise((resolve) => setTimeout(resolve, 500)) } diff --git a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts index 3a65cdf5203..1122d1ce14d 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts @@ -219,12 +219,18 @@ async function main(): Promise { if (values['report-only']) { const results = readdirSync(out) .filter((file) => /^arm-.*\.json$/.test(file)) - .map((file) => JSON.parse(readFileSync(path.join(out, file), 'utf8')) as ArmResult) + .map((file): ArmResult => JSON.parse(readFileSync(path.join(out, file), 'utf8'))) .sort((a, b) => a.startedAt.localeCompare(b.startedAt)) writeComparison(out, workload, results, log) return } - const families = new Set(values.families.split(',').map((family) => family.trim()) as Family[]) + const isFamily = (name: string): name is Family => FAMILIES.some((family) => family === name) + const families = new Set( + values.families + .split(',') + .map((family) => family.trim()) + .filter(isFamily) + ) const { resolveArm, writeProviderStub } = await import('./native-chat-4a-arm') const { seedProfile, seedDirFor } = await import('./native-chat-4a-fixture') const { createSeededTestRepo } = await import('../../../../tests/e2e/helpers/seeded-test-repo') diff --git a/config/scripts/performance/native-chat/native-chat-4a-fixture.ts b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts index d21de95034a..b93bb3cb0cf 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-fixture.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-fixture.ts @@ -48,6 +48,11 @@ async function countItems(page: Page, sessionId: string): Promise { } } +/** A manifest this harness wrote from `recipe`; only the recipe digest decides reuse. */ +function isManifestFrom(value: unknown, recipe: string): value is Manifest { + return typeof value === 'object' && value !== null && 'recipe' in value && value.recipe === recipe +} + const recipeOf = (workload: Workload, toolOutputBytes: number): string => createHash('sha256') .update(JSON.stringify({ history: workload.history, toolOutputBytes })) @@ -105,13 +110,15 @@ export async function seedProfile(input: { const manifestPath = `${seedDir}.json` const recipe = recipeOf(input.workload, input.toolOutputBytes) if (existsSync(manifestPath) && existsSync(seedDir)) { - const cached = JSON.parse(readFileSync(manifestPath, 'utf8')) as Partial - if (cached.recipe !== recipe) { + const cached: unknown = JSON.parse(readFileSync(manifestPath, 'utf8')) + if (!isManifestFrom(cached, recipe)) { + const recorded = + typeof cached === 'object' && cached !== null && 'recipe' in cached ? cached.recipe : null throw new Error( - `${manifestPath} was seeded from another recipe (${cached.recipe ?? 'none recorded'}, now ${recipe}); move it and ${seedDir} aside or use a new --out` + `${manifestPath} was seeded from another recipe (${String(recorded ?? 'none recorded')}, now ${recipe}); move it and ${seedDir} aside or use a new --out` ) } - return cached as Manifest + return cached } const started = Date.now() const launch = await launchArm(input.arm, input.profileDir, input.stubCommand) diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts index db41444fdde..a53773415cb 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-lifecycle.ts @@ -1,6 +1,6 @@ // 4a process-lifecycle rows: cold/warm open and rewind; first launch on the seeded profile lives in the conversion module. import { unconvertedOpenTarget } from './native-chat-4a-fixture' -import { markerSeenAt, rendererNow, rpc, watchMarkers } from './native-chat-4a-probes' +import { markerSeenAt, rendererNow, rendererTime, rpc, watchMarkers } from './native-chat-4a-probes' import type { Sample } from './native-chat-4a-report' import { addRow, @@ -170,7 +170,7 @@ export async function rewindRows(ctx: RunContext): Promise { goneMarker, { timeout: 600_000, polling: 16 } ) - .then(async (handle) => (await handle.jsonValue()) as number) + .then(rendererTime) await waitForDurable(page(ctx), other, `4A-TAIL-${competingTag}`) return { hostMs: answer.hostMs, diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts index f3ec51ca9da..1adae66e6ef 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-reads.ts @@ -1,7 +1,7 @@ // 4a read rows on the retained large chat: history pages, item detail, outline, fork, and record operations on the many-chat profile. import { randomUUID } from 'node:crypto' import { operationId } from './native-chat-4a-arm' -import { rpc } from './native-chat-4a-probes' +import { rendererTime, rpc } from './native-chat-4a-probes' import type { Sample } from './native-chat-4a-report' import { addRow, @@ -65,7 +65,7 @@ export async function historyRows(ctx: RunContext): Promise { } // Whatever the streaming rows left on the tail; named shapes are verified rows of their own (native-chat-4a-ops-shapes.ts). - const items = [...tail.page.items, ...older.page.items] as Item[] + const items: Item[] = [...tail.page.items, ...older.page.items] const sized = items .map((item) => ({ item, bytes: bytesOf(item) })) .sort((a, b) => b.bytes - a.bytes) @@ -157,7 +157,7 @@ export async function recordRows(ctx: RunContext): Promise { name, { timeout: 60_000, polling: 16 } ) - .then(async (handle) => (await handle.jsonValue()) as number) + .then(rendererTime) .catch(() => Number.NaN) return { hostMs: answer.hostMs, diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts index 0578bba0fd9..790e8c5eed7 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-shapes.ts @@ -35,7 +35,7 @@ async function corpusChat(ctx: RunContext, prompts: string[]): Promise { return chat.sessionId } -const itemsOf = (read: Page): WireItem[] => read.items as WireItem[] +const itemsOf = (read: Page): WireItem[] => read.items const byProviderId = (items: WireItem[], id: string): WireItem | undefined => items.find((item) => JSON.stringify(item).includes(`"${id}"`)) diff --git a/config/scripts/performance/native-chat/native-chat-4a-probes.ts b/config/scripts/performance/native-chat/native-chat-4a-probes.ts index 1c21792c1f7..4ae01aa88e1 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-probes.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-probes.ts @@ -6,7 +6,35 @@ import type { ElectronApplication, Page } from '@stablyai/playwright-test' import { retryTransientMainEvaluate } from '../../../../tests/e2e/helpers/electron-main-evaluate-retry' import { summarizeEventLoopDelay } from '../../../../tests/e2e/helpers/event-loop-delay-report' -export type RpcAnswer = { ok: true; result: T } | { ok: false; error: unknown } +type ProbeState = { seen: Map; mutations: number[]; watch: Set } + +declare global { + // oxlint-disable-next-line typescript-eslint/consistent-type-definitions -- declaration merging requires interface + interface Window { + /** The 4a visibility probe, installed by installVisibilityProbe. */ + __4a?: ProbeState + } +} + +/** The refusal some writes (create, fork) carry inside a successful call: `{ ok: false, refusal }`. */ +function inResultRefusal(result: unknown): { refusal: unknown } | null { + return typeof result === 'object' && + result !== null && + 'ok' in result && + result.ok === false && + 'refusal' in result + ? { refusal: result.refusal } + : null +} + +/** A waitForFunction handle whose value is a renderer-clock time. */ +export async function rendererTime(handle: { jsonValue(): Promise }): Promise { + const value = await handle.jsonValue() + if (typeof value !== 'number') { + throw new Error(`expected a renderer time, got ${JSON.stringify(value)}`) + } + return value +} /** One RPC from the renderer, timed on the renderer clock: the host span every row reports beside its visible end point. */ export async function rpc( @@ -18,21 +46,18 @@ export async function rpc( const answer = await page.evaluate( async ({ method, params }) => { const startedAt = performance.now() - const reply = (await window.api.runtime.call({ method, params })) as RpcAnswer + const reply = await window.api.runtime.call({ method, params }) return { reply, startedAt, hostMs: performance.now() - startedAt } }, { method, params } ) - // Some writes (create, fork) refuse inside a successful call: `{ ok: false, refusal }`. - const result = answer.reply.ok - ? (answer.reply.result as { ok?: unknown; refusal?: unknown }) - : null - const inResult = result?.ok === false && 'refusal' in result - const refused = !answer.reply.ok || inResult + const inResult = answer.reply.ok ? inResultRefusal(answer.reply.result) : null + const refused = !answer.reply.ok || inResult !== null if (refused && !options.allowRefusal) { - const reason = answer.reply.ok ? result?.refusal : answer.reply.error + const reason = answer.reply.ok ? inResult?.refusal : answer.reply.error throw new Error(`${method} refused: ${JSON.stringify(reason).slice(0, 400)}`) } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the runtime RPC answers untyped JSON; each caller names the shape its method returns and reads optional fields defensively. const value = (answer.reply.ok ? answer.reply.result : answer.reply.error) as T return { value, refused, startedAt: answer.startedAt, hostMs: answer.hostMs } } @@ -40,18 +65,11 @@ export async function rpc( /** Install once per page: records when text first appears in the chat root and every transcript mutation batch. */ export async function installVisibilityProbe(page: Page): Promise { await page.evaluate(() => { - const probe = window as unknown as { - __4a?: { seen: Map; mutations: number[]; watch: Set } - } - if (probe.__4a) { + if (window.__4a) { return } - const state = { - seen: new Map(), - mutations: [] as number[], - watch: new Set() - } - probe.__4a = state + const state: ProbeState = { seen: new Map(), mutations: [], watch: new Set() } + window.__4a = state const check = (): void => { const now = performance.now() state.mutations.push(now) @@ -86,8 +104,10 @@ export async function installVisibilityProbe(page: Page): Promise { export async function watchMarkers(page: Page, markers: string[]): Promise { await page.evaluate((markers) => { - const state = (window as unknown as { __4a: { watch: Set; seen: Map } }) - .__4a + const state = window.__4a + if (!state) { + throw new Error('4a visibility probe not installed') + } for (const marker of markers) { state.seen.delete(marker) state.watch.add(marker) @@ -103,9 +123,12 @@ export async function markerSeenAt( ): Promise { const handle = await page .waitForFunction( - (marker) => - (window as unknown as { __4a: { seen: Map } }).__4a.seen.get(marker) ?? - false, + (marker) => { + if (!window.__4a) { + throw new Error('4a visibility probe not installed') + } + return window.__4a.seen.get(marker) ?? false + }, marker, { timeout: timeoutMs, polling: 16 } ) @@ -117,7 +140,7 @@ export async function markerSeenAt( } ) }) - return (await handle.jsonValue()) as number + return rendererTime(handle) } /** One line on the chat the user sees, for timeout messages. */ @@ -136,9 +159,10 @@ export function visibleChatState(page: Page): Promise { export async function mutationGaps(page: Page, from: number, to: number): Promise { return page.evaluate( ({ from, to }) => { - const times = (window as unknown as { __4a: { mutations: number[] } }).__4a.mutations.filter( - (t) => t >= from && t <= to - ) + if (!window.__4a) { + throw new Error('4a visibility probe not installed') + } + const times = window.__4a.mutations.filter((t) => t >= from && t <= to) return times.slice(1).map((time, index) => time - times[index]!) }, { from, to } @@ -155,12 +179,14 @@ export async function hostWorkMark(app: ElectronApplication): Promise<() => Prom const start = await retryTransientMainEvaluate(() => app.evaluate(() => ({ cpu: process.cpuUsage(), + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this callback runs in Electron's main process, where performance is Node's perf_hooks object; the DOM lib this file needs types it as the browser one. elu: (performance as unknown as NodePerformance).eventLoopUtilization() })) ) return async () => { const delta = await retryTransientMainEvaluate(() => app.evaluate((_electron, start) => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: main-process callback; performance is Node's perf_hooks object, typed here by the DOM lib. const node = performance as unknown as NodePerformance return { cpu: process.cpuUsage(start.cpu), diff --git a/config/scripts/performance/native-chat/native-chat-4a-session.ts b/config/scripts/performance/native-chat/native-chat-4a-session.ts index 241054f0930..3cc679cc5b0 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-session.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-session.ts @@ -1,10 +1,10 @@ // Chat operations through the host RPC surface both arms serve, with envelopes built by each arm's own protocol bundle. -// Declares the window.__store and window.api globals these page.evaluate callbacks use. -import type {} from '../../../../tests/e2e/helpers/runtime-types' +// Its AppStore import also declares the window.__store and window.api globals these page.evaluate callbacks use. +import type { AppStore } from '../../../../tests/e2e/helpers/runtime-types' import { randomUUID } from 'node:crypto' import type { Page } from '@stablyai/playwright-test' import { operationId, type Arm } from './native-chat-4a-arm' -import { rpc, visibleChatState } from './native-chat-4a-probes' +import { rendererTime, rpc, visibleChatState } from './native-chat-4a-probes' type HistoryItem = { itemId: string; body?: { kind?: string; role?: string; state?: string } } export type HistoryPage = { @@ -184,11 +184,15 @@ export async function openChatTab( { worktreeId, sessionId }, { timeout: 60_000 } ) - const id = (await tabId.jsonValue()) as string + const id = await tabId.jsonValue() + if (typeof id !== 'string') { + throw new Error(`${sessionId}: no tab id for the chat`) + } // Unified chat tabs activate through their group; a worktree switch can restore another tab first, so re-apply until it holds. await page.waitForFunction( ({ worktreeId, id }) => { - const state = window.__store!.getState() + const store: AppStore = window.__store! + const state = store.getState() const group = (state.groupsByWorktree[worktreeId] ?? []).find((candidate) => candidate.tabOrder.includes(id) ) @@ -198,7 +202,7 @@ export async function openChatTab( if (state.activeWorktreeId !== worktreeId) { state.setActiveWorktree(worktreeId) } - window.__store!.getState().activateTab(id, { worktreeId }) + store.getState().activateTab(id, { worktreeId }) return false }, { worktreeId, id }, @@ -248,5 +252,5 @@ export async function waitForWorking( } ) }) - return (await handle.jsonValue()) as number + return rendererTime(handle) } diff --git a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts index de021cb3056..242db51cf3d 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-workloads.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-workloads.ts @@ -122,7 +122,7 @@ export const WORKLOADS: Record = { } export function workloadFor(profile: string, conversionSourceBytes?: number): Workload { - const workload = WORKLOADS[profile as Workload['profile']] + const workload = profile === 'smoke' || profile === 'full' ? WORKLOADS[profile] : undefined if (!workload) { throw new Error(`Unknown profile ${profile}; expected ${Object.keys(WORKLOADS).join(', ')}`) } From 2d7525c12e535d6c3669320bb3695b68d652de96 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:56:06 -0800 Subject: [PATCH 14/15] Name the 4a arm's bundled envelope builders by their tracked source files The protocol bundle imported from './src/shared/agent-session/...', a folder the path-literal ratchet reads as an unresolved import. Each module is now a tracked-file constant joined onto the arm's own checkout, the same form runtime-file-watcher-resource-probe.mjs uses; the bundle and its exports are unchanged. --- .../performance/native-chat/native-chat-4a-arm.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm.ts b/config/scripts/performance/native-chat/native-chat-4a-arm.ts index 09feb7e4eba..098883f0b62 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-arm.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-arm.ts @@ -25,6 +25,10 @@ import { installVisibilityProbe } from './native-chat-4a-probes' const HARNESS_ROOT = path.resolve(import.meta.dirname, '../../../..') const SYSTEM_DIRS = ['/usr/bin', '/bin', '/usr/sbin', '/sbin'] +// The envelope builders each arm bundles from its own checkout. +const MUTATION_SOURCE = 'src/shared/agent-session/lifecycle/structured-agent-session-mutation.ts' +const CREATE_SOURCE = 'src/shared/agent-session/structured-agent-session-create.ts' +const SEND_SOURCE = 'src/shared/agent-session/requests/structured-agent-session-send-mutation.ts' /** Envelope builders compiled from the arm's own source, so each host checks fingerprints it computes itself. */ export type ArmProtocol = { @@ -84,13 +88,13 @@ export async function resolveArm(name: string, root: string, scratch: string): P throw new Error(`${name}: the electron package in ${absolute} names no executable`) } const outfile = path.join(scratch, `${name}-protocol.cjs`) - const shared = './src/shared/agent-session' + const source = (file: string): string => JSON.stringify(path.join(absolute, file)) await build({ stdin: { contents: [ - `export * from '${shared}/lifecycle/structured-agent-session-mutation'`, - `export { createStructuredAgentSessionId } from '${shared}/structured-agent-session-create'`, - `export { structuredAgentSessionMessageSendMutation, agentSessionSendBodyFingerprint } from '${shared}/requests/structured-agent-session-send-mutation'` + `export * from ${source(MUTATION_SOURCE)}`, + `export { createStructuredAgentSessionId } from ${source(CREATE_SOURCE)}`, + `export { structuredAgentSessionMessageSendMutation, agentSessionSendBodyFingerprint } from ${source(SEND_SOURCE)}` ].join(';'), resolveDir: absolute }, From 562246bbe73e235d0d3ae68a158192ce9278bc0b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:31:58 -0800 Subject: [PATCH 15/15] Inject the deletion scenarios' record removal at the SQL record store's reads The SQL record store has no transact or record-deletion command, so the authoritative-deletion cases make the store's public record reads answer as for a missing record and restore them before teardown. Assertions unchanged. --- .../automation-terminal-launch.unit.test.ts | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/tests/e2e/automation-terminal-launch.unit.test.ts b/tests/e2e/automation-terminal-launch.unit.test.ts index 81fd0d1fb6d..276f32b3862 100644 --- a/tests/e2e/automation-terminal-launch.unit.test.ts +++ b/tests/e2e/automation-terminal-launch.unit.test.ts @@ -21,6 +21,7 @@ import type { OrcaRuntimeService } from '../../src/main/runtime/orca-runtime' import type { AgentLaunchResult } from '../../src/shared/agent-launch/contracts/agent-launch-intent' import { buildProfileStateCutoverFixture } from '../../src/main/persistence/profile-state-cutover-fixture' import { launchAutomationChatAgent } from '../../src/main/automations/automation-chat-agent-launch' +import { agentSessionRefusalError } from '../../src/shared/agent-session/failures/agent-session-wire-refusals' const mocks = vi.hoisted(() => { const hostState: { @@ -496,19 +497,45 @@ it.each(['before-admission', 'accepted-reply', 'accepted-lost-reply'] as const)( const current = { ...run, createdAt: HOST_TEST_NOW } const rig = await createQueuedMessageTestRig({ restartable: true }) mocks.actualHost = rig.host - let removedRecord = rig.store.getRecord(HOST_TEST_SESSION)! - const removeRecord = () => - rig.store['transact']((draft) => { - removedRecord = draft.records.get(HOST_TEST_SESSION)! - draft.records.delete(HOST_TEST_SESSION) - }) + // The SQL record store has no record-deletion command, so deletion is injected at its record + // reads: every read the refusal and the fresh decision consult answers as for a missing record. + const { store } = rig + const reads = { + select: store.selectWritableRecord, + record: store.getRecord, + conversation: store.getConversation, + require: store.requireWritableRecord + } + const removed: { mockRestore: () => void }[] = [] + const removeRecord = () => { + const deleted = (id: string) => id === HOST_TEST_SESSION + removed.push( + vi + .spyOn(store, 'selectWritableRecord') + .mockImplementation((id) => (deleted(id) ? { kind: 'missing' } : reads.select(id))), + vi + .spyOn(store, 'getRecord') + .mockImplementation((id) => (deleted(id) ? null : reads.record(id))), + vi + .spyOn(store, 'getConversation') + .mockImplementation((id) => (deleted(id) ? null : reads.conversation(id))), + vi.spyOn(store, 'requireWritableRecord').mockImplementation((id) => { + if (deleted(id)) { + throw agentSessionRefusalError('agent_session_identity_required', { + reason: 'recordMissing' + }) + } + return reads.require(id) + }) + ) + } try { const { runtime } = fixture() vi.spyOn(runtime, 'openAgentSessionRecordStore').mockResolvedValue(rig.store) const sent = rig.host.send.bind(rig.host) const send = vi.spyOn(rig.host, 'send').mockImplementationOnce(async (...args) => { if (when === 'before-admission') { - await removeRecord() + removeRecord() } const reply = await sent(...args) if (when === 'before-admission') { @@ -521,7 +548,7 @@ it.each(['before-admission', 'accepted-reply', 'accepted-lost-reply'] as const)( }) } else { expect(reply.ok).toBe(true) - await removeRecord() + removeRecord() if (when === 'accepted-lost-reply') { throw new Error('accepted reply lost after deletion') } @@ -566,9 +593,9 @@ it.each(['before-admission', 'accepted-reply', 'accepted-lost-reply'] as const)( } finally { mocks.actualHost = null // Restore the injected deletion only after assertions so ordinary host teardown can run. - await rig.store['transact']((draft) => { - draft.records.set(HOST_TEST_SESSION, removedRecord) - }) + for (const read of removed) { + read.mockRestore() + } await rig.dispose() } }