diff --git a/.gitignore b/.gitignore index 17bb18529af..64854199a35 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,7 @@ docs/** !docs/reference/linux-glibc-compatibility.md !docs/reference/relay-grace-time-reconfiguration.md !docs/reference/remote-wire-compatibility.md +!docs/reference/renderer-agent-status-performance.md !docs/reference/windows-setup-shell.md # Stably CLI (only docs/ are tracked) diff --git a/config/scripts/idle-cpu-process-sampling.mjs b/config/scripts/idle-cpu-process-sampling.mjs new file mode 100644 index 00000000000..81f9218e5ad --- /dev/null +++ b/config/scripts/idle-cpu-process-sampling.mjs @@ -0,0 +1,323 @@ +import { execFileSync, spawnSync } from 'node:child_process' + +function parseCpuTimeSeconds(value) { + const trimmed = String(value || '').trim() + if (!trimmed) { + return null + } + const [dayOrTime, maybeTime] = trimmed.includes('-') ? trimmed.split('-', 2) : [null, trimmed] + const days = dayOrTime === null ? 0 : Number(dayOrTime) + const parts = maybeTime.split(':').map(Number) + if (!Number.isFinite(days) || parts.some((part) => !Number.isFinite(part))) { + return null + } + if (parts.length === 3) { + return days * 86400 + parts[0] * 3600 + parts[1] * 60 + parts[2] + } + if (parts.length === 2) { + return days * 86400 + parts[0] * 60 + parts[1] + } + if (parts.length === 1) { + return days * 86400 + parts[0] + } + return null +} + +function parseUnixProcesses(stdout) { + const rows = [] + for (const raw of stdout.split('\n')) { + const line = raw.trim() + if (!line) { + continue + } + const match = line.match(/^(\d+)\s+(\d+)\s+([\d.]+)\s+(\d+)\s+(\S+)\s+(.+)$/) + if (!match) { + continue + } + rows.push({ + pid: Number(match[1]), + ppid: Number(match[2]), + percentCpu: Number(match[3]), + rssBytes: Number(match[4]) * 1024, + cpuTimeSeconds: parseCpuTimeSeconds(match[5]), + command: match[6] + }) + } + return rows +} + +function readUnixProcesses() { + const stdout = execFileSync('ps', ['-axo', 'pid=,ppid=,pcpu=,rss=,cputime=,command='], { + encoding: 'utf8', + env: { ...process.env, LC_ALL: 'C', LANG: 'C' }, + maxBuffer: 20 * 1024 * 1024 + }) + return parseUnixProcesses(stdout) +} + +function readWindowsProcesses() { + const script = + 'Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,WorkingSetSize,CommandLine | ConvertTo-Json -Compress' + const result = spawnSync('powershell.exe', ['-NoProfile', '-Command', script], { + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }) + if (result.status !== 0) { + throw new Error(result.stderr || 'PowerShell process enumeration failed') + } + const parsed = JSON.parse(result.stdout || '[]') + const entries = Array.isArray(parsed) ? parsed : [parsed] + return entries.map((entry) => ({ + pid: Number(entry.ProcessId), + ppid: Number(entry.ParentProcessId), + percentCpu: 0, + cpuTimeSeconds: null, + rssBytes: Number(entry.WorkingSetSize) || 0, + command: String(entry.CommandLine || '') + })) +} + +export function readProcessRows() { + return process.platform === 'win32' ? readWindowsProcesses() : readUnixProcesses() +} + +export function descendantsOf(rows, rootPid) { + const children = new Map() + for (const row of rows) { + const list = children.get(row.ppid) ?? [] + list.push(row) + children.set(row.ppid, list) + } + const result = [] + const stack = [rootPid] + const seen = new Set() + while (stack.length > 0) { + const pid = stack.pop() + if (seen.has(pid)) { + continue + } + seen.add(pid) + const row = rows.find((candidate) => candidate.pid === pid) + if (row) { + result.push(row) + } + for (const child of children.get(pid) ?? []) { + stack.push(child.pid) + } + } + return result +} + +export function classify(row, rootPid) { + const command = row.command.toLowerCase() + if (row.pid === rootPid) { + return 'main' + } + if (command.includes('daemon-entry')) { + return 'daemon' + } + if (command.includes('--type=gpu-process')) { + return 'gpu' + } + if (command.includes('--type=renderer')) { + return 'renderer' + } + if (command.includes('--type=utility')) { + return 'utility' + } + if (command.includes('--type=')) { + return 'electron-other' + } + if (command.includes('node') || command.includes('/pi') || command.endsWith(' pi')) { + return 'agent-or-node' + } + return 'other-descendant' +} + +const DEFAULT_WORKLOAD_OVERRUN_LIMIT_MS = 120_000 + +export async function sampleProcessTreeUntilWorkloadsComplete({ + rootPid, + requestedDurationMs, + intervalMs, + workloadPromise, + maxWorkloadOverrunMs = DEFAULT_WORKLOAD_OVERRUN_LIMIT_MS, + readRows = readProcessRows, + now = Date.now, + wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +}) { + const samplingStartedAt = now() + const requestedDeadline = samplingStartedAt + requestedDurationMs + const hardDeadline = requestedDeadline + maxWorkloadOverrunMs + let workloadSettled = false + let workloadResult + let workloadError + let workloadSettledAt = null + void workloadPromise.then( + (result) => { + workloadResult = result + workloadSettled = true + workloadSettledAt = now() + }, + (error) => { + workloadError = error + workloadSettled = true + workloadSettledAt = now() + } + ) + const samples = [] + let previousSnapshot = null + const needsFinalWorkloadSample = () => + workloadSettledAt !== null && (previousSnapshot?.at ?? -Infinity) < workloadSettledAt + while ( + now() <= requestedDeadline || + samples.length === 0 || + !workloadSettled || + needsFinalWorkloadSample() + ) { + const sampledAt = now() + if (workloadError) { + throw workloadError + } + if ( + (!workloadSettled && sampledAt >= hardDeadline) || + (workloadSettledAt !== null && workloadSettledAt > hardDeadline) + ) { + throw new Error( + `Benchmark workload exceeded the ${maxWorkloadOverrunMs}ms sampling overrun limit` + ) + } + const processRows = descendantsOf(readRows(), rootPid) + const rawProcesses = processRows.map((row) => ({ ...row, kind: classify(row, rootPid) })) + if (previousSnapshot) { + const elapsedSeconds = Math.max(0.001, (sampledAt - previousSnapshot.at) / 1000) + const previousByPid = new Map(previousSnapshot.processes.map((proc) => [proc.pid, proc])) + const processes = rawProcesses.map((row) => { + const previous = previousByPid.get(row.pid) + const canComputeDelta = + typeof row.cpuTimeSeconds === 'number' && typeof previous?.cpuTimeSeconds === 'number' + const cpu = canComputeDelta + ? Math.max(0, ((row.cpuTimeSeconds - previous.cpuTimeSeconds) / elapsedSeconds) * 100) + : row.percentCpu + return { ...row, cpu } + }) + samples.push({ + at: sampledAt, + elapsedMs: sampledAt - previousSnapshot.at, + totalCpuPercent: processes.reduce((sum, proc) => sum + proc.cpu, 0), + totalRssBytes: processes.reduce((sum, proc) => sum + proc.rssBytes, 0), + processes + }) + } + previousSnapshot = { at: sampledAt, processes: rawProcesses } + await wait(intervalMs) + } + if (workloadError) { + throw workloadError + } + const measuredDurationMs = Math.max( + 0, + (previousSnapshot?.at ?? samplingStartedAt) - samplingStartedAt + ) + return { + samples, + workloadResult, + samplingWindow: { + requestedDurationMs, + measuredDurationMs, + maxWorkloadOverrunMs, + extendedForWorkload: workloadSettledAt !== null && workloadSettledAt > requestedDeadline, + workloadSettledElapsedMs: + workloadSettledAt === null ? null : Math.max(0, workloadSettledAt - samplingStartedAt), + workloadSettledBeforeStop: workloadSettled + } + } +} + +function mean(values) { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length +} + +function percentile(sorted, fraction) { + if (sorted.length === 0) { + return 0 + } + const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1) + return sorted[index] +} + +export function summarizeSamples(samples) { + const byKind = new Map() + for (const sample of samples) { + for (const proc of sample.processes) { + const bucket = byKind.get(proc.kind) ?? { cpuValues: [], rssValues: [], maxProcessCount: 0 } + bucket.cpuValues.push(proc.cpu) + bucket.rssValues.push(proc.rssBytes) + byKind.set(proc.kind, bucket) + } + const counts = new Map() + for (const proc of sample.processes) { + counts.set(proc.kind, (counts.get(proc.kind) ?? 0) + 1) + } + for (const [kind, count] of counts) { + byKind.get(kind).maxProcessCount = Math.max(byKind.get(kind).maxProcessCount, count) + } + } + const summary = {} + for (const [kind, values] of byKind) { + const cpuSorted = [...values.cpuValues].sort((a, b) => a - b) + const rssSumBySample = samples.map((sample) => + sample.processes + .filter((proc) => proc.kind === kind) + .reduce((sum, proc) => sum + proc.rssBytes, 0) + ) + summary[kind] = { + meanCpuPercent: mean(values.cpuValues), + p95CpuPercent: percentile(cpuSorted, 0.95), + maxCpuPercent: Math.max(0, ...values.cpuValues), + meanRssBytes: mean(rssSumBySample), + maxProcessCount: values.maxProcessCount + } + } + summary.total = { + meanCpuPercent: mean(samples.map((sample) => sample.totalCpuPercent)), + p95CpuPercent: percentile( + samples.map((sample) => sample.totalCpuPercent).sort((a, b) => a - b), + 0.95 + ), + meanRssBytes: mean(samples.map((sample) => sample.totalRssBytes)) + } + return summary +} + +export function summarizeProcessInventory(samples) { + const inventory = {} + for (const sample of samples) { + const counts = new Map() + for (const proc of sample.processes) { + counts.set(proc.kind, (counts.get(proc.kind) ?? 0) + 1) + const entry = inventory[proc.kind] ?? { + maxProcessCount: 0, + maxCpuPercent: 0, + commandSamples: [] + } + entry.maxCpuPercent = Math.max(entry.maxCpuPercent, proc.cpu) + if (!entry.commandSamples.includes(proc.command) && entry.commandSamples.length < 6) { + entry.commandSamples.push(proc.command) + } + inventory[proc.kind] = entry + } + for (const [kind, count] of counts) { + inventory[kind].maxProcessCount = Math.max(inventory[kind].maxProcessCount, count) + } + } + return inventory +} + +export function terminateProcesses(processes) { + for (const proc of processes) { + try { + process.kill(proc.pid) + } catch {} + } +} diff --git a/config/scripts/idle-cpu-process-sampling.test.mjs b/config/scripts/idle-cpu-process-sampling.test.mjs new file mode 100644 index 00000000000..d452c6e3ce0 --- /dev/null +++ b/config/scripts/idle-cpu-process-sampling.test.mjs @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest' +import { sampleProcessTreeUntilWorkloadsComplete } from './idle-cpu-process-sampling.mjs' + +function createClock(workloadCompletesAt) { + let currentMs = 0 + let resolveWorkload + const workloadPromise = new Promise((resolve) => { + resolveWorkload = resolve + }) + const wait = async (durationMs) => { + currentMs += durationMs + if (currentMs >= workloadCompletesAt) { + resolveWorkload('complete') + } + await Promise.resolve() + } + const readRows = () => [ + { + pid: 10, + ppid: 0, + percentCpu: 0, + rssBytes: 1_024, + cpuTimeSeconds: currentMs / 2_000, + command: 'electron' + } + ] + return { now: () => currentMs, readRows, wait, workloadPromise } +} + +describe('idle CPU process sampling window', () => { + it('extends through a slow workload and captures a final CPU delta', async () => { + const clock = createClock(40) + const result = await sampleProcessTreeUntilWorkloadsComplete({ + rootPid: 10, + requestedDurationMs: 20, + intervalMs: 10, + maxWorkloadOverrunMs: 100, + ...clock + }) + + expect(result.workloadResult).toBe('complete') + expect(result.samples.map((sample) => sample.at)).toEqual([10, 20, 30, 40]) + expect(result.samplingWindow).toEqual({ + requestedDurationMs: 20, + measuredDurationMs: 40, + maxWorkloadOverrunMs: 100, + extendedForWorkload: true, + workloadSettledElapsedMs: 40, + workloadSettledBeforeStop: true + }) + }) + + it('invalidates a run that reaches the workload overrun guard', async () => { + const clock = createClock(Infinity) + await expect( + sampleProcessTreeUntilWorkloadsComplete({ + rootPid: 10, + requestedDurationMs: 20, + intervalMs: 10, + maxWorkloadOverrunMs: 20, + ...clock + }) + ).rejects.toThrow('exceeded the 20ms sampling overrun limit') + }) +}) diff --git a/config/scripts/idle-cpu-renderer-scale-fixture.mjs b/config/scripts/idle-cpu-renderer-scale-fixture.mjs new file mode 100644 index 00000000000..4759b768e27 --- /dev/null +++ b/config/scripts/idle-cpu-renderer-scale-fixture.mjs @@ -0,0 +1,202 @@ +export async function configureRendererScaleFixture(page, options, repoPath) { + return page.evaluate( + ({ agentsPerWorktree, lineageDepth, repoPath }) => { + const store = window.__store + if (!store) { + throw new Error('window.__store is not available') + } + const normalizePath = (value) => + String(value ?? '') + .replaceAll('\\', '/') + .toLowerCase() + const primaryPath = normalizePath(repoPath) + const compare = (left, right) => (left < right ? -1 : left > right ? 1 : 0) + const state = store.getState() + const worktrees = Object.values(state.worktreesByRepo) + .flat() + .filter((worktree) => !worktree.isArchived) + .sort((left, right) => { + const primaryOrder = + Number(normalizePath(right.path) === primaryPath) - + Number(normalizePath(left.path) === primaryPath) + return ( + primaryOrder || + compare(normalizePath(left.path), normalizePath(right.path)) || + compare(left.id, right.id) + ) + }) + const appliedLineageDepth = Math.min(lineageDepth, Math.max(0, worktrees.length - 1)) + if (lineageDepth === 0 && agentsPerWorktree === 0) { + return { + applied: false, + requestedLineageDepth: lineageDepth, + appliedLineageDepth, + agentsPerWorktree, + seededAgentRows: 0 + } + } + + state.setActiveView('terminal') + state.setSidebarOpen(true) + state.setGroupBy('none') + state.setSortBy('recent') + state.setShowActiveOnly(false) + state.setShowSleepingWorkspaces(true) + state.setHideDefaultBranchWorkspace(false) + state.setFilterRepoIds([]) + + const lineageById = { ...store.getState().worktreeLineageById } + const lineageParentIds = new Set() + if (appliedLineageDepth > 0) { + for (const worktree of worktrees) { + delete lineageById[worktree.id] + if (!worktree.instanceId) { + throw new Error(`Worktree ${worktree.id} has no instanceId for lineage seeding`) + } + } + for (let index = 1; index < worktrees.length; index += 1) { + const child = worktrees[index] + const parent = worktrees[Math.min(index - 1, appliedLineageDepth - 1)] + lineageParentIds.add(parent.id) + lineageById[child.id] = { + worktreeId: child.id, + worktreeInstanceId: child.instanceId, + parentWorktreeId: parent.id, + parentWorktreeInstanceId: parent.instanceId, + origin: 'manual', + capture: { source: 'manual-action', confidence: 'explicit' }, + createdAt: 1_700_000_000_000 + index + } + } + } + const collapsedGroups = new Set(store.getState().collapsedGroups) + for (const parentId of lineageParentIds) { + collapsedGroups.delete(`lineage:${parentId}`) + } + store.setState({ worktreeLineageById: lineageById, collapsedGroups }) + + let seededAgentRows = 0 + if (agentsPerWorktree > 0) { + store.getState().setWorktreeCardMode('Default') + store.getState().setAgentActivityDisplayMode('full') + const fixtureNow = Date.now() + worktrees.forEach((worktree, worktreeIndex) => { + const next = store.getState() + const tab = + next.tabsByWorktree[worktree.id]?.[0] ?? + next.createTab(worktree.id, undefined, undefined, { + activate: false, + id: `idle-cpu-tab-${worktreeIndex}` + }) + for (let agentIndex = 0; agentIndex < agentsPerWorktree; agentIndex += 1) { + const agentType = agentIndex % 2 === 0 ? 'codex' : 'claude' + const leafSequence = + BigInt(worktreeIndex) * BigInt(agentsPerWorktree) + BigInt(agentIndex + 1) + const leafId = `00000000-0000-4000-8000-${leafSequence.toString(16).padStart(12, '0')}` + store.getState().setAgentStatus( + `${tab.id}:${leafId}`, + { + state: 'working', + prompt: `Idle CPU agent ${worktreeIndex + 1}.${agentIndex + 1}`, + agentType + }, + agentType, + { updatedAt: fixtureNow, stateStartedAt: fixtureNow }, + { tabId: tab.id, worktreeId: worktree.id } + ) + seededAgentRows += 1 + } + }) + } + return { + applied: true, + requestedLineageDepth: lineageDepth, + appliedLineageDepth, + lineageEdges: appliedLineageDepth > 0 ? Math.max(0, worktrees.length - 1) : 0, + expandedLineageGroups: lineageParentIds.size, + agentsPerWorktree, + seededAgentRows, + orderedWorktreeIds: worktrees.map((worktree) => worktree.id) + } + }, + { agentsPerWorktree: options.agentsPerWorktree, lineageDepth: options.lineageDepth, repoPath } + ) +} + +export async function collectRendererCensus(page, configuredLineageDepth) { + return page.evaluate((configuredDepth) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('window.__store is not available') + } + const worktrees = Object.values(state.worktreesByRepo).flat() + const worktreeIds = new Set(worktrees.map((worktree) => worktree.id)) + const lineageDepthById = new Map() + const getDepth = (worktreeId, trail = new Set()) => { + if (lineageDepthById.has(worktreeId)) { + return lineageDepthById.get(worktreeId) + } + const lineage = state.worktreeLineageById[worktreeId] + if (!lineage || !worktreeIds.has(lineage.parentWorktreeId) || trail.has(worktreeId)) { + return 0 + } + const nextTrail = new Set(trail) + nextTrail.add(worktreeId) + const depth = 1 + getDepth(lineage.parentWorktreeId, nextTrail) + lineageDepthById.set(worktreeId, depth) + return depth + } + const logicalDepths = worktrees.map((worktree) => getDepth(worktree.id)) + const lineageParentIds = new Set( + Object.values(state.worktreeLineageById) + .filter((lineage) => worktreeIds.has(lineage.worktreeId)) + .map((lineage) => lineage.parentWorktreeId) + ) + const sidebar = document.querySelector('[data-worktree-sidebar]') + const mountedWorktreeIds = [ + ...new Set( + [...(sidebar?.querySelectorAll('[data-worktree-id]') ?? [])] + .map((element) => element.getAttribute('data-worktree-id')) + .filter(Boolean) + ) + ] + const mountedAgentRows = [...(sidebar?.querySelectorAll('*') ?? [])].filter( + (element) => + element.classList.contains('group/agent-row') || + element.classList.contains('compact-agent-row') + ).length + let diagnosticCensus = null + try { + diagnosticCensus = window.__orcaTypingDiagnostic?.report().census ?? null + } catch {} + const collapsedLineageGroups = [...lineageParentIds].filter((parentId) => + state.collapsedGroups.has(`lineage:${parentId}`) + ).length + return { + capturedAt: new Date().toISOString(), + worktrees: { + store: worktrees.length, + mountedCards: sidebar?.querySelectorAll('[data-worktree-card-surface]').length ?? 0, + mountedUnique: mountedWorktreeIds.length, + mountedIds: mountedWorktreeIds.slice(0, 100), + mountedIdsTruncated: Math.max(0, mountedWorktreeIds.length - 100) + }, + agentRows: { + storeLive: Object.keys(state.agentStatusByPaneKey ?? {}).length, + storeRetained: Object.keys(state.retainedAgentsByPaneKey ?? {}).length, + mounted: mountedAgentRows, + diagnosticMounted: diagnosticCensus?.agentRows.mountedDom ?? null + }, + storeListeners: diagnosticCensus?.storeListeners ?? null, + lineage: { + configuredDepth, + measuredMaxDepth: Math.max(0, ...logicalDepths), + edges: logicalDepths.filter((depth) => depth > 0).length, + groups: lineageParentIds.size, + expandedGroups: lineageParentIds.size - collapsedLineageGroups, + collapsedGroups: collapsedLineageGroups + }, + diagnostic: diagnosticCensus + } + }, configuredLineageDepth) +} diff --git a/config/scripts/idle-cpu-renderer-timing-probe.mjs b/config/scripts/idle-cpu-renderer-timing-probe.mjs new file mode 100644 index 00000000000..b010bcb3433 --- /dev/null +++ b/config/scripts/idle-cpu-renderer-timing-probe.mjs @@ -0,0 +1,191 @@ +const RENDERER_TIMER_INTERVAL_MS = 100 + +export async function startRendererTimingProbe(page) { + await page.evaluate((timerIntervalMs) => { + const maxSamples = 5_000 + const maxEntries = 80 + let phaseStartedAt = performance.now() + let phaseStartedAtIso = new Date().toISOString() + let timerId = null + let driftCount = 0 + let driftSamples = [] + let longTaskEntries = [] + let observer = null + let longTaskSupported = false + + const round = (value) => Math.round(value * 100) / 100 + const summarize = (values, totalCount = values.length) => { + const sorted = [...values].sort((left, right) => left - right) + const percentile = (fraction) => + sorted.length === 0 + ? null + : sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] + return { + count: totalCount, + retainedCount: sorted.length, + mean: + sorted.length === 0 + ? null + : round(sorted.reduce((sum, value) => sum + value, 0) / sorted.length), + p50: sorted.length === 0 ? null : round(percentile(0.5)), + p95: sorted.length === 0 ? null : round(percentile(0.95)), + max: sorted.length === 0 ? null : round(sorted.at(-1)) + } + } + const recordLongTasks = (entries) => { + for (const entry of entries) { + longTaskEntries.push({ + startTime: entry.startTime, + duration: entry.duration, + name: entry.name + }) + if (longTaskEntries.length > maxSamples) { + longTaskEntries.shift() + } + } + } + try { + longTaskSupported = PerformanceObserver.supportedEntryTypes?.includes('longtask') === true + if (longTaskSupported) { + observer = new PerformanceObserver((list) => recordLongTasks(list.getEntries())) + observer.observe({ type: 'longtask', buffered: true }) + } + } catch { + observer = null + longTaskSupported = false + } + const scheduleTimer = () => { + const expectedAt = performance.now() + timerIntervalMs + timerId = setTimeout(() => { + driftCount += 1 + driftSamples.push(Math.max(0, performance.now() - expectedAt)) + if (driftSamples.length > maxSamples) { + driftSamples.shift() + } + scheduleTimer() + }, timerIntervalMs) + } + const snapshot = (reset) => { + if (observer) { + recordLongTasks(observer.takeRecords()) + } + const capturedAt = performance.now() + const phaseLongTasks = longTaskEntries.filter((entry) => entry.startTime >= phaseStartedAt) + const durations = phaseLongTasks.map((entry) => entry.duration) + const result = { + startedAt: phaseStartedAtIso, + capturedAt: new Date().toISOString(), + durationMs: round(capturedAt - phaseStartedAt), + timerIntervalMs, + timerDriftMs: summarize(driftSamples, driftCount), + longTasks: { + supported: longTaskSupported, + ...summarize(durations), + totalDurationMs: round(durations.reduce((sum, value) => sum + value, 0)), + entries: phaseLongTasks.slice(-maxEntries).map((entry) => ({ + startMs: round(entry.startTime - phaseStartedAt), + durationMs: round(entry.duration), + name: entry.name + })), + entriesTruncated: Math.max(0, phaseLongTasks.length - maxEntries) + } + } + if (reset) { + phaseStartedAt = capturedAt + phaseStartedAtIso = new Date().toISOString() + driftCount = 0 + driftSamples = [] + longTaskEntries = [] + } + return result + } + scheduleTimer() + window.__orcaIdleCpuTimingProbe = { + snapshot: () => snapshot(true), + stop: () => { + if (timerId !== null) { + clearTimeout(timerId) + } + const result = snapshot(false) + observer?.disconnect() + return result + } + } + }, RENDERER_TIMER_INTERVAL_MS) +} + +export async function snapshotRendererTimingProbe(page) { + return page.evaluate(() => window.__orcaIdleCpuTimingProbe?.snapshot() ?? null) +} + +export async function stopRendererTimingProbe(page) { + return page.evaluate(() => window.__orcaIdleCpuTimingProbe?.stop() ?? null) +} + +export async function runZustandPublications(page, count, intervalMs) { + return page.evaluate( + ({ count, intervalMs }) => + new Promise((resolve, reject) => { + const store = window.__store + if (!store) { + reject(new Error('window.__store is not available')) + return + } + const maxSamples = 5_000 + const startedAt = performance.now() + const startedAtIso = new Date().toISOString() + const schedulingDriftMs = [] + let completed = 0 + const finish = () => { + const sorted = [...schedulingDriftMs].sort((left, right) => left - right) + const percentile = (fraction) => + sorted.length === 0 + ? null + : sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] + const round = (value) => Math.round(value * 100) / 100 + resolve({ + requested: count, + completed, + intervalMs, + startedAt: startedAtIso, + completedAt: new Date().toISOString(), + durationMs: round(performance.now() - startedAt), + schedulingDriftMs: { + retainedCount: sorted.length, + p50: sorted.length === 0 ? null : round(percentile(0.5)), + p95: sorted.length === 0 ? null : round(percentile(0.95)), + max: sorted.length === 0 ? null : round(sorted.at(-1)) + } + }) + } + const publish = () => { + const publishedAt = performance.now() + const scheduledAt = startedAt + completed * intervalMs + schedulingDriftMs.push(Math.max(0, publishedAt - scheduledAt)) + if (schedulingDriftMs.length > maxSamples) { + schedulingDriftMs.shift() + } + try { + // Why: an empty partial notifies every subscriber without changing domain state. + store.setState({}) + } catch (error) { + reject(error) + return + } + completed += 1 + if (completed >= count) { + finish() + return + } + const nextAt = startedAt + completed * intervalMs + setTimeout(publish, Math.max(0, nextAt - performance.now())) + } + if (count === 0) { + finish() + } else { + setTimeout(publish, 0) + } + }), + { count, intervalMs } + ) +} diff --git a/config/scripts/run-idle-cpu-benchmark.mjs b/config/scripts/run-idle-cpu-benchmark.mjs index 53cabd3380b..3892de8845d 100644 --- a/config/scripts/run-idle-cpu-benchmark.mjs +++ b/config/scripts/run-idle-cpu-benchmark.mjs @@ -1,15 +1,34 @@ #!/usr/bin/env node import { _electron as electron } from '@stablyai/playwright-test' -import { execFileSync, spawnSync } from 'node:child_process' +import { execFileSync } from 'node:child_process' import { existsSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' import os from 'node:os' import path from 'node:path' +import { + descendantsOf, + readProcessRows, + sampleProcessTreeUntilWorkloadsComplete, + summarizeProcessInventory, + summarizeSamples, + terminateProcesses +} from './idle-cpu-process-sampling.mjs' +import { + collectRendererCensus, + configureRendererScaleFixture +} from './idle-cpu-renderer-scale-fixture.mjs' +import { + runZustandPublications, + snapshotRendererTimingProbe, + startRendererTimingProbe, + stopRendererTimingProbe +} from './idle-cpu-renderer-timing-probe.mjs' import { installSyntheticVisibleSpinners } from './idle-cpu-synthetic-spinners.mjs' const DEFAULT_WARMUP_MS = 15_000 const DEFAULT_SAMPLE_MS = 30_000 const DEFAULT_INTERVAL_MS = 1_000 const DEFAULT_WORKTREE_COUNT = 1 +const DEFAULT_ZUSTAND_PUBLICATION_INTERVAL_MS = 100 const ONBOARDING_FINAL_STEP = 3 const ONBOARDING_FLOW_VERSION = 2 @@ -19,6 +38,10 @@ function parseArgs(argv) { sampleMs: DEFAULT_SAMPLE_MS, intervalMs: DEFAULT_INTERVAL_MS, worktrees: DEFAULT_WORKTREE_COUNT, + lineageDepth: 0, + agentsPerWorktree: 0, + zustandPublications: 0, + zustandPublicationIntervalMs: DEFAULT_ZUSTAND_PUBLICATION_INTERVAL_MS, skipBuild: false, headful: false, output: null, @@ -47,6 +70,14 @@ function parseArgs(argv) { options.intervalMs = Number(readValue()) } else if (arg === '--worktrees') { options.worktrees = Number(readValue()) + } else if (arg === '--lineage-depth') { + options.lineageDepth = Number(readValue()) + } else if (arg === '--agents-per-worktree') { + options.agentsPerWorktree = Number(readValue()) + } else if (arg === '--zustand-publications') { + options.zustandPublications = Number(readValue()) + } else if (arg === '--zustand-publication-interval-ms') { + options.zustandPublicationIntervalMs = Number(readValue()) } else if (arg === '--output') { options.output = readValue() } else if (arg === '--skip-build') { @@ -73,6 +104,10 @@ function parseArgs(argv) { 'sampleMs', 'intervalMs', 'worktrees', + 'lineageDepth', + 'agentsPerWorktree', + 'zustandPublications', + 'zustandPublicationIntervalMs', 'syntheticVisibleSpinners', 'syntheticSpinnerSteps' ]) { @@ -82,16 +117,33 @@ function parseArgs(argv) { } options.worktrees = Math.max(1, Math.floor(options.worktrees)) options.intervalMs = Math.max(250, Math.floor(options.intervalMs)) + options.lineageDepth = Math.floor(options.lineageDepth) + options.agentsPerWorktree = Math.floor(options.agentsPerWorktree) + options.zustandPublications = Math.floor(options.zustandPublications) + options.zustandPublicationIntervalMs = Math.max( + 1, + Math.floor(options.zustandPublicationIntervalMs) + ) options.syntheticVisibleSpinners = Math.max(0, Math.floor(options.syntheticVisibleSpinners)) options.syntheticSpinnerSteps = Math.max(1, Math.floor(options.syntheticSpinnerSteps)) if (!['smooth', 'steps'].includes(options.syntheticSpinnerAnimation)) { throw new Error(`Invalid --synthetic-spinner-animation: ${options.syntheticSpinnerAnimation}`) } + if (options.lineageDepth > 0 && options.worktrees < 2) { + throw new Error('--lineage-depth requires at least two --worktrees') + } + const publicationSpanMs = + Math.max(0, options.zustandPublications - 1) * options.zustandPublicationIntervalMs + if (publicationSpanMs > options.sampleMs) { + throw new Error( + `Zustand publication span ${publicationSpanMs}ms exceeds --sample-ms ${options.sampleMs}` + ) + } return options } function printUsage() { console.log( - `Usage: node config/scripts/run-idle-cpu-benchmark.mjs [options]\n\nOptions:\n --warmup-ms Time to wait after app readiness before sampling (default ${DEFAULT_WARMUP_MS})\n --sample-ms Sampling window duration (default ${DEFAULT_SAMPLE_MS})\n --interval-ms Sampling cadence (default ${DEFAULT_INTERVAL_MS})\n --worktrees Seed repo worktree count, including primary (default ${DEFAULT_WORKTREE_COUNT})\n --headful Show the Electron window while measuring\n --skip-build Reuse out/main/index.js instead of building first\n --output Write JSON report to this path\n --disable-renderer-animations Inject measurement-only CSS that disables animations/transitions\n --synthetic-visible-spinners Measurement-only: add visible working spinners\n --synthetic-spinner-animation Spinner animation style (default smooth)\n --synthetic-spinner-steps Step count for --synthetic-spinner-animation steps (default 12)\n` + `Usage: node config/scripts/run-idle-cpu-benchmark.mjs [options]\n\nOptions:\n --warmup-ms Time to wait after app readiness before sampling (default ${DEFAULT_WARMUP_MS})\n --sample-ms Sampling window duration (default ${DEFAULT_SAMPLE_MS})\n --interval-ms Sampling cadence (default ${DEFAULT_INTERVAL_MS})\n --worktrees Seed repo worktree count, including primary (default ${DEFAULT_WORKTREE_COUNT})\n --lineage-depth Nest all worktrees under one expanded lineage, up to this depth\n --agents-per-worktree Seed this many visible inline agent rows per worktree\n --zustand-publications Publish exactly this many store updates during sampling\n --zustand-publication-interval-ms Publication cadence (default ${DEFAULT_ZUSTAND_PUBLICATION_INTERVAL_MS})\n --headful Show the Electron window while measuring\n --skip-build Reuse out/main/index.js instead of building first\n --output Write JSON report to this path\n --disable-renderer-animations Inject measurement-only CSS that disables animations/transitions\n --synthetic-visible-spinners Measurement-only: add visible working spinners\n --synthetic-spinner-animation Spinner animation style (default smooth)\n --synthetic-spinner-steps Step count for --synthetic-spinner-animation steps (default 12)\n` ) } function run(command, args, options = {}) { @@ -188,140 +240,6 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)) } -function parseCpuTimeSeconds(value) { - const trimmed = String(value || '').trim() - if (!trimmed) { - return null - } - const [dayOrTime, maybeTime] = trimmed.includes('-') ? trimmed.split('-', 2) : [null, trimmed] - const days = dayOrTime === null ? 0 : Number(dayOrTime) - const parts = maybeTime.split(':').map(Number) - if (!Number.isFinite(days) || parts.some((part) => !Number.isFinite(part))) { - return null - } - if (parts.length === 3) { - return days * 86400 + parts[0] * 3600 + parts[1] * 60 + parts[2] - } - if (parts.length === 2) { - return days * 86400 + parts[0] * 60 + parts[1] - } - if (parts.length === 1) { - return days * 86400 + parts[0] - } - return null -} - -function parseUnixProcesses(stdout) { - const rows = [] - for (const raw of stdout.split('\n')) { - const line = raw.trim() - if (!line) { - continue - } - const match = line.match(/^(\d+)\s+(\d+)\s+([\d.]+)\s+(\d+)\s+(\S+)\s+(.+)$/) - if (!match) { - continue - } - rows.push({ - pid: Number(match[1]), - ppid: Number(match[2]), - percentCpu: Number(match[3]), - rssBytes: Number(match[4]) * 1024, - cpuTimeSeconds: parseCpuTimeSeconds(match[5]), - command: match[6] - }) - } - return rows -} - -function readUnixProcesses() { - const stdout = execFileSync('ps', ['-axo', 'pid=,ppid=,pcpu=,rss=,cputime=,command='], { - encoding: 'utf8', - env: { ...process.env, LC_ALL: 'C', LANG: 'C' }, - maxBuffer: 20 * 1024 * 1024 - }) - return parseUnixProcesses(stdout) -} - -function readWindowsProcesses() { - const script = - 'Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,WorkingSetSize,CommandLine | ConvertTo-Json -Compress' - const result = spawnSync('powershell.exe', ['-NoProfile', '-Command', script], { - encoding: 'utf8', - maxBuffer: 20 * 1024 * 1024 - }) - if (result.status !== 0) { - throw new Error(result.stderr || 'PowerShell process enumeration failed') - } - const parsed = JSON.parse(result.stdout || '[]') - const entries = Array.isArray(parsed) ? parsed : [parsed] - return entries.map((entry) => ({ - pid: Number(entry.ProcessId), - ppid: Number(entry.ParentProcessId), - percentCpu: 0, - cpuTimeSeconds: null, - rssBytes: Number(entry.WorkingSetSize) || 0, - command: String(entry.CommandLine || '') - })) -} - -function readProcessRows() { - return process.platform === 'win32' ? readWindowsProcesses() : readUnixProcesses() -} - -function descendantsOf(rows, rootPid) { - const children = new Map() - for (const row of rows) { - const list = children.get(row.ppid) ?? [] - list.push(row) - children.set(row.ppid, list) - } - const result = [] - const stack = [rootPid] - const seen = new Set() - while (stack.length > 0) { - const pid = stack.pop() - if (seen.has(pid)) { - continue - } - seen.add(pid) - const row = rows.find((candidate) => candidate.pid === pid) - if (row) { - result.push(row) - } - for (const child of children.get(pid) ?? []) { - stack.push(child.pid) - } - } - return result -} - -function classify(row, rootPid) { - const command = row.command.toLowerCase() - if (row.pid === rootPid) { - return 'main' - } - if (command.includes('daemon-entry')) { - return 'daemon' - } - if (command.includes('--type=gpu-process')) { - return 'gpu' - } - if (command.includes('--type=renderer')) { - return 'renderer' - } - if (command.includes('--type=utility')) { - return 'utility' - } - if (command.includes('--type=')) { - return 'electron-other' - } - if (command.includes('node') || command.includes('/pi') || command.endsWith(' pi')) { - return 'agent-or-node' - } - return 'other-descendant' -} - async function collectRendererIdleState(page) { return page.evaluate(() => { const describeElement = (element) => { @@ -364,93 +282,6 @@ async function collectRendererIdleState(page) { }) } -function summarizeSamples(samples) { - const byKind = new Map() - for (const sample of samples) { - for (const proc of sample.processes) { - const bucket = byKind.get(proc.kind) ?? { cpuValues: [], rssValues: [], maxProcessCount: 0 } - bucket.cpuValues.push(proc.cpu) - bucket.rssValues.push(proc.rssBytes) - byKind.set(proc.kind, bucket) - } - const counts = new Map() - for (const proc of sample.processes) { - counts.set(proc.kind, (counts.get(proc.kind) ?? 0) + 1) - } - for (const [kind, count] of counts) { - byKind.get(kind).maxProcessCount = Math.max(byKind.get(kind).maxProcessCount, count) - } - } - const summary = {} - for (const [kind, values] of byKind) { - const cpuSorted = [...values.cpuValues].sort((a, b) => a - b) - const rssSumBySample = samples.map((sample) => - sample.processes - .filter((proc) => proc.kind === kind) - .reduce((sum, proc) => sum + proc.rssBytes, 0) - ) - summary[kind] = { - meanCpuPercent: mean(values.cpuValues), - p95CpuPercent: percentile(cpuSorted, 0.95), - maxCpuPercent: Math.max(0, ...values.cpuValues), - meanRssBytes: mean(rssSumBySample), - maxProcessCount: values.maxProcessCount - } - } - summary.total = { - meanCpuPercent: mean(samples.map((sample) => sample.totalCpuPercent)), - p95CpuPercent: percentile( - samples.map((sample) => sample.totalCpuPercent).sort((a, b) => a - b), - 0.95 - ), - meanRssBytes: mean(samples.map((sample) => sample.totalRssBytes)) - } - return summary -} - -function summarizeProcessInventory(samples) { - const inventory = {} - for (const sample of samples) { - const counts = new Map() - for (const proc of sample.processes) { - counts.set(proc.kind, (counts.get(proc.kind) ?? 0) + 1) - const entry = inventory[proc.kind] ?? { - maxProcessCount: 0, - maxCpuPercent: 0, - commandSamples: [] - } - entry.maxCpuPercent = Math.max(entry.maxCpuPercent, proc.cpu) - if (!entry.commandSamples.includes(proc.command) && entry.commandSamples.length < 6) { - entry.commandSamples.push(proc.command) - } - inventory[proc.kind] = entry - } - for (const [kind, count] of counts) { - inventory[kind].maxProcessCount = Math.max(inventory[kind].maxProcessCount, count) - } - } - return inventory -} -function mean(values) { - return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length -} - -function percentile(sorted, fraction) { - if (sorted.length === 0) { - return 0 - } - const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1) - return sorted[index] -} - -function terminateProcesses(processes) { - for (const proc of processes) { - try { - process.kill(proc.pid) - } catch {} - } -} - async function main() { const options = parseArgs(process.argv.slice(2)) const root = path.resolve(import.meta.dirname, '..', '..') @@ -491,6 +322,11 @@ async function main() { const page = await app.firstWindow({ timeout: 120_000 }) await page.waitForLoadState('domcontentloaded') await page.waitForFunction(() => Boolean(window.__store), null, { timeout: 30_000 }) + await page.waitForFunction( + () => window.__store?.getState().workspaceSessionReady === true, + null, + { timeout: 60_000 } + ) const measurementCss = [] if (options.disableRendererAnimations) { measurementCss.push( @@ -506,63 +342,111 @@ async function main() { options.syntheticSpinnerAnimation, options.syntheticSpinnerSteps ) - await page.evaluate(async (repoPath) => { - await window.api.repos.add({ path: repoPath }) + const fixtureState = await page.evaluate(async (repoPath) => { + const added = await window.api.repos.add({ path: repoPath }) + if ('error' in added) { + return { error: added.error } + } const store = window.__store await store?.getState().fetchRepos() - const repo = store?.getState().repos.find((candidate) => candidate.path === repoPath) + const repo = store?.getState().repos.find((candidate) => candidate.id === added.repo.id) if (repo) { - await store.getState().updateRepo(repo.id, { externalWorktreeVisibility: 'show' }) - await store.getState().fetchWorktrees(repo.id) + const detected = await store + .getState() + .fetchWorktrees(repo.id, { requireAuthoritative: true }) + const importedWorktreePaths = ( + store.getState().detectedWorktreesByRepo[repo.id]?.worktrees ?? [] + ) + .filter((worktree) => !worktree.selectedCheckout) + .map((worktree) => worktree.path) + const updated = await store.getState().updateRepo(repo.id, { + externalWorktreeVisibility: 'show', + importedExternalWorktreePaths: importedWorktreePaths, + externalWorktreeInboxBaselinePaths: importedWorktreePaths + }) + const refreshed = await store + .getState() + .fetchWorktrees(repo.id, { requireAuthoritative: true }) + return { + detected, + updated, + refreshed, + detectedCount: store.getState().detectedWorktreesByRepo[repo.id]?.worktrees.length ?? 0, + importedCount: importedWorktreePaths.length, + visibleCount: store.getState().worktreesByRepo[repo.id]?.length ?? 0 + } } + return { error: 'repo-not-found' } }, repoDir) + console.log(`[idle-cpu] fixture ${JSON.stringify(fixtureState)}`) await page.waitForFunction( - () => window.__store?.getState().workspaceSessionReady === true, + (expectedWorktrees) => { + const state = window.__store?.getState() + return ( + state?.workspaceSessionReady === true && + Object.values(state.worktreesByRepo).flat().length === expectedWorktrees + ) + }, + options.worktrees, + { timeout: 180_000 } + ) + const scaleFixtureState = await configureRendererScaleFixture(page, options, repoDir) + if (scaleFixtureState.applied) { + console.log(`[idle-cpu] scale fixture ${JSON.stringify(scaleFixtureState)}`) + } + await page.waitForFunction( + (expectedAgentRows) => { + const state = window.__store?.getState() + return ( + state !== undefined && + Object.keys(state.agentStatusByPaneKey ?? {}).length >= expectedAgentRows + ) + }, + scaleFixtureState.seededAgentRows, + { timeout: 30_000 } + ) + await page.waitForFunction( + () => Boolean(document.querySelector('[data-worktree-sidebar] [data-worktree-id]')), null, - { timeout: 60_000 } + { timeout: 30_000 } ) console.log( - `[idle-cpu] root pid=${rootPid}; warmup=${options.warmupMs}ms sample=${options.sampleMs}ms interval=${options.intervalMs}ms worktrees=${options.worktrees}` + `[idle-cpu] root pid=${rootPid}; warmup=${options.warmupMs}ms sample=${options.sampleMs}ms interval=${options.intervalMs}ms worktrees=${options.worktrees} lineage-depth=${options.lineageDepth} agents/worktree=${options.agentsPerWorktree} publications=${options.zustandPublications}` ) + await startRendererTimingProbe(page) await sleep(options.warmupMs) const rendererIdleState = await collectRendererIdleState(page) - const deadline = Date.now() + options.sampleMs - const samples = [] - let previousSnapshot = null - while (Date.now() <= deadline || samples.length === 0) { - const sampledAt = Date.now() - const processRows = descendantsOf(readProcessRows(), rootPid) - const rawProcesses = processRows.map((row) => ({ ...row, kind: classify(row, rootPid) })) - if (previousSnapshot) { - const elapsedSeconds = Math.max(0.001, (sampledAt - previousSnapshot.at) / 1000) - const previousByPid = new Map(previousSnapshot.processes.map((proc) => [proc.pid, proc])) - const processes = rawProcesses.map((row) => { - const previous = previousByPid.get(row.pid) - const canComputeDelta = - typeof row.cpuTimeSeconds === 'number' && typeof previous?.cpuTimeSeconds === 'number' - const cpu = canComputeDelta - ? Math.max(0, ((row.cpuTimeSeconds - previous.cpuTimeSeconds) / elapsedSeconds) * 100) - : row.percentCpu - return { ...row, cpu } - }) - samples.push({ - at: sampledAt, - elapsedMs: sampledAt - previousSnapshot.at, - totalCpuPercent: processes.reduce((sum, proc) => sum + proc.cpu, 0), - totalRssBytes: processes.reduce((sum, proc) => sum + proc.rssBytes, 0), - processes - }) - } - previousSnapshot = { at: sampledAt, processes: rawProcesses } - await sleep(options.intervalMs) - } + const rendererCensusBefore = await collectRendererCensus(page, options.lineageDepth) + const rendererTimingBefore = await snapshotRendererTimingProbe(page) + const publicationPromise = runZustandPublications( + page, + options.zustandPublications, + options.zustandPublicationIntervalMs + ) + const sampled = await sampleProcessTreeUntilWorkloadsComplete({ + rootPid, + requestedDurationMs: options.sampleMs, + intervalMs: options.intervalMs, + workloadPromise: publicationPromise + }) + const samples = sampled.samples + const zustandPublications = sampled.workloadResult + const rendererTimingAfter = await stopRendererTimingProbe(page) + const rendererCensusAfter = await collectRendererCensus(page, options.lineageDepth) const report = { benchmark: 'orca-idle-cpu', createdAt: new Date().toISOString(), options, rootPid, platform: { platform: process.platform, arch: process.arch, cpus: os.cpus().length }, + fixtureState, + scaleFixtureState, rendererIdleState, + rendererCensusBefore, + rendererCensusAfter, + rendererTiming: { before: rendererTimingBefore, after: rendererTimingAfter }, + zustandPublications, + samplingWindow: sampled.samplingWindow, sampleCount: samples.length, summary: summarizeSamples(samples), processInventory: summarizeProcessInventory(samples), @@ -578,7 +462,13 @@ async function main() { { summary: report.summary, processInventory: report.processInventory, - sampleCount: report.sampleCount + sampleCount: report.sampleCount, + scaleFixtureState: report.scaleFixtureState, + rendererCensusBefore: report.rendererCensusBefore, + rendererCensusAfter: report.rendererCensusAfter, + rendererTiming: report.rendererTiming, + zustandPublications: report.zustandPublications, + samplingWindow: report.samplingWindow }, null, 2 diff --git a/docs/reference/renderer-agent-status-performance.md b/docs/reference/renderer-agent-status-performance.md new file mode 100644 index 00000000000..8ed43d868ce --- /dev/null +++ b/docs/reference/renderer-agent-status-performance.md @@ -0,0 +1,328 @@ +# Renderer agent-status performance + +## Status + +This design is adopted for the renderer's high-frequency agent-status path. It +keeps the existing event semantics while bounding the amount of synchronous +store fanout performed for one IPC burst. + +It lands in slices. This document and the `bench:idle-cpu` harness land first so +the store changes can be reviewed against a baseline someone else measured. Until +the store slice lands, the `setAgentStatuses` / `transactAgentStatuses` actions +and the agent-status write workload described below are not yet on `main`; the +harness measures scale, listener census, and raw publication fanout only. + +## Context + +Orca can display a large expanded worktree lineage inside one virtualized list +row. Virtualizing the root row does not virtualize its descendants, so a +100-worktree lineage can mount 100 `WorktreeCard` instances at once. + +Agent-status IPC events are bursty. The renderer already groups live events into +a 33 ms window, but the original flush applied every queued event with a +separate Zustand write. Zustand synchronously visits every listener for every +publication. The resulting work therefore grew with both the number of status +events and the number of mounted subscriptions: + +```text +burst work ~= status events x store listeners x selector work +``` + +A production trace captured the renderer repeatedly entering +`flushLiveAgentStatusBurst -> applyAgentStatus -> setAgentStatus -> setState` +through `Set.forEach`. A deterministic 100-worktree fixture reproduces the +structural multiplier; see "Baseline on `main`" below for the currently measured +listener count and publication cost. + +The production app later recovered substantially when all configured remote +hosts were removed. Host removal can stop relay/reconnect traffic, remove +mounted remote worktrees, or both, depending on host type and removal options. +That observation identifies remote presence as the production trigger but does +not by itself distinguish traffic volume from mounted-listener fanout. + +A read-only reconnect audit ruled out systematic double status emission from a +full PTY replay: replay bytes bypass OSC status parsing. Reconnect still causes +a full terminal-buffer repaint for every attached remote pane, which is a +separate source of renderer work and remains a follow-up investigation. + +## Goals + +- Keep a large expanded lineage responsive during dense agent-status traffic. +- Preserve every ordered status transition, including repeated updates for one + pane inside the same burst. +- Publish agent-status state once for a deferred live burst. +- Preserve selector identity and child render isolation. +- Make the regression reproducible without relying on a user's production data. + +## Non-goals + +- Changing the client/server status payload or remote protocol. +- Deduplicating status events by pane. +- Changing agent freshness, retention, history, title, completion, or provider + session behavior. +- Changing remote reconnect, PTY replay, or terminal repaint behavior. +- Redesigning lineage presentation or collapsing worktrees automatically. + +## Design + +### Bound mounted subscription fanout + +Sidebar components select cohesive state bundles with shallow equality instead +of registering one listener per field. Derived arrays and maps retain their +existing shallow identity behavior so unrelated store writes do not rerender a +card. Full agent-list mode keeps its child-level subscription boundary; compact +mode passes the already selected rows to avoid selecting the same inputs twice. + +The deterministic 100-worktree fixture pins the resulting listener budget: + +| Surface | Listener budget | +| ------------------------------ | --------------: | +| Worktree card state and caches | 2 | +| Agent-row inputs | 1 | +| Worktree activity status | 1 | +| Closed context menu | 1 | + +Unmount tests require the listener count to return to its prior baseline. In the +bundled prototype, the fixture without seeded agents fell from 8,518 listeners +to 1,218; with 100 visible agent rows the candidate mounted 1,618. Compare +against the census in "Baseline on `main`", which the harness reports directly. + +### Share working-spinner phase without per-element animation queries + +Working rows keep the existing compositor-driven CSS animation and shared +visual phase. Each mount derives one negative animation delay from the document +timeline instead of querying `getAnimations()` and mutating the animation start +time. This removes per-row Web Animations setup from dense status transitions +without adding a JavaScript animation clock. + +### Fold a burst in event order + +The store exposes the single-update action and two batch forms: + +- `setAgentStatus(paneKey, payload, ...)` retains the positional single-update + API for the immediate live path. +- `setAgentStatuses(updates)` applies a prebuilt ordered list, while + `transactAgentStatuses(operation)` lets IPC derive each update against the + exact staged state before the single commit. + +Both entry points reuse the same single-update state transition. The batch +reducer passes each resulting state into the next update, so a sequence such as +`working -> waiting -> done` retains the same history and timestamps as three +sequential calls. Updates are never keyed or deduplicated before the fold. + +The live IPC queue is spliced before it is processed. This preserves the +existing reentrancy guarantee: a synchronous subscriber can enqueue another +event without causing the current queue to be drained recursively. The first +event outside an active burst remains immediate; events accumulated within the +33 ms window are applied as one ordered transaction. Startup snapshots and +bounded pending-hydration retries use the same transaction path instead of +publishing once per restored pane. + +Each transaction builds pane-routing ownership once with the same first-match +semantics as the standalone resolver. Split-layout leaf membership is indexed +once per layout root, so a large snapshot performs linear tab and leaf work +instead of rescanning every mounted worktree for every pane. + +### Run effects after the transaction + +Generated-title work that requires committed state is deferred until after the +transaction. Accepted updates also request freshness scheduling; the outer batch +coalesces those requests and schedules the shared freshness timer once after its +single commit. Generated-title requests are folded in event order and published +together, including first-write and forced-replacement semantics. Resolved tab +titles are projected while the transaction folds, then final title changes are +published together. Completion-triggered review refreshes remain deferred +microtasks. + +Bulk title application preserves event order and duplicate-tab behavior while +indexing owners once, cloning each changed owner array once, and replacing each +top-level map once. This keeps the post-commit title phase linear in mounted +tabs plus changed titles. + +This separation is important: invoking store actions from inside a Zustand +updater would re-enter the store, while running an effect before the commit +would let it observe stale state. + +## Semantic invariants + +Sequential and batched application must agree on: + +- live and retained agent maps; +- state history, `updatedAt`, and `stateStartedAt`; +- agent identity, model, prompt, tools, assistant messages, and subagents; +- orchestration and provider-session continuity; +- sleeping-session and launch-config recovery records; +- retired/closed-pane rejection and inherited-status suppression; +- retention cleanup and live-map eviction; +- `agentStatusEpoch` and `sortEpoch`; +- automation completion observation across intermediate transitions; +- generated-title inputs, freshness scheduling, and completion refreshes. + +Equivalence tests use fixed timestamps and include repeated same-pane +transitions. A publication-count test subscribes to the real store and requires +one notification for a non-empty batch and none for an empty batch. + +## Benchmark contract + +The benchmark launches an E2E-mode Electron build with the store exposed only +for instrumentation. It creates 100 worktrees in one expanded lineage, verifies +100 mounted cards, captures the store listener census, and then applies seeded +ordered agent-status traffic through the real store action. + +The benchmark measures the synchronous store action, not the live IPC leading +edge or post-commit notification path. A real-store snapshot test covers the +end-to-end budget for 100 panes with auto-generated titles enabled: one status, +one bulk generated-title, and one bulk resolved-title publication. Disabling +generated titles removes that middle publication, independent of pane count. + +The artifact records only fixed diagnostic fields needed for comparison: + +- requested and completed batches and updates; +- store action calls and observed publications; +- elapsed time, throughput, and scheduling drift; +- final-state verification; +- renderer mean, p95, and maximum CPU; +- renderer timer drift and long tasks; +- mounted-card and listener counts. + +Raw process inventories, temporary paths, pane identifiers, and DOM text are +diagnostic-only and must not be embedded in the shareable report. + +Run baseline and candidate on the same machine and OS with the same Electron +build mode. CPU samples from macOS and Linux are comparable within that +constraint; Windows process CPU collection currently cannot support this +comparison. + +## Harness + +`pnpm run bench:idle-cpu` drives `config/scripts/run-idle-cpu-benchmark.mjs`, +which composes four modules: + +| Module | Responsibility | +| ------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `idle-cpu-renderer-scale-fixture.mjs` | Seeds the lineage, agent rows, and sidebar view state; takes the mounted-card and listener census | +| `idle-cpu-renderer-timing-probe.mjs` | In-page timer drift and long-task probe; runs the no-op publication workload | +| `idle-cpu-process-sampling.mjs` | Classifies the Electron process tree and samples per-role CPU/RSS | +| `idle-cpu-synthetic-spinners.mjs` | Measurement-only visible spinners | + +The sampling window extends past `--sample-ms` until the workload settles, and +fails the run rather than reporting a truncated window if the workload overruns +the guard. That is why a 2,000-publication run reports a measured window longer +than the requested one. + +`--zustand-publications` publishes an empty partial through the real store, so +each publication costs exactly one full subscriber visit and nothing else. It +isolates the `listeners x selector work` half of the burst-cost model from +agent-status payload work, and it is store-API independent — it measures the +same thing before and after the batching slice. + +The agent-status write workload (`--agent-status-batches`, +`--agent-status-write-mode`) is not in this harness yet. It depends on +`setAgentStatuses`, so it lands with the store slice. + +## Baseline on `main` + +Measured on `main` at `077f5a11cd4` (macOS, arm64, 16 CPUs), Electron built with +`electron-vite --mode e2e`, headless, 100 worktrees at lineage depth 99 with 100 +seeded agent rows, 10 s warmup and a 30 s sampling window. + +Fixture scale is confirmed by the census rather than assumed: 100 store +worktrees, 100 mounted cards, 100 mounted agent rows, and **9,279 store +listeners**. That listener count is the multiplier the design targets. + +2,000 no-op store publications at a 1 ms cadence, three repetitions. The +listener census was 9,279 in every run. + +| Measure | Median | Runs | +| ------------------------ | ----------: | ------------------------------ | +| Wall time to complete | 12,325.7 ms | 13,148.6 / 12,325.7 / 11,870.5 | +| p50 scheduling drift | 5,150.3 ms | 5,432.3 / 5,150.3 / 4,945.2 | +| p95 scheduling drift | 9,802.9 ms | 10,570.4 / 9,802.9 / 9,390.0 | +| Renderer mean CPU | 18.25% | 20.25 / 18.25 / 15.88 | +| Renderer p95 CPU | 32.59% | 40.07 / 32.59 / 31.28 | +| Renderer timer drift p95 | 7.0 ms | 7.3 / 5.3 / 7.0 | + +2,000 publications requested over 2 s take about 12 s, so the renderer sustains +roughly 160 publications per second at this scale. Each publication is +individually short - the long-task observer recorded zero entries in all three +runs - so the cost surfaces as scheduling drift and sustained CPU rather than as +discrete long tasks. Compare drift and CPU here, not long-task counts. + +The idle control at the same scale with `--zustand-publications 0` reports 6.63% +renderer mean CPU, 17.11% p95, and 1.6 ms p95 timer drift. Roughly 11.6 points +of mean renderer CPU are therefore attributable to publication fanout rather than +to the mounted fixture itself. 200 spinner animations run in both cases, so the +control also bounds the animation cost out of the comparison. + +Reproduce with: + +```bash +pnpm run bench:idle-cpu -- --worktrees 100 --lineage-depth 99 \ + --agents-per-worktree 1 --warmup-ms 10000 --sample-ms 30000 \ + --zustand-publications 2000 --zustand-publication-interval-ms 1 \ + --output /tmp/idle-cpu-baseline.json +``` + +## Results + +Three repetitions used 100 mounted worktrees, lineage depth 99, 100 seeded +agent rows, and verified final state. Medians from the regenerated evidence set +are: + +| Single 2,000-update burst | Sequential | Batched | +| ------------------------- | ---------: | ---------: | +| Status-state publications | 2,000 | 1 | +| Store action time | 3,692.0 ms | 188.7 ms | +| Update throughput | 541.7/s | 10,598.8/s | +| Renderer mean CPU | 36.2% | 2.9% | +| Renderer p95 CPU | 107.3% | 8.2% | +| p95 long task | 4,653 ms | 216 ms | + +The direct store transaction performs 99.95% fewer status-state publications, +spends 94.9% less time in the store action, and processes updates 19.6 times +faster. Renderer mean CPU falls 92.0%, renderer p95 CPU falls 92.4%, and the p95 +long task falls 95.4%. + +The 60-burst × 32-update case at 33 ms is a sustained saturation stress, not a +real-time production SLO. Publications fall from 1,920 to 60 and median store +action time falls from 2,791.9 ms to 323.3 ms. Median completion time falls from +5,298.2 ms to 2,710.6 ms, p95 scheduling drift falls from 3,073.0 ms to 664.9 +ms, and long-task count falls from 57 to 1. Renderer p95 CPU remains saturated +and noisy in this cadence, so it is not used as the discriminating measure. + +The 20-pane artificial OpenCode regression passes with 12.4 ms median key echo, +25.2 ms worst key echo, 19.4 ms maximum timer drift, and zero dropped renderer +backlogs. + +These figures come from the bundled prototype and are restated here as the +target. They are re-measured with the harness when the store slice lands. + +## Acceptance criteria + +- The 100-worktree fixture stays at or below the pinned listener budgets. +- A deferred transaction performs one status-state publication while preserving + ordered final state, including live-map eviction at the 500-row cap. +- A 100-pane startup snapshot performs one status and one bulk resolved-title + publication with generated titles disabled; enabling generated titles adds at + most one ordered bulk publication while preserving final statuses and titles. +- Sequential-versus-batch equivalence tests pass across same-pane transitions + and side-effect-bearing updates. +- Renderer CPU tails and scheduling drift improve in repeated candidate runs. +- The 20-pane artificial terminal test reports no dropped output backlog and no + material typing-latency regression. +- Web typecheck, focused unit tests, lint, max-lines ratchet, and E2E build pass. + +## Compatibility + +This is renderer-local. It adds no RPC field, stream opcode, persisted data, Git +command, or provider-specific contract. Native, WSL, SSH, relay, folder +workspace, and git-worktree status events enter the same renderer action. Mixed +client/server versions therefore need no capability negotiation. + +## Failure containment + +The first live event remains immediate. Startup replay and bounded pending +retries fold synchronously without waiting for the 33 ms live-burst window, but +publish their accepted updates together. Empty batches are no-ops. If an update +is stale or targets retired authority, the reducer skips only that update and +continues folding later events in order.