mirror of
https://github.com/stablyai/orca.git
synced 2026-10-01 00:02:10 +00:00
* test: reproduce accumulated-workspace typing latency through real PTYs * test: make the bench harness self-checks falsifiable Review found four assertions that could not fail and one fixture gap: - `missingPtyArrivalCount`/`missingEchoCount` were hardcoded `0` and `validateExpectedSeqs` throws before them, so every assertion on them was vacuous and every report read `0`. The throw is the real guard and is already covered; drop the vestigial fields. - An absent status controller returned an all-zero result, which satisfied its own accepted-equals-generated equality. Assert presence first. - The byte-pacing control had only an upper bound, so a generator emitting no stream bytes passed. Add the lower bound. - `lineageEvery: 1` built zero lineage: no ordinal satisfies `% 1 === 1`. Offset the interval and cover the densest setting. - The documented control command never set ORCA_TYPING_BENCH, so it skipped instead of running.
255 lines
9.0 KiB
TypeScript
255 lines
9.0 KiB
TypeScript
import path from 'node:path'
|
|
import type { SleepingAgentSessionRecord } from '../../src/shared/agent-session-resume'
|
|
import type { Repo } from '../../src/shared/repo-types'
|
|
import type {
|
|
TerminalLayoutSnapshot,
|
|
TerminalPaneLayoutNode,
|
|
TerminalTab
|
|
} from '../../src/shared/terminal-tab-types'
|
|
import type { Tab, TabGroup, TabGroupLayoutNode } from '../../src/shared/tab-types'
|
|
import type { WorktreeLineage } from '../../src/shared/worktree/lineage-types'
|
|
import type { Worktree } from '../../src/shared/worktree/types'
|
|
import type { NormalizedAccumulatedWorkspaceFixtureOptions } from './accumulated-workspace-profile'
|
|
|
|
export type SyntheticLiveStatus = {
|
|
paneKey: string
|
|
tabId: string
|
|
worktreeId: string
|
|
prompt: string
|
|
}
|
|
|
|
export type AccumulatedWorkspaceSeed = {
|
|
repos: Repo[]
|
|
worktreesByRepo: Record<string, Worktree[]>
|
|
tabsByWorktree: Record<string, TerminalTab[]>
|
|
ptyIdsByTabId: Record<string, string[]>
|
|
terminalLayoutsByTabId: Record<string, TerminalLayoutSnapshot>
|
|
unifiedTabsByWorktree: Record<string, Tab[]>
|
|
groupsByWorktree: Record<string, TabGroup[]>
|
|
activeGroupIdByWorktree: Record<string, string>
|
|
layoutByWorktree: Record<string, TabGroupLayoutNode>
|
|
sleepingAgentSessionsByPaneKey: Record<string, SleepingAgentSessionRecord>
|
|
worktreeLineageById: Record<string, WorktreeLineage>
|
|
liveStatuses: SyntheticLiveStatus[]
|
|
}
|
|
|
|
function bucketSize(total: number, buckets: number, index: number): number {
|
|
return Math.floor(total / buckets) + (index < total % buckets ? 1 : 0)
|
|
}
|
|
|
|
function paneId(ordinal: number): string {
|
|
return `00000000-0000-4000-8000-${String(ordinal).padStart(12, '0')}`
|
|
}
|
|
|
|
function layoutRoot(panes: readonly string[]): TerminalPaneLayoutNode {
|
|
return panes.slice(1).reduce<TerminalPaneLayoutNode>(
|
|
(second, leafId) => ({
|
|
type: 'split',
|
|
direction: 'vertical',
|
|
first: { type: 'leaf', leafId },
|
|
second
|
|
}),
|
|
{ type: 'leaf', leafId: panes[0] }
|
|
)
|
|
}
|
|
|
|
export function buildAccumulatedWorkspaceSeed(
|
|
config: NormalizedAccumulatedWorkspaceFixtureOptions,
|
|
now = Date.now()
|
|
): AccumulatedWorkspaceSeed {
|
|
const syntheticRoot = path.join(path.parse(process.cwd()).root, 'orca-typing-benchmark')
|
|
const repos: Repo[] = Array.from({ length: config.repositories }, (_, index) => ({
|
|
id: `synthetic-repo-${index}`,
|
|
path: path.join(syntheticRoot, `repo-${index}`),
|
|
displayName: `Synthetic repo ${index}`,
|
|
badgeColor: '#737373',
|
|
addedAt: now - index,
|
|
kind: 'git'
|
|
}))
|
|
const worktreesByRepo: Record<string, Worktree[]> = Object.fromEntries(
|
|
repos.map((repo) => [repo.id, []])
|
|
)
|
|
const rows: Worktree[] = []
|
|
const parentIdByWorktreeId = new Map<string, string>()
|
|
for (let index = 0; index < config.worktrees; index += 1) {
|
|
const repoIndex = index % config.repositories
|
|
const repo = repos[repoIndex]
|
|
const repoOrdinal = Math.floor(index / config.repositories)
|
|
const id = `synthetic-worktree-${index}`
|
|
// Ordinals 1, 1 + lineageEvery, ...; `% lineageEvery === 1` skipped everything at an interval of 1.
|
|
if (repoOrdinal >= 1 && (repoOrdinal - 1) % config.lineageEvery === 0) {
|
|
parentIdByWorktreeId.set(id, `synthetic-worktree-${index - config.repositories}`)
|
|
}
|
|
const row: Worktree = {
|
|
id,
|
|
instanceId: `synthetic-instance-${index}`,
|
|
repoId: repo.id,
|
|
path: path.join(repo.path, `worktree-${repoOrdinal}`),
|
|
displayName: `Synthetic workspace ${index}`,
|
|
comment: `Accumulated benchmark workspace ${index}`,
|
|
linkedIssue: null,
|
|
linkedPR: null,
|
|
linkedLinearIssue: null,
|
|
isArchived: false,
|
|
isUnread: index % 5 === 0,
|
|
isPinned: index % 11 === 0,
|
|
sortOrder: repoOrdinal,
|
|
lastActivityAt: now - index * 1_000,
|
|
head: '0000000000000000000000000000000000000000',
|
|
branch: `synthetic-${index}`,
|
|
isBare: false,
|
|
isMainWorktree: repoOrdinal === 0
|
|
}
|
|
rows.push(row)
|
|
worktreesByRepo[repo.id].push(row)
|
|
}
|
|
|
|
const tabsByWorktree: Record<string, TerminalTab[]> = {}
|
|
const ptyIdsByTabId: Record<string, string[]> = {}
|
|
const terminalLayoutsByTabId: Record<string, TerminalLayoutSnapshot> = {}
|
|
const unifiedTabsByWorktree: Record<string, Tab[]> = {}
|
|
const groupsByWorktree: Record<string, TabGroup[]> = {}
|
|
const activeGroupIdByWorktree: Record<string, string> = {}
|
|
const layoutByWorktree: Record<string, TabGroupLayoutNode> = {}
|
|
const sleepingAgentSessionsByPaneKey: Record<string, SleepingAgentSessionRecord> = {}
|
|
const worktreeLineageById: Record<string, WorktreeLineage> = {}
|
|
const liveStatuses: SyntheticLiveStatus[] = []
|
|
let terminalOrdinal = 0
|
|
let extraUnifiedOrdinal = 0
|
|
let paneOrdinal = 0
|
|
|
|
for (const [worktreeIndex, row] of rows.entries()) {
|
|
const groupId = `synthetic-group-${worktreeIndex}`
|
|
const terminalCount = bucketSize(config.terminalTabs, config.worktrees, worktreeIndex)
|
|
const extraUnifiedCount = bucketSize(
|
|
config.unifiedTabs - config.terminalTabs,
|
|
config.worktrees,
|
|
worktreeIndex
|
|
)
|
|
const terminalTabs: TerminalTab[] = []
|
|
const unifiedTabs: Tab[] = []
|
|
for (let tabIndex = 0; tabIndex < terminalCount; tabIndex += 1) {
|
|
const tabId = `synthetic-tab-${terminalOrdinal}`
|
|
const panes = Array.from({ length: config.panesPerTab }, () => paneId(++paneOrdinal))
|
|
terminalTabs.push({
|
|
id: tabId,
|
|
ptyId: null,
|
|
worktreeId: row.id,
|
|
title: `Terminal ${tabIndex + 1}`,
|
|
defaultTitle: `Terminal ${tabIndex + 1}`,
|
|
customTitle: null,
|
|
color: null,
|
|
sortOrder: tabIndex,
|
|
createdAt: now - terminalOrdinal
|
|
})
|
|
unifiedTabs.push({
|
|
id: tabId,
|
|
entityId: tabId,
|
|
groupId,
|
|
worktreeId: row.id,
|
|
contentType: 'terminal',
|
|
label: `Terminal ${tabIndex + 1}`,
|
|
customLabel: null,
|
|
color: null,
|
|
sortOrder: tabIndex,
|
|
createdAt: now - terminalOrdinal
|
|
})
|
|
ptyIdsByTabId[tabId] = []
|
|
terminalLayoutsByTabId[tabId] = {
|
|
root: layoutRoot(panes),
|
|
activeLeafId: panes[0],
|
|
expandedLeafId: null,
|
|
titlesByLeafId: Object.fromEntries(
|
|
panes.map((leafId, index) => [leafId, `Pane ${index + 1}`])
|
|
)
|
|
}
|
|
for (const [tabPaneIndex, leafId] of panes.entries()) {
|
|
const paneKey = `${tabId}:${leafId}`
|
|
const zeroBasedPaneOrdinal = paneOrdinal - panes.length + tabPaneIndex
|
|
if (zeroBasedPaneOrdinal < config.liveStatuses) {
|
|
liveStatuses.push({
|
|
paneKey,
|
|
tabId,
|
|
worktreeId: row.id,
|
|
prompt: `Synthetic live task ${worktreeIndex}`
|
|
})
|
|
} else if (zeroBasedPaneOrdinal < config.liveStatuses + config.sleepingRecords) {
|
|
sleepingAgentSessionsByPaneKey[paneKey] = {
|
|
paneKey,
|
|
tabId,
|
|
worktreeId: row.id,
|
|
agent: 'codex',
|
|
providerSession: {
|
|
key: 'session_id',
|
|
id: `synthetic-session-${zeroBasedPaneOrdinal + 1}`
|
|
},
|
|
prompt: `Synthetic parked task ${worktreeIndex}`,
|
|
state: 'working',
|
|
capturedAt: now,
|
|
updatedAt: now,
|
|
terminalTitle: `Parked ${worktreeIndex}`,
|
|
origin: 'worktree-sleep',
|
|
restoreOnTabOpenOnly: true
|
|
}
|
|
}
|
|
}
|
|
terminalOrdinal += 1
|
|
}
|
|
for (let tabIndex = 0; tabIndex < extraUnifiedCount; tabIndex += 1) {
|
|
const entityId = path.join(row.path, `synthetic-file-${extraUnifiedOrdinal}.ts`)
|
|
unifiedTabs.push({
|
|
id: entityId,
|
|
entityId,
|
|
groupId,
|
|
worktreeId: row.id,
|
|
contentType: 'editor',
|
|
label: `synthetic-file-${extraUnifiedOrdinal}.ts`,
|
|
customLabel: null,
|
|
color: null,
|
|
sortOrder: terminalCount + tabIndex,
|
|
createdAt: now - config.terminalTabs - extraUnifiedOrdinal
|
|
})
|
|
extraUnifiedOrdinal += 1
|
|
}
|
|
const tabOrder = unifiedTabs.map(({ id }) => id)
|
|
tabsByWorktree[row.id] = terminalTabs
|
|
unifiedTabsByWorktree[row.id] = unifiedTabs
|
|
groupsByWorktree[row.id] = [
|
|
{ id: groupId, worktreeId: row.id, activeTabId: tabOrder[0] ?? null, tabOrder }
|
|
]
|
|
activeGroupIdByWorktree[row.id] = groupId
|
|
layoutByWorktree[row.id] = { type: 'leaf', groupId }
|
|
}
|
|
|
|
for (const [rowIndex, row] of rows.entries()) {
|
|
const parentWorktreeId = parentIdByWorktreeId.get(row.id)
|
|
if (!parentWorktreeId) {
|
|
continue
|
|
}
|
|
worktreeLineageById[row.id] = {
|
|
worktreeId: row.id,
|
|
worktreeInstanceId: `synthetic-instance-${rowIndex}`,
|
|
parentWorktreeId,
|
|
parentWorktreeInstanceId: `synthetic-instance-${parentWorktreeId.split('-').at(-1)}`,
|
|
origin: 'manual',
|
|
capture: { source: 'manual-action', confidence: 'explicit' },
|
|
createdAt: now - rowIndex * 1_000
|
|
}
|
|
}
|
|
|
|
return {
|
|
repos,
|
|
worktreesByRepo,
|
|
tabsByWorktree,
|
|
ptyIdsByTabId,
|
|
terminalLayoutsByTabId,
|
|
unifiedTabsByWorktree,
|
|
groupsByWorktree,
|
|
activeGroupIdByWorktree,
|
|
layoutByWorktree,
|
|
sleepingAgentSessionsByPaneKey,
|
|
worktreeLineageById,
|
|
liveStatuses
|
|
}
|
|
}
|