mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
test: remove obsolete manual tools and duplicate launch coverage
This commit is contained in:
@@ -1,162 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from 'node:assert/strict'
|
||||
import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { parseArgs } from 'node:util'
|
||||
import { runProcessSync } from '@orca/process-host'
|
||||
import {
|
||||
loadClaudeUsageBenchmarkScanner,
|
||||
readClaudeUsageBenchmarkBaselineSources
|
||||
} from './claude-usage-benchmark-scanner.mjs'
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../../../..', import.meta.url))
|
||||
const STORE_PATH = 'src/main/claude-usage/store.ts'
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
'baseline-ref': { type: 'string', default: '51e7181850b022bae2d41091fe1229accef4bd50' },
|
||||
'usage-keys': { type: 'string', default: '16384' },
|
||||
output: { type: 'string' }
|
||||
}
|
||||
})
|
||||
const usageKeys = Number(values['usage-keys'])
|
||||
assert(
|
||||
Number.isSafeInteger(usageKeys) && usageKeys > 7,
|
||||
'usage-keys must be an integer above seven'
|
||||
)
|
||||
process.env.ORCA_CLAUDE_USAGE_APPEND_BENCH_BASELINE = values['baseline-ref']
|
||||
|
||||
function serializationIndent(source) {
|
||||
const indent = /jsonIndent:\s*(\d+)/.exec(source)
|
||||
return indent ? Number(indent[1]) : undefined
|
||||
}
|
||||
|
||||
function sourceHash(source) {
|
||||
return createHash('sha256').update(source).digest('hex')
|
||||
}
|
||||
|
||||
function persistedState(result) {
|
||||
return {
|
||||
schemaVersion: 6,
|
||||
worktreeFingerprint: '[]',
|
||||
processedFiles: result.processedFiles,
|
||||
sessions: result.sessions,
|
||||
dailyAggregates: result.dailyAggregates,
|
||||
scanState: {
|
||||
enabled: true,
|
||||
lastScanStartedAt: 1,
|
||||
lastScanCompletedAt: 2,
|
||||
lastScanError: null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function usageRecord(cwd, index, inputTokens = 100) {
|
||||
return `${JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: 'active-session',
|
||||
timestamp: new Date(Date.UTC(2026, 9, 9, 12, 0, index)).toISOString(),
|
||||
requestId: `request-${index}`,
|
||||
cwd,
|
||||
gitBranch: 'benchmark',
|
||||
message: {
|
||||
id: `message-${index}`,
|
||||
model: 'claude-sonnet-4-6',
|
||||
usage: {
|
||||
input_tokens: inputTokens,
|
||||
output_tokens: 10,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 20,
|
||||
cache_creation: { ephemeral_1h_input_tokens: 5 }
|
||||
}
|
||||
}
|
||||
})}\n`
|
||||
}
|
||||
|
||||
const home = await mkdtemp(join(tmpdir(), 'orca-claude-cache-representation-'))
|
||||
try {
|
||||
const project = join(home, '.claude', 'projects', 'benchmark')
|
||||
const cwd = join(home, 'repo')
|
||||
await mkdir(project, { recursive: true })
|
||||
await mkdir(cwd)
|
||||
const transcriptPath = join(project, 'active-session.jsonl')
|
||||
const history = Array.from({ length: usageKeys }, (_, index) => usageRecord(cwd, index)).join('')
|
||||
const baseline = await readClaudeUsageBenchmarkBaselineSources()
|
||||
const storeRead = runProcessSync({
|
||||
program: 'git',
|
||||
args: ['show', `${baseline.baselineCommit}:${STORE_PATH}`],
|
||||
cwd: ROOT,
|
||||
timeoutMs: 10_000,
|
||||
maxOutputBytes: 1024 * 1024,
|
||||
env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
|
||||
})
|
||||
assert.equal(storeRead.code, 0, storeRead.stderr)
|
||||
const currentStore = await readFile(join(ROOT, ...STORE_PATH.split('/')), 'utf8')
|
||||
const stores = { baseline: storeRead.stdout, current: currentStore }
|
||||
const loaded = await Promise.all([
|
||||
loadClaudeUsageBenchmarkScanner(home, baseline.sources, new Map(), baseline.baselineCommit),
|
||||
loadClaudeUsageBenchmarkScanner(home)
|
||||
])
|
||||
const arms = {}
|
||||
const results = []
|
||||
for (const [index, arm] of ['baseline', 'current'].entries()) {
|
||||
const { scanner, bundleSha256, sourceFingerprints } = loaded[index]
|
||||
await writeFile(transcriptPath, history)
|
||||
const cold = await scanner.scanClaudeUsageFiles([], [], undefined, [])
|
||||
await appendFile(transcriptPath, usageRecord(cwd, 7, 2_000))
|
||||
const result = await scanner.scanClaudeUsageFiles(
|
||||
[],
|
||||
JSON.parse(JSON.stringify(cold.processedFiles)),
|
||||
undefined,
|
||||
[]
|
||||
)
|
||||
const fresh = await scanner.scanClaudeUsageFiles([], [], undefined, [])
|
||||
assert.deepEqual(result, fresh, `${arm} resume differs from a fresh scan`)
|
||||
assert.equal(result.processedFiles[0].ownedDedupeKeys.length, usageKeys)
|
||||
assert.equal(result.sessions[0].totalInputTokens, usageKeys * 100 + 1_900)
|
||||
const state = persistedState(result)
|
||||
const compact = JSON.stringify(state)
|
||||
const indented = JSON.stringify(state, null, 2)
|
||||
const indent = serializationIndent(stores[arm])
|
||||
assert.deepEqual(JSON.parse(compact), JSON.parse(indented))
|
||||
assert.deepEqual(structuredClone(state), state)
|
||||
arms[arm] = {
|
||||
compactFullStateBytes: Buffer.byteLength(compact),
|
||||
indentedFullStateBytes: Buffer.byteLength(indented),
|
||||
productionFullStateBytes: Buffer.byteLength(JSON.stringify(state, null, indent)),
|
||||
productionIndent: indent ?? null,
|
||||
storeSha256: sourceHash(stores[arm]),
|
||||
ownedKeys: result.processedFiles[0].ownedDedupeKeys.length,
|
||||
bundleSha256,
|
||||
sourceFingerprints
|
||||
}
|
||||
results.push(result)
|
||||
}
|
||||
assert.deepEqual(results[0].sessions, results[1].sessions)
|
||||
assert.deepEqual(results[0].dailyAggregates, results[1].dailyAggregates)
|
||||
const report = {
|
||||
purpose: 'Actual scanner cache representation byte counts; no timing or peak-memory claim.',
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
baselineRef: baseline.baselineRef,
|
||||
baselineCommit: baseline.baselineCommit,
|
||||
usageKeys,
|
||||
historyBytes: Buffer.byteLength(history),
|
||||
stateFields: 'schema v6; source cache, session/daily projections and fixed scan-state metadata',
|
||||
exactColdParity: true,
|
||||
crossArmProjectionParity: true,
|
||||
serializationParity: true,
|
||||
structuredCloneParity: true,
|
||||
toolingSha256: sourceHash(await readFile(import.meta.filename)),
|
||||
arms
|
||||
}
|
||||
if (values.output) {
|
||||
await writeFile(values.output, `${JSON.stringify(report, null, 2)}\n`)
|
||||
}
|
||||
console.log(JSON.stringify(report))
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true })
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
// Run pnpm exec tsx config/scripts/native-chat-live-session-benchmark.ts. Synthetic 2KB messages stress fallback-key normalization.
|
||||
// Excludes parsing, IPC/remote latency, React/DOM; results are not average production size or end-to-end frame latency.
|
||||
|
||||
import { deepStrictEqual, strictEqual } from 'node:assert'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import type {
|
||||
NativeChatMessage,
|
||||
NativeChatSession
|
||||
} from '../../../../src/shared/native-chat/native-chat-types'
|
||||
import { getVerifiedNativeChatCommands } from '../../../../src/shared/native-chat/native-chat-agent-profiles'
|
||||
import { surfaceSkillInvocationUserTurns } from '../../../../src/shared/native-chat/native-chat-command-envelope'
|
||||
import { prepareNativeChatLiveMessages } from '../../../../src/renderer/src/components/native-chat/session/terminal/assembly/native-chat-live-message-preparation'
|
||||
import { mergeNativeChatLiveSession } from '../../../../src/shared/native-chat/native-chat-live-status'
|
||||
import { assembleNativeChatSession } from '../../../../src/renderer/src/components/native-chat/session/terminal/assembly/native-chat-session-assembler'
|
||||
|
||||
type Operation = (index: number) => number
|
||||
type ExpectedChecksum = (iterations: number) => number
|
||||
type Calibration = { iterations: number; elapsedMs: number; capped: boolean }
|
||||
|
||||
const TARGET_SAMPLE_MS = 50
|
||||
const MAX_ITERATIONS = 16_777_216
|
||||
const ROUNDS = 10
|
||||
let checksum = 0
|
||||
let expectedChecksum = 0
|
||||
let validatedCases = 0
|
||||
let cappedCalibrations = 0
|
||||
const sessionSink: { value: NativeChatSession | null } = { value: null }
|
||||
let messageArraySink: NativeChatMessage[] | null = null
|
||||
let contentSink = ''
|
||||
const benchmarkStartedAt = performance.now()
|
||||
|
||||
function proseFixture(count: number, bytes: number, withTurnId: boolean): NativeChatMessage[] {
|
||||
const payload = 'Ab Cd '.repeat(Math.ceil(bytes / 7)).slice(0, bytes)
|
||||
return Array.from({ length: count }, (_, index) => ({
|
||||
id: `message-${index}`,
|
||||
role: index % 2 === 0 ? ('user' as const) : ('assistant' as const),
|
||||
blocks: [{ type: 'text' as const, text: `${payload}-${index}` }],
|
||||
timestamp: index,
|
||||
source: 'transcript' as const,
|
||||
...(withTurnId ? { turnId: `turn-${index}` } : {})
|
||||
}))
|
||||
}
|
||||
|
||||
function toolFixture(count: number): NativeChatMessage[] {
|
||||
return Array.from({ length: count }, (_, index) => ({
|
||||
id: `tool-${index}`,
|
||||
role: index % 2 === 0 ? ('assistant' as const) : ('tool' as const),
|
||||
blocks:
|
||||
index % 2 === 0
|
||||
? [
|
||||
{
|
||||
type: 'tool-call' as const,
|
||||
name: 'read',
|
||||
input: { path: `${index}.txt`, context: 'x'.repeat(512) }
|
||||
}
|
||||
]
|
||||
: [{ type: 'tool-result' as const, output: `result-${index}-${'x'.repeat(512)}` }],
|
||||
timestamp: index,
|
||||
source: 'transcript' as const
|
||||
}))
|
||||
}
|
||||
|
||||
function legacySession(messages: NativeChatMessage[]): NativeChatSession {
|
||||
return assembleNativeChatSession({
|
||||
sources: { transcript: messages },
|
||||
sessionId: 'benchmark',
|
||||
agent: 'claude'
|
||||
})
|
||||
}
|
||||
|
||||
function legacyMessageUpdateSession(messages: NativeChatMessage[]): NativeChatSession {
|
||||
const commandNames = new Set(
|
||||
getVerifiedNativeChatCommands('claude').map((command) => command.name)
|
||||
)
|
||||
return legacySession(surfaceSkillInvocationUserTurns(messages, commandNames))
|
||||
}
|
||||
|
||||
function directMessageUpdateSession(messages: NativeChatMessage[]): NativeChatSession {
|
||||
return mergeNativeChatLiveSession({
|
||||
messages: prepareNativeChatLiveMessages(messages, 'claude'),
|
||||
sessionId: 'benchmark',
|
||||
agent: 'claude',
|
||||
hookState: null
|
||||
})
|
||||
}
|
||||
|
||||
function blockContent(message: NativeChatMessage): string {
|
||||
const block = message.blocks[0]
|
||||
if (!block) {
|
||||
return ''
|
||||
}
|
||||
if (block.type === 'text') {
|
||||
return block.text
|
||||
}
|
||||
if (block.type === 'tool-call') {
|
||||
return block.name
|
||||
}
|
||||
if (block.type === 'tool-result') {
|
||||
return block.output
|
||||
}
|
||||
if (block.type === 'image-ref') {
|
||||
return block.path ?? block.url ?? block.alt ?? ''
|
||||
}
|
||||
return block.type === 'background-task' ? block.taskId : block.groupId
|
||||
}
|
||||
|
||||
function messageWeight(message: NativeChatMessage, content: string): number {
|
||||
const idTail = message.id.length > 0 ? message.id.charCodeAt(message.id.length - 1) : 0
|
||||
const contentTail = content.length > 0 ? content.charCodeAt(content.length - 1) : 0
|
||||
return message.id.length + idTail + message.role.charCodeAt(0) + content.length + contentTail
|
||||
}
|
||||
|
||||
function consumeSession(session: NativeChatSession, index: number): number {
|
||||
sessionSink.value = session
|
||||
messageArraySink = session.messages
|
||||
const message = session.messages[index % session.messages.length]
|
||||
if (!message) {
|
||||
contentSink = ''
|
||||
return session.status.charCodeAt(0)
|
||||
}
|
||||
contentSink = blockContent(message)
|
||||
return session.status.charCodeAt(0) + messageWeight(message, contentSink)
|
||||
}
|
||||
|
||||
function cyclicChecksum(values: readonly number[], iterations: number): number {
|
||||
if (values.length === 0) {
|
||||
return 0
|
||||
}
|
||||
const cycle = values.reduce((sum, value) => sum + value, 0)
|
||||
const fullCycles = Math.floor(iterations / values.length)
|
||||
let total = cycle * fullCycles
|
||||
for (let index = 0; index < iterations % values.length; index += 1) {
|
||||
total += values[index]!
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
function sessionExpectedChecksum(session: NativeChatSession): ExpectedChecksum {
|
||||
const statusWeight = session.status.charCodeAt(0)
|
||||
const weights = session.messages.map((message) => {
|
||||
const content = blockContent(message)
|
||||
return messageWeight(message, content)
|
||||
})
|
||||
return (iterations) => statusWeight * iterations + cyclicChecksum(weights, iterations)
|
||||
}
|
||||
|
||||
function runSample(operation: Operation, expected: ExpectedChecksum, iterations: number): number {
|
||||
let sampleChecksum = 0
|
||||
const startedAt = performance.now()
|
||||
for (let index = 0; index < iterations; index += 1) {
|
||||
sampleChecksum += operation(index)
|
||||
}
|
||||
const elapsedMs = performance.now() - startedAt
|
||||
checksum += sampleChecksum
|
||||
expectedChecksum += expected(iterations)
|
||||
return elapsedMs
|
||||
}
|
||||
|
||||
function median(samples: number[]): number {
|
||||
return [...samples].sort((left, right) => left - right)[Math.floor(samples.length / 2)]!
|
||||
}
|
||||
|
||||
function calibrate(operation: Operation, expected: ExpectedChecksum): Calibration {
|
||||
let iterations = 1
|
||||
let reachedTarget = false
|
||||
while (true) {
|
||||
const elapsedMs = runSample(operation, expected, iterations)
|
||||
if (elapsedMs >= TARGET_SAMPLE_MS) {
|
||||
if (reachedTarget) {
|
||||
return { iterations, elapsedMs, capped: false }
|
||||
}
|
||||
reachedTarget = true
|
||||
continue
|
||||
}
|
||||
if (iterations >= MAX_ITERATIONS) {
|
||||
return { iterations, elapsedMs, capped: true }
|
||||
}
|
||||
reachedTarget = false
|
||||
iterations = Math.min(iterations * 2, MAX_ITERATIONS)
|
||||
}
|
||||
}
|
||||
|
||||
function benchmark(
|
||||
name: string,
|
||||
baseline: Operation,
|
||||
optimized: Operation,
|
||||
expected: ExpectedChecksum
|
||||
): void {
|
||||
const baselineValue = baseline(0)
|
||||
const optimizedValue = optimized(0)
|
||||
strictEqual(optimizedValue, baselineValue, `${name}: timed arms returned different checksums`)
|
||||
validatedCases += 1
|
||||
const baselineCalibration = calibrate(baseline, expected)
|
||||
const optimizedCalibration = calibrate(optimized, expected)
|
||||
cappedCalibrations += Number(baselineCalibration.capped) + Number(optimizedCalibration.capped)
|
||||
const baselineSamples: number[] = []
|
||||
const optimizedSamples: number[] = []
|
||||
for (let round = 0; round < ROUNDS; round += 1) {
|
||||
if (round % 2 === 0) {
|
||||
baselineSamples.push(
|
||||
runSample(baseline, expected, baselineCalibration.iterations) /
|
||||
baselineCalibration.iterations
|
||||
)
|
||||
optimizedSamples.push(
|
||||
runSample(optimized, expected, optimizedCalibration.iterations) /
|
||||
optimizedCalibration.iterations
|
||||
)
|
||||
} else {
|
||||
optimizedSamples.push(
|
||||
runSample(optimized, expected, optimizedCalibration.iterations) /
|
||||
optimizedCalibration.iterations
|
||||
)
|
||||
baselineSamples.push(
|
||||
runSample(baseline, expected, baselineCalibration.iterations) /
|
||||
baselineCalibration.iterations
|
||||
)
|
||||
}
|
||||
}
|
||||
const baselineMs = median(baselineSamples)
|
||||
const optimizedMs = median(optimizedSamples)
|
||||
const speedup = baselineMs / Math.max(optimizedMs, Number.EPSILON)
|
||||
console.log(
|
||||
`${name}\t${baselineCalibration.iterations}\t${optimizedCalibration.iterations}\t${baselineCalibration.elapsedMs.toFixed(1)}\t${optimizedCalibration.elapsedMs.toFixed(1)}\t${baselineMs.toFixed(6)}\t${optimizedMs.toFixed(6)}\t${speedup.toFixed(1)}x`
|
||||
)
|
||||
}
|
||||
|
||||
function benchmarkSessionArms(
|
||||
name: string,
|
||||
baselineSession: () => NativeChatSession,
|
||||
optimizedSession: () => NativeChatSession
|
||||
): void {
|
||||
const expectedSession = baselineSession()
|
||||
deepStrictEqual(optimizedSession(), expectedSession, `${name}: session mismatch`)
|
||||
const expected = sessionExpectedChecksum(expectedSession)
|
||||
benchmark(
|
||||
name,
|
||||
(index) => consumeSession(baselineSession(), index),
|
||||
(index) => consumeSession(optimizedSession(), index),
|
||||
expected
|
||||
)
|
||||
}
|
||||
|
||||
function benchmarkMessageUpdate(name: string, messages: NativeChatMessage[]): void {
|
||||
benchmarkSessionArms(
|
||||
name,
|
||||
() => legacyMessageUpdateSession(messages),
|
||||
() => directMessageUpdateSession(messages)
|
||||
)
|
||||
}
|
||||
|
||||
const prose300 = proseFixture(300, 2_048, false)
|
||||
const fixtures = [
|
||||
['300 x 2KB prose, no turnId', prose300],
|
||||
['300 x 2KB prose, with turnId', proseFixture(300, 2_048, true)],
|
||||
['300 tool-heavy, no turnId', toolFixture(300)],
|
||||
['100 x 2KB prose, no turnId', proseFixture(100, 2_048, false)],
|
||||
['500 x 2KB prose, no turnId', proseFixture(500, 2_048, false)]
|
||||
] as const
|
||||
|
||||
console.log(
|
||||
`Node ${process.version}; ${ROUNDS} alternating interleaved median rounds; ${TARGET_SAMPLE_MS} ms calibration target; ${MAX_ITERATIONS} iteration cap`
|
||||
)
|
||||
console.log(
|
||||
'case\tbaseline iters\toptimized iters\tbaseline cal ms\toptimized cal ms\tbaseline ms/op\toptimized ms/op\tspeedup'
|
||||
)
|
||||
|
||||
for (const [name, messages] of fixtures) {
|
||||
benchmarkMessageUpdate(name, messages)
|
||||
}
|
||||
|
||||
benchmarkSessionArms(
|
||||
'300 x 2KB status-only frame',
|
||||
() => legacySession(prose300),
|
||||
() =>
|
||||
mergeNativeChatLiveSession({
|
||||
messages: prose300,
|
||||
sessionId: 'benchmark',
|
||||
agent: 'claude',
|
||||
hookState: null
|
||||
})
|
||||
)
|
||||
|
||||
strictEqual(checksum, expectedChecksum, 'benchmark checksum accounting drifted')
|
||||
strictEqual(sessionSink.value?.sessionId, 'benchmark', 'session outputs did not escape')
|
||||
strictEqual(Array.isArray(messageArraySink), true, 'message arrays did not escape')
|
||||
strictEqual(contentSink.length > 0, true, 'message content did not escape')
|
||||
console.log(
|
||||
`validated=${validatedCases} cases, checksum=${checksum}, capped calibrations=${cappedCalibrations}, runtime=${(
|
||||
performance.now() - benchmarkStartedAt
|
||||
).toFixed(0)} ms`
|
||||
)
|
||||
@@ -1,137 +0,0 @@
|
||||
// Run: ORCA_BACKGROUND_LAUNCH=1 node --import tsx config/scripts/codex-capacity-status-benchmark.ts
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { createCodexCapacityNoticeTracker } from '../../../../src/shared/codex-native-notice'
|
||||
import { HeadlessEmulator } from '../../../../src/main/daemon/emulation/headless-emulator'
|
||||
import { observeCodexCapacityTerminalStatus } from '../../../../src/main/runtime/terminal/output/codex-notice-terminal-status'
|
||||
|
||||
const fixture = readFileSync(
|
||||
new URL(
|
||||
'../../../../src/main/runtime/__fixtures__/codex-model-capacity-fullscreen.txt',
|
||||
import.meta.url
|
||||
),
|
||||
'utf8'
|
||||
)
|
||||
const notice = fixture.indexOf('■ Selected model')
|
||||
if (notice === -1) {
|
||||
throw new Error('Capacity notice missing from benchmark fixture')
|
||||
}
|
||||
const redraw = fixture.slice(0, fixture.lastIndexOf('\x1b[?2026h', notice))
|
||||
const scenarios = [
|
||||
{ name: 'plain-4K', data: 'x'.repeat(4096), iterations: 30_000 },
|
||||
{ name: 'captured-TUI', data: redraw, iterations: 30_000 },
|
||||
{
|
||||
name: 'warning-footer',
|
||||
data: '\x1b[?2026h\x1b[21;1H⚠ 1 warning · f2 to view\x1b[?2026l',
|
||||
iterations: 30_000
|
||||
},
|
||||
{ name: 'tiny-typing', data: 'a', iterations: 30_000 },
|
||||
{ name: 'marker-flood', data: '■'.repeat(4096), iterations: 1000 },
|
||||
{
|
||||
name: 'large-after-warning',
|
||||
data: `⚠ unrelated\n${'x'.repeat(1024 * 1024)}`,
|
||||
iterations: 128
|
||||
}
|
||||
]
|
||||
|
||||
function measure(iterations: number, observe: () => void): number {
|
||||
const start = performance.now()
|
||||
for (let index = 0; index < iterations; index++) {
|
||||
observe()
|
||||
}
|
||||
return performance.now() - start
|
||||
}
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
const run = () => {
|
||||
const tracker = createCodexCapacityNoticeTracker()
|
||||
let screenReads = 0
|
||||
const readScreen = () => {
|
||||
screenReads++
|
||||
return []
|
||||
}
|
||||
const elapsedMs = measure(scenario.iterations, () => tracker.observe(scenario.data, readScreen))
|
||||
if (screenReads !== 0) {
|
||||
throw new Error(`${scenario.name} read the screen ${screenReads} times`)
|
||||
}
|
||||
return elapsedMs
|
||||
}
|
||||
run()
|
||||
const samples = Array.from({ length: 5 }, run).sort((left, right) => left - right)
|
||||
const elapsedMs = samples[2]
|
||||
textResult(scenario.name, elapsedMs, scenario.iterations, scenario.data.length)
|
||||
}
|
||||
|
||||
for (const launchAgent of [undefined, 'codex'] as const) {
|
||||
const emulator = new HeadlessEmulator({ cols: 80, rows: 24 })
|
||||
const data = 'x'.repeat(4096)
|
||||
const iterations = 30_000
|
||||
let statusReads = 0
|
||||
const readStatus = () => {
|
||||
statusReads++
|
||||
return undefined
|
||||
}
|
||||
try {
|
||||
const observe = () =>
|
||||
observeCodexCapacityTerminalStatus(
|
||||
emulator,
|
||||
data,
|
||||
'node /some/wrapper.cjs',
|
||||
launchAgent,
|
||||
readStatus,
|
||||
1
|
||||
)
|
||||
measure(iterations, observe)
|
||||
const samples = Array.from({ length: 5 }, () => measure(iterations, observe)).sort(
|
||||
(left, right) => left - right
|
||||
)
|
||||
if (statusReads !== 0) {
|
||||
throw new Error(`Ordinary output read status ${statusReads} times`)
|
||||
}
|
||||
textResult(launchAgent ?? 'unrelated-terminal', samples[2], iterations, data.length)
|
||||
} finally {
|
||||
emulator.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
async function measureScreenRead(): Promise<void> {
|
||||
const emulator = new HeadlessEmulator({ cols: 40, rows: 24 })
|
||||
try {
|
||||
await emulator.write(fixture)
|
||||
for (const name of ['screen-only', 'capacity-parse-and-screen']) {
|
||||
const observe = () => {
|
||||
if (name === 'screen-only') {
|
||||
emulator.getVisibleLines()
|
||||
} else if (
|
||||
!createCodexCapacityNoticeTracker().observe(fixture, () => emulator.getVisibleLines())
|
||||
) {
|
||||
throw new Error('Captured capacity notice was not detected')
|
||||
}
|
||||
}
|
||||
measure(1000, observe)
|
||||
const samples = Array.from({ length: 5 }, () => measure(1000, observe)).sort(
|
||||
(left, right) => left - right
|
||||
)
|
||||
textResult(name, samples[2], 1000, fixture.length)
|
||||
}
|
||||
} finally {
|
||||
emulator.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
void measureScreenRead().catch((error: unknown) => {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
|
||||
function textResult(name: string, elapsedMs: number, iterations: number, chars: number): void {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
name,
|
||||
chars,
|
||||
iterations,
|
||||
microsecondsPerChunk: Number(((elapsedMs * 1000) / iterations).toFixed(3)),
|
||||
millisecondsPerMiB: Number((elapsedMs / ((chars * iterations) / 1048576)).toFixed(3))
|
||||
})
|
||||
)
|
||||
}
|
||||
-70
@@ -1,22 +1,14 @@
|
||||
import { structuredAgentSessionStopControl } from '../../../../../shared/agent-session/view/structured-agent-session-stop-control'
|
||||
import { isStructuredAgentSessionMainAgentWorking } from '../../../../../shared/agent-session/view/structured-agent-session-main-agent-working'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { startStructuredAgentLaunch } from './structured-agent-session-launch'
|
||||
import { holdStructuredAgentSessionLaunchOption } from './structured-agent-session-launch-options'
|
||||
import { recordHostModelCatalogSnapshot } from '@/runtime/native-chat/model-catalogs/host-model-catalog-snapshots'
|
||||
import { newAgentLaunchRequestId } from './routing/agent-launch-request-id'
|
||||
import {
|
||||
mocks,
|
||||
deferred,
|
||||
start,
|
||||
createParams,
|
||||
target,
|
||||
created,
|
||||
draft
|
||||
} from './structured-agent-session-launch-create-message.test-fixture'
|
||||
import { structuredAgentSessionCreateFingerprint } from '../../../../../shared/agent-session/lifecycle/structured-agent-session-mutation'
|
||||
import { getStructuredAgentSessionReadOwner } from '@/components/native-chat/session/structured/read/structured-agent-session-read-owner'
|
||||
import { getStructuredAgentSessionPendingSends } from '@/components/native-chat/session/structured/delivery/structured-agent-session-pending-sends'
|
||||
import { readStructuredAgentLaunchRecord } from './structured-agent-session-launch-persistence'
|
||||
import { getStructuredLaunchStateBySessionId } from './structured-agent-session-launch-registry'
|
||||
import { structuredSessionSelectionController } from '../options/structured-agent-session-selection-controller'
|
||||
@@ -24,68 +16,6 @@ import { setRuntimeEnvironmentConnectionGenerationForTests } from '@/store/slice
|
||||
import { replaceRuntimeEnvironmentRevisions } from '@/runtime/rpc/environment-revisions/runtime-environment-revision'
|
||||
|
||||
export function registerCreateSelectionTests(): void {
|
||||
it('freezes the message and held options in create and publishes without another read or send', async () => {
|
||||
mocks.exactSupport.mockResolvedValue(true)
|
||||
recordHostModelCatalogSnapshot(target, 'codex', null, {
|
||||
origin: 'probe',
|
||||
fetchedAt: 1,
|
||||
defaultHoldsInEveryWorkspace: true,
|
||||
models: [
|
||||
{
|
||||
id: 'picked-model',
|
||||
label: 'Picked',
|
||||
isDefault: false,
|
||||
efforts: [
|
||||
{ value: 'low', label: 'Low' },
|
||||
{ value: 'high', label: 'High' }
|
||||
]
|
||||
}
|
||||
]
|
||||
})
|
||||
const create = deferred<unknown>()
|
||||
mocks.call.mockReturnValue(create.promise)
|
||||
const launch = start()
|
||||
holdStructuredAgentSessionLaunchOption(launch.sessionId, 'model', 'picked-model')
|
||||
holdStructuredAgentSessionLaunchOption(launch.sessionId, 'effort', 'high')
|
||||
await vi.waitFor(() => expect(mocks.call).toHaveBeenCalledOnce())
|
||||
const params = createParams()
|
||||
expect(params.firstMessage?.body.blocks).toEqual([{ type: 'text', text: 'opening text' }])
|
||||
expect(params.options).toEqual({ model: 'picked-model', effort: 'high' })
|
||||
expect(params.firstMessage?.selection).toEqual({ model: 'picked-model', effort: 'high' })
|
||||
expect(params.envelope.payloadFingerprint).toBe(
|
||||
structuredAgentSessionCreateFingerprint({ sessionId: launch.sessionId, ...params })
|
||||
)
|
||||
expect(getStructuredAgentSessionPendingSends(launch.sessionId)[0]?.phase).toBe('sending')
|
||||
const stop = vi.fn(async () => null)
|
||||
expect(
|
||||
structuredAgentSessionStopControl({
|
||||
published: false,
|
||||
host: { stopsConversation: true, stop },
|
||||
transportState: { turnId: null, isWorking: false },
|
||||
sends: { sending: true, stopSends: vi.fn() }
|
||||
}).canStop
|
||||
).toBe(false)
|
||||
create.resolve(created())
|
||||
await launch.launchResult
|
||||
await expect(launch.promptDeliveryResult).resolves.toMatchObject({ delivered: true })
|
||||
const state = getStructuredAgentSessionReadOwner(launch.sessionId, target).getSnapshot().state
|
||||
expect(state.fence).toBe(1)
|
||||
const isWorking = isStructuredAgentSessionMainAgentWorking(null, state.submissions, state.fence)
|
||||
expect(isWorking).toBe(true)
|
||||
const control = structuredAgentSessionStopControl({
|
||||
published: true,
|
||||
host: { stopsConversation: true, stop },
|
||||
transportState: { turnId: null, isWorking },
|
||||
sends: { sending: false, stopSends: vi.fn() }
|
||||
})
|
||||
expect(control.canStop).toBe(true)
|
||||
await control.stop()
|
||||
expect(stop).toHaveBeenCalledWith(null, expect.any(Function))
|
||||
expect(draft(launch.sessionId)).toBe('')
|
||||
expect(mocks.inventory).not.toHaveBeenCalled()
|
||||
expect(mocks.call.mock.calls.map(([, method]) => method)).toEqual(['agentSession.create'])
|
||||
})
|
||||
|
||||
it('keeps frozen selection completed when exact create proof arrives after the deadline', async () => {
|
||||
const create = deferred<unknown>()
|
||||
mocks.call.mockReturnValue(create.promise)
|
||||
|
||||
Reference in New Issue
Block a user