test: remove repeated readiness and telemetry schema cases (#41983)

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
OrcaWin
2026-11-13 03:24:50 -08:00
committed by GitHub
co-authored by OrcaWin
parent f4d294dac2
commit 7658aebc4d
6 changed files with 15 additions and 381 deletions
@@ -6401,7 +6401,6 @@
"src/shared/native-chat/transcript/native-chat-unverifiable-turn-status.test.ts": 150,
"src/shared/native-file-drop.test.ts": 59,
"src/shared/ndjson-retained-chunk-memory.test.ts": 327,
"src/shared/nested-repo-telemetry-schema.test.ts": 196,
"src/shared/nested-repo-telemetry.test.ts": 55,
"src/shared/network-proxy.test.ts": 44,
"src/shared/network/manual-address.test.ts": 68,
@@ -1,13 +1,6 @@
import { describe, vi } from 'vitest'
import './agent-transcript-replay-test-harness'
import { describeScreenRuledAgentTranscripts } from './screen-ruled-agent-transcript-suite'
vi.mock('electron', () => ({
BrowserWindow: { fromId: vi.fn(() => null) },
webContents: { fromId: vi.fn(() => null) },
ipcMain: { on: vi.fn(), removeListener: vi.fn() },
app: { getPath: vi.fn(() => '/tmp') }
}))
import { describe, expect, it } from 'vitest'
import { finalReplayFrame } from './agent-transcript-replay-test-harness'
import { evaluateAgentStateRules } from '../../agent-state-rules/agent-state-rules-engine'
// prime-agent 0.9.8 on macOS, OpenRouter, isolated HOME; 0.9.5 recorded for PR #22154 (see
// each .meta.json); STA-8741.
@@ -30,11 +23,17 @@ const NOT_READY = [
]
describe('Prime Agent readiness from captured bytes', () => {
describeScreenRuledAgentTranscripts({
agent: 'prime-agent',
foregroundProcess: 'prime-agent',
ready: READY,
notReady: NOT_READY,
readyWithoutScreen: READY.map(({ name }) => name)
it.each(READY)('$name ($what) reads ready', async ({ name, cols, rows }) => {
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
expect(
evaluateAgentStateRules('prime-agent', { readScreenLines: () => ruledScreenLines })?.state
).toBe('idle')
})
it.each(NOT_READY)('$name ($what) does not read ready', async ({ name, cols, rows }) => {
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
expect(
evaluateAgentStateRules('prime-agent', { readScreenLines: () => ruledScreenLines })?.state
).not.toBe('idle')
})
})
@@ -1,249 +0,0 @@
// One replay suite for every agent whose readiness its live screen decides (terminal-wait-detection.ts).
import { describe, expect, it, vi } from 'vitest'
import {
createTranscriptPane,
TRANSCRIPT_PANE_PTY_ID,
waitForTranscriptIdle
} from '../../orca-runtime-tests/agent-transcript-pane-test-harness'
import {
finalReadProjection,
finalReplayFrame,
readRuntimeFixture,
type TranscriptReplayResize
} from './agent-transcript-replay-test-harness'
import { isQuietReadyScreenBody } from './terminal-wait-detection'
import { evaluateAgentStateRules } from '../../agent-state-rules/agent-state-rules-engine'
import type { TuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent'
export type ScreenRuledFixture = { name: string; cols: number; rows: number; what: string }
export type ScreenRuledAgentSuite = {
agent: TuiAgent
foregroundProcess: string
ready: readonly ScreenRuledFixture[]
notReady: readonly ScreenRuledFixture[]
/** Ready recordings the text rules or the quiet-process lane settle with no screen. */
readyWithoutScreen: readonly string[]
}
// Mismatched grids garble cursor-addressed chrome (#23475 review).
const MISMATCHED_GRIDS = [
[80, 24],
[100, 30],
[60, 20],
[150, 50]
] as const
// 8s covers 3s quiescence and the 2s poll; 5s refusal allows both plus at least one poll tick.
const READY_TIMEOUT_MS = 8_000
const REFUSAL_TIMEOUT_MS = 5_000
// Allow for multi-MB animated recordings taking seconds in onPtyData.
const PANE_SETUP_SLACK_MS = 30_000
function chunkCount(name: string): number {
return Math.ceil(readRuntimeFixture(name).length / 64)
}
/** Whether the agent's own rules read `screenLines` as an idle composer. */
export function readsIdleComposer(agent: TuiAgent, screenLines: readonly string[]): boolean {
return evaluateAgentStateRules(agent, { readScreenLines: () => screenLines })?.state === 'idle'
}
export function describeScreenRuledAgentTranscripts(suite: ScreenRuledAgentSuite): void {
const { agent } = suite
const rule = (screenLines: readonly string[]): boolean => readsIdleComposer(agent, screenLines)
const [firstReady] = suite.ready
if (!firstReady) {
throw new Error(`${agent}: a suite needs a ready recording`)
}
describe('the screen rule on the final recorded frame', () => {
it.each(suite.ready)('$name ($what) reads ready', async ({ name, cols, rows }) => {
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
expect(rule(ruledScreenLines)).toBe(true)
})
it.each(suite.notReady)('$name ($what) does not', async ({ name, cols, rows }) => {
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
expect(rule(ruledScreenLines)).toBe(false)
})
})
describe('fails safe on a garbled grid', () => {
it.each(suite.notReady)('$name never reads ready', async ({ name, cols, rows }) => {
const resize: TranscriptReplayResize = {
atChunk: Math.floor(chunkCount(name) / 2),
cols: 100,
rows: 30
}
const frames = [
...(await Promise.all(MISMATCHED_GRIDS.map(([c, r]) => finalReplayFrame(name, c, r)))),
await finalReplayFrame(name, cols, rows, resize)
]
for (const { ruledScreenLines, waitText } of frames) {
expect(isQuietReadyScreenBody(waitText, agent, () => ruledScreenLines)).toBe(false)
}
})
})
describe('through the runtime', () => {
async function pane(fixture: ScreenRuledFixture) {
const created = await createTranscriptPane({
paneTitle: 'Terminal',
foregroundProcess: suite.foregroundProcess,
launchAgent: agent,
data: readRuntimeFixture(fixture.name),
size: { cols: fixture.cols, rows: fixture.rows }
})
// Drain emulator parsing: one multi-MB chunk can outlive the output quiet clock unlike a live PTY's silence.
await created.runtime.readTerminal(created.handle, { screen: true })
return created
}
it.each(suite.ready)(
'$name: a tui-idle wait settles once the pane is quiet',
async (fixture) => {
const { runtime, handle } = await pane(fixture)
await expect(
waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS)
).resolves.toMatchObject({
condition: 'tui-idle',
satisfied: true
})
},
READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS
)
it.each(suite.notReady)(
'$name: a tui-idle wait does not settle ready',
async (fixture) => {
const { runtime, handle } = await pane(fixture)
const result = await waitForTranscriptIdle({ runtime, handle }, REFUSAL_TIMEOUT_MS).catch(
(error: unknown) => ({ satisfied: false, error: String(error) })
)
expect(result.satisfied).toBe(false)
},
REFUSAL_TIMEOUT_MS + PANE_SETUP_SLACK_MS
)
// Why 100x30: a grid no recording was painted for. The PTY reports its real size.
it.each(suite.ready)(
'$name: on a grid the PTY does not have, the lanes the screen would shut decide',
async (fixture) => {
const options = {
paneTitle: 'Terminal',
foregroundProcess: suite.foregroundProcess,
launchAgent: agent,
data: readRuntimeFixture(fixture.name),
size: { cols: 100, rows: 30 }
}
const { runtime, handle } = await createTranscriptPane(options)
await runtime.readTerminal(handle, { screen: true })
options.size = { cols: fixture.cols, rows: fixture.rows }
const result = await waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS).catch(
() => ({ satisfied: false })
)
expect(result.satisfied).toBe(suite.readyWithoutScreen.includes(fixture.name))
},
READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS
)
// Reattach reflows without a TUI repaint.
it(
'on a grid reflowed to the PTY size without a repaint, the lanes the screen would shut decide',
async () => {
const options = {
paneTitle: 'Terminal',
foregroundProcess: suite.foregroundProcess,
launchAgent: agent,
data: readRuntimeFixture(firstReady.name),
size: { cols: 100, rows: 30 }
}
const { runtime, handle } = await createTranscriptPane(options)
options.size = { cols: firstReady.cols, rows: firstReady.rows }
runtime.reflowHeadlessTerminalToPtyGrid(
TRANSCRIPT_PANE_PTY_ID,
firstReady.cols,
firstReady.rows
)
// Echoing the already-reflowed size sends no SIGWINCH, so the TUI never repaints.
runtime.onExternalPtyResize(TRANSCRIPT_PANE_PTY_ID, firstReady.cols, firstReady.rows)
await runtime.readTerminal(handle, { screen: true })
const result = await waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS).catch(
() => ({ satisfied: false })
)
expect(result.satisfied).toBe(suite.readyWithoutScreen.includes(firstReady.name))
},
READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS
)
// Why no bytes: a restored pane has no output clock, so the provider's screen is all it has.
// Why the read projection: it is what the probe's screen read returns, draft blanked.
async function probedVerdict(fixture: ScreenRuledFixture): Promise<boolean> {
const { lines, draft } = await finalReadProjection(fixture.name, fixture.cols, fixture.rows)
const { runtime, handle } = await createTranscriptPane({
paneTitle: 'Terminal',
foregroundProcess: suite.foregroundProcess,
launchAgent: agent,
data: ''
})
const read = vi.spyOn(runtime, 'readTerminal').mockResolvedValue({
handle,
status: 'running',
tail: lines,
...(draft === undefined ? {} : { draft }),
truncated: false,
nextCursor: null,
source: 'screen'
})
const result = await waitForTranscriptIdle({ runtime, handle }, 2_500).catch(() => ({
satisfied: false
}))
// Presence precondition: the visible-screen probe actually ran.
expect(read).toHaveBeenCalled()
return result.satisfied === true
}
it.each(suite.ready)(
'$name: a restored pane settles from a visible-screen read',
async (fixture) => {
expect(await probedVerdict(fixture)).toBe(true)
},
15_000
)
it.each(suite.notReady)(
'$name: a restored pane does not settle from a visible-screen read',
async (fixture) => {
expect(await probedVerdict(fixture)).toBe(false)
},
15_000
)
it('waits for quiet again after a ready pane repaints', async () => {
const { runtime, handle } = await pane(firstReady)
vi.useFakeTimers({
toFake: ['Date', 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval']
})
try {
const settled = vi.fn()
const waiting = runtime.waitForTerminal(handle, {
condition: 'tui-idle',
timeoutMs: READY_TIMEOUT_MS
})
void waiting.then(settled, () => {})
await vi.advanceTimersByTimeAsync(2_000)
expect(settled).not.toHaveBeenCalled()
runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, readRuntimeFixture(firstReady.name), Date.now())
await runtime.readTerminal(handle, { screen: true })
await vi.advanceTimersByTimeAsync(2_000)
expect(settled).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(2_000)
await expect(waiting).resolves.toMatchObject({ condition: 'tui-idle', satisfied: true })
} finally {
vi.useRealTimers()
}
})
})
}
@@ -1,97 +0,0 @@
import { describe, expect, it } from 'vitest'
import { eventSchemas } from './telemetry-events'
const attemptId = '2fbac1e3-5094-45b4-80a6-90281e6e9e09'
describe('nested repo import telemetry schemas', () => {
it('rejects raw paths, repo names, and raw errors via .strict()', () => {
expect(
eventSchemas.add_repo_nested_scan_result.safeParse({
attempt_id: attemptId,
surface: 'onboarding',
runtime_kind: 'local',
result: 'review_shown',
selected_path_kind: 'non_git_folder',
found_count: 1,
found_count_bucket: '1',
truncated: false,
timed_out: false,
selected_path: '/Users/alice/work/platform'
}).success
).toBe(false)
expect(
eventSchemas.add_repo_nested_import_action.safeParse({
attempt_id: attemptId,
surface: 'sidebar',
runtime_kind: 'local',
action: 'import_group',
found_count: 1,
found_count_bucket: '1',
selected_count: 1,
selected_count_bucket: '1',
all_selected: true,
repo_name: 'secret-service'
}).success
).toBe(false)
expect(
eventSchemas.add_repo_nested_import_result.safeParse({
attempt_id: attemptId,
surface: 'sidebar',
runtime_kind: 'ssh',
mode: 'group',
outcome: 'failed',
found_count: 1,
found_count_bucket: '1',
selected_count: 1,
selected_count_bucket: '1',
imported_count: 0,
imported_count_bucket: '0',
already_known_count: 0,
already_known_count_bucket: '0',
failed_count: 1,
failed_count_bucket: '1',
all_selected: true,
error_message: 'failed at /Users/alice/work/platform'
}).success
).toBe(false)
})
it('rejects mismatched exact counts and buckets', () => {
expect(
eventSchemas.add_repo_nested_import_action.safeParse({
attempt_id: attemptId,
surface: 'sidebar',
runtime_kind: 'local',
action: 'import_group',
found_count: 1,
found_count_bucket: '16+',
selected_count: 1,
selected_count_bucket: '1',
all_selected: true
}).success
).toBe(false)
expect(
eventSchemas.add_repo_nested_import_result.safeParse({
attempt_id: attemptId,
surface: 'sidebar',
runtime_kind: 'local',
mode: 'group',
outcome: 'success',
found_count: 2,
found_count_bucket: '2-3',
selected_count: 2,
selected_count_bucket: '2-3',
imported_count: 2,
imported_count_bucket: '16+',
already_known_count: 0,
already_known_count_bucket: '0',
failed_count: 0,
failed_count_bucket: '0',
all_selected: true
}).success
).toBe(false)
})
})
@@ -1,17 +0,0 @@
import { describe, expect, it } from 'vitest'
import { eventSchemas } from './telemetry-events'
describe('profile state authority telemetry', () => {
it('rejects paths or other unbounded diagnostic fields', () => {
expect(
eventSchemas.profile_state_authority_selected.safeParse({
backend: 'sqlite',
classification: 'sqlite-only',
authority_mode: 'sqlite-established',
runtime: 'desktop',
migrated: false,
database_path: '/private/profile.sqlite'
}).success
).toBe(false)
})
})
@@ -1,4 +1,3 @@
import './telemetry-orca-cli-feature-tip.spec'
import './telemetry-events-feature-education.spec'
import './telemetry-feature-wall-events.spec'
import './profile-state-telemetry-schema.spec'