mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 03:15:01 +00:00
test: remove repeated readiness and telemetry schema cases (#41983)
Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
@@ -6401,7 +6401,6 @@
|
||||
"src/shared/native-chat/transcript/native-chat-unverifiable-turn-status.test.ts": 150,
|
||||
"src/shared/native-file-drop.test.ts": 59,
|
||||
"src/shared/ndjson-retained-chunk-memory.test.ts": 327,
|
||||
"src/shared/nested-repo-telemetry-schema.test.ts": 196,
|
||||
"src/shared/nested-repo-telemetry.test.ts": 55,
|
||||
"src/shared/network-proxy.test.ts": 44,
|
||||
"src/shared/network/manual-address.test.ts": 68,
|
||||
|
||||
@@ -1,13 +1,6 @@
|
||||
import { describe, vi } from 'vitest'
|
||||
import './agent-transcript-replay-test-harness'
|
||||
import { describeScreenRuledAgentTranscripts } from './screen-ruled-agent-transcript-suite'
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
BrowserWindow: { fromId: vi.fn(() => null) },
|
||||
webContents: { fromId: vi.fn(() => null) },
|
||||
ipcMain: { on: vi.fn(), removeListener: vi.fn() },
|
||||
app: { getPath: vi.fn(() => '/tmp') }
|
||||
}))
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { finalReplayFrame } from './agent-transcript-replay-test-harness'
|
||||
import { evaluateAgentStateRules } from '../../agent-state-rules/agent-state-rules-engine'
|
||||
|
||||
// prime-agent 0.9.8 on macOS, OpenRouter, isolated HOME; 0.9.5 recorded for PR #22154 (see
|
||||
// each .meta.json); STA-8741.
|
||||
@@ -30,11 +23,17 @@ const NOT_READY = [
|
||||
]
|
||||
|
||||
describe('Prime Agent readiness from captured bytes', () => {
|
||||
describeScreenRuledAgentTranscripts({
|
||||
agent: 'prime-agent',
|
||||
foregroundProcess: 'prime-agent',
|
||||
ready: READY,
|
||||
notReady: NOT_READY,
|
||||
readyWithoutScreen: READY.map(({ name }) => name)
|
||||
it.each(READY)('$name ($what) reads ready', async ({ name, cols, rows }) => {
|
||||
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
|
||||
expect(
|
||||
evaluateAgentStateRules('prime-agent', { readScreenLines: () => ruledScreenLines })?.state
|
||||
).toBe('idle')
|
||||
})
|
||||
|
||||
it.each(NOT_READY)('$name ($what) does not read ready', async ({ name, cols, rows }) => {
|
||||
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
|
||||
expect(
|
||||
evaluateAgentStateRules('prime-agent', { readScreenLines: () => ruledScreenLines })?.state
|
||||
).not.toBe('idle')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
// One replay suite for every agent whose readiness its live screen decides (terminal-wait-detection.ts).
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
createTranscriptPane,
|
||||
TRANSCRIPT_PANE_PTY_ID,
|
||||
waitForTranscriptIdle
|
||||
} from '../../orca-runtime-tests/agent-transcript-pane-test-harness'
|
||||
import {
|
||||
finalReadProjection,
|
||||
finalReplayFrame,
|
||||
readRuntimeFixture,
|
||||
type TranscriptReplayResize
|
||||
} from './agent-transcript-replay-test-harness'
|
||||
import { isQuietReadyScreenBody } from './terminal-wait-detection'
|
||||
import { evaluateAgentStateRules } from '../../agent-state-rules/agent-state-rules-engine'
|
||||
import type { TuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent'
|
||||
|
||||
export type ScreenRuledFixture = { name: string; cols: number; rows: number; what: string }
|
||||
|
||||
export type ScreenRuledAgentSuite = {
|
||||
agent: TuiAgent
|
||||
foregroundProcess: string
|
||||
ready: readonly ScreenRuledFixture[]
|
||||
notReady: readonly ScreenRuledFixture[]
|
||||
/** Ready recordings the text rules or the quiet-process lane settle with no screen. */
|
||||
readyWithoutScreen: readonly string[]
|
||||
}
|
||||
|
||||
// Mismatched grids garble cursor-addressed chrome (#23475 review).
|
||||
const MISMATCHED_GRIDS = [
|
||||
[80, 24],
|
||||
[100, 30],
|
||||
[60, 20],
|
||||
[150, 50]
|
||||
] as const
|
||||
|
||||
// 8s covers 3s quiescence and the 2s poll; 5s refusal allows both plus at least one poll tick.
|
||||
const READY_TIMEOUT_MS = 8_000
|
||||
const REFUSAL_TIMEOUT_MS = 5_000
|
||||
// Allow for multi-MB animated recordings taking seconds in onPtyData.
|
||||
const PANE_SETUP_SLACK_MS = 30_000
|
||||
|
||||
function chunkCount(name: string): number {
|
||||
return Math.ceil(readRuntimeFixture(name).length / 64)
|
||||
}
|
||||
|
||||
/** Whether the agent's own rules read `screenLines` as an idle composer. */
|
||||
export function readsIdleComposer(agent: TuiAgent, screenLines: readonly string[]): boolean {
|
||||
return evaluateAgentStateRules(agent, { readScreenLines: () => screenLines })?.state === 'idle'
|
||||
}
|
||||
|
||||
export function describeScreenRuledAgentTranscripts(suite: ScreenRuledAgentSuite): void {
|
||||
const { agent } = suite
|
||||
const rule = (screenLines: readonly string[]): boolean => readsIdleComposer(agent, screenLines)
|
||||
const [firstReady] = suite.ready
|
||||
if (!firstReady) {
|
||||
throw new Error(`${agent}: a suite needs a ready recording`)
|
||||
}
|
||||
|
||||
describe('the screen rule on the final recorded frame', () => {
|
||||
it.each(suite.ready)('$name ($what) reads ready', async ({ name, cols, rows }) => {
|
||||
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
|
||||
expect(rule(ruledScreenLines)).toBe(true)
|
||||
})
|
||||
it.each(suite.notReady)('$name ($what) does not', async ({ name, cols, rows }) => {
|
||||
const { ruledScreenLines } = await finalReplayFrame(name, cols, rows)
|
||||
expect(rule(ruledScreenLines)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('fails safe on a garbled grid', () => {
|
||||
it.each(suite.notReady)('$name never reads ready', async ({ name, cols, rows }) => {
|
||||
const resize: TranscriptReplayResize = {
|
||||
atChunk: Math.floor(chunkCount(name) / 2),
|
||||
cols: 100,
|
||||
rows: 30
|
||||
}
|
||||
const frames = [
|
||||
...(await Promise.all(MISMATCHED_GRIDS.map(([c, r]) => finalReplayFrame(name, c, r)))),
|
||||
await finalReplayFrame(name, cols, rows, resize)
|
||||
]
|
||||
for (const { ruledScreenLines, waitText } of frames) {
|
||||
expect(isQuietReadyScreenBody(waitText, agent, () => ruledScreenLines)).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('through the runtime', () => {
|
||||
async function pane(fixture: ScreenRuledFixture) {
|
||||
const created = await createTranscriptPane({
|
||||
paneTitle: 'Terminal',
|
||||
foregroundProcess: suite.foregroundProcess,
|
||||
launchAgent: agent,
|
||||
data: readRuntimeFixture(fixture.name),
|
||||
size: { cols: fixture.cols, rows: fixture.rows }
|
||||
})
|
||||
// Drain emulator parsing: one multi-MB chunk can outlive the output quiet clock unlike a live PTY's silence.
|
||||
await created.runtime.readTerminal(created.handle, { screen: true })
|
||||
return created
|
||||
}
|
||||
|
||||
it.each(suite.ready)(
|
||||
'$name: a tui-idle wait settles once the pane is quiet',
|
||||
async (fixture) => {
|
||||
const { runtime, handle } = await pane(fixture)
|
||||
await expect(
|
||||
waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS)
|
||||
).resolves.toMatchObject({
|
||||
condition: 'tui-idle',
|
||||
satisfied: true
|
||||
})
|
||||
},
|
||||
READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS
|
||||
)
|
||||
|
||||
it.each(suite.notReady)(
|
||||
'$name: a tui-idle wait does not settle ready',
|
||||
async (fixture) => {
|
||||
const { runtime, handle } = await pane(fixture)
|
||||
const result = await waitForTranscriptIdle({ runtime, handle }, REFUSAL_TIMEOUT_MS).catch(
|
||||
(error: unknown) => ({ satisfied: false, error: String(error) })
|
||||
)
|
||||
expect(result.satisfied).toBe(false)
|
||||
},
|
||||
REFUSAL_TIMEOUT_MS + PANE_SETUP_SLACK_MS
|
||||
)
|
||||
|
||||
// Why 100x30: a grid no recording was painted for. The PTY reports its real size.
|
||||
it.each(suite.ready)(
|
||||
'$name: on a grid the PTY does not have, the lanes the screen would shut decide',
|
||||
async (fixture) => {
|
||||
const options = {
|
||||
paneTitle: 'Terminal',
|
||||
foregroundProcess: suite.foregroundProcess,
|
||||
launchAgent: agent,
|
||||
data: readRuntimeFixture(fixture.name),
|
||||
size: { cols: 100, rows: 30 }
|
||||
}
|
||||
const { runtime, handle } = await createTranscriptPane(options)
|
||||
await runtime.readTerminal(handle, { screen: true })
|
||||
options.size = { cols: fixture.cols, rows: fixture.rows }
|
||||
const result = await waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS).catch(
|
||||
() => ({ satisfied: false })
|
||||
)
|
||||
expect(result.satisfied).toBe(suite.readyWithoutScreen.includes(fixture.name))
|
||||
},
|
||||
READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS
|
||||
)
|
||||
|
||||
// Reattach reflows without a TUI repaint.
|
||||
it(
|
||||
'on a grid reflowed to the PTY size without a repaint, the lanes the screen would shut decide',
|
||||
async () => {
|
||||
const options = {
|
||||
paneTitle: 'Terminal',
|
||||
foregroundProcess: suite.foregroundProcess,
|
||||
launchAgent: agent,
|
||||
data: readRuntimeFixture(firstReady.name),
|
||||
size: { cols: 100, rows: 30 }
|
||||
}
|
||||
const { runtime, handle } = await createTranscriptPane(options)
|
||||
options.size = { cols: firstReady.cols, rows: firstReady.rows }
|
||||
runtime.reflowHeadlessTerminalToPtyGrid(
|
||||
TRANSCRIPT_PANE_PTY_ID,
|
||||
firstReady.cols,
|
||||
firstReady.rows
|
||||
)
|
||||
// Echoing the already-reflowed size sends no SIGWINCH, so the TUI never repaints.
|
||||
runtime.onExternalPtyResize(TRANSCRIPT_PANE_PTY_ID, firstReady.cols, firstReady.rows)
|
||||
await runtime.readTerminal(handle, { screen: true })
|
||||
const result = await waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS).catch(
|
||||
() => ({ satisfied: false })
|
||||
)
|
||||
expect(result.satisfied).toBe(suite.readyWithoutScreen.includes(firstReady.name))
|
||||
},
|
||||
READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS
|
||||
)
|
||||
|
||||
// Why no bytes: a restored pane has no output clock, so the provider's screen is all it has.
|
||||
// Why the read projection: it is what the probe's screen read returns, draft blanked.
|
||||
async function probedVerdict(fixture: ScreenRuledFixture): Promise<boolean> {
|
||||
const { lines, draft } = await finalReadProjection(fixture.name, fixture.cols, fixture.rows)
|
||||
const { runtime, handle } = await createTranscriptPane({
|
||||
paneTitle: 'Terminal',
|
||||
foregroundProcess: suite.foregroundProcess,
|
||||
launchAgent: agent,
|
||||
data: ''
|
||||
})
|
||||
const read = vi.spyOn(runtime, 'readTerminal').mockResolvedValue({
|
||||
handle,
|
||||
status: 'running',
|
||||
tail: lines,
|
||||
...(draft === undefined ? {} : { draft }),
|
||||
truncated: false,
|
||||
nextCursor: null,
|
||||
source: 'screen'
|
||||
})
|
||||
const result = await waitForTranscriptIdle({ runtime, handle }, 2_500).catch(() => ({
|
||||
satisfied: false
|
||||
}))
|
||||
// Presence precondition: the visible-screen probe actually ran.
|
||||
expect(read).toHaveBeenCalled()
|
||||
return result.satisfied === true
|
||||
}
|
||||
|
||||
it.each(suite.ready)(
|
||||
'$name: a restored pane settles from a visible-screen read',
|
||||
async (fixture) => {
|
||||
expect(await probedVerdict(fixture)).toBe(true)
|
||||
},
|
||||
15_000
|
||||
)
|
||||
|
||||
it.each(suite.notReady)(
|
||||
'$name: a restored pane does not settle from a visible-screen read',
|
||||
async (fixture) => {
|
||||
expect(await probedVerdict(fixture)).toBe(false)
|
||||
},
|
||||
15_000
|
||||
)
|
||||
|
||||
it('waits for quiet again after a ready pane repaints', async () => {
|
||||
const { runtime, handle } = await pane(firstReady)
|
||||
vi.useFakeTimers({
|
||||
toFake: ['Date', 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval']
|
||||
})
|
||||
try {
|
||||
const settled = vi.fn()
|
||||
const waiting = runtime.waitForTerminal(handle, {
|
||||
condition: 'tui-idle',
|
||||
timeoutMs: READY_TIMEOUT_MS
|
||||
})
|
||||
void waiting.then(settled, () => {})
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
expect(settled).not.toHaveBeenCalled()
|
||||
|
||||
runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, readRuntimeFixture(firstReady.name), Date.now())
|
||||
await runtime.readTerminal(handle, { screen: true })
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
expect(settled).not.toHaveBeenCalled()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
await expect(waiting).resolves.toMatchObject({ condition: 'tui-idle', satisfied: true })
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { eventSchemas } from './telemetry-events'
|
||||
|
||||
const attemptId = '2fbac1e3-5094-45b4-80a6-90281e6e9e09'
|
||||
|
||||
describe('nested repo import telemetry schemas', () => {
|
||||
it('rejects raw paths, repo names, and raw errors via .strict()', () => {
|
||||
expect(
|
||||
eventSchemas.add_repo_nested_scan_result.safeParse({
|
||||
attempt_id: attemptId,
|
||||
surface: 'onboarding',
|
||||
runtime_kind: 'local',
|
||||
result: 'review_shown',
|
||||
selected_path_kind: 'non_git_folder',
|
||||
found_count: 1,
|
||||
found_count_bucket: '1',
|
||||
truncated: false,
|
||||
timed_out: false,
|
||||
selected_path: '/Users/alice/work/platform'
|
||||
}).success
|
||||
).toBe(false)
|
||||
|
||||
expect(
|
||||
eventSchemas.add_repo_nested_import_action.safeParse({
|
||||
attempt_id: attemptId,
|
||||
surface: 'sidebar',
|
||||
runtime_kind: 'local',
|
||||
action: 'import_group',
|
||||
found_count: 1,
|
||||
found_count_bucket: '1',
|
||||
selected_count: 1,
|
||||
selected_count_bucket: '1',
|
||||
all_selected: true,
|
||||
repo_name: 'secret-service'
|
||||
}).success
|
||||
).toBe(false)
|
||||
|
||||
expect(
|
||||
eventSchemas.add_repo_nested_import_result.safeParse({
|
||||
attempt_id: attemptId,
|
||||
surface: 'sidebar',
|
||||
runtime_kind: 'ssh',
|
||||
mode: 'group',
|
||||
outcome: 'failed',
|
||||
found_count: 1,
|
||||
found_count_bucket: '1',
|
||||
selected_count: 1,
|
||||
selected_count_bucket: '1',
|
||||
imported_count: 0,
|
||||
imported_count_bucket: '0',
|
||||
already_known_count: 0,
|
||||
already_known_count_bucket: '0',
|
||||
failed_count: 1,
|
||||
failed_count_bucket: '1',
|
||||
all_selected: true,
|
||||
error_message: 'failed at /Users/alice/work/platform'
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects mismatched exact counts and buckets', () => {
|
||||
expect(
|
||||
eventSchemas.add_repo_nested_import_action.safeParse({
|
||||
attempt_id: attemptId,
|
||||
surface: 'sidebar',
|
||||
runtime_kind: 'local',
|
||||
action: 'import_group',
|
||||
found_count: 1,
|
||||
found_count_bucket: '16+',
|
||||
selected_count: 1,
|
||||
selected_count_bucket: '1',
|
||||
all_selected: true
|
||||
}).success
|
||||
).toBe(false)
|
||||
|
||||
expect(
|
||||
eventSchemas.add_repo_nested_import_result.safeParse({
|
||||
attempt_id: attemptId,
|
||||
surface: 'sidebar',
|
||||
runtime_kind: 'local',
|
||||
mode: 'group',
|
||||
outcome: 'success',
|
||||
found_count: 2,
|
||||
found_count_bucket: '2-3',
|
||||
selected_count: 2,
|
||||
selected_count_bucket: '2-3',
|
||||
imported_count: 2,
|
||||
imported_count_bucket: '16+',
|
||||
already_known_count: 0,
|
||||
already_known_count_bucket: '0',
|
||||
failed_count: 0,
|
||||
failed_count_bucket: '0',
|
||||
all_selected: true
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -1,17 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { eventSchemas } from './telemetry-events'
|
||||
|
||||
describe('profile state authority telemetry', () => {
|
||||
it('rejects paths or other unbounded diagnostic fields', () => {
|
||||
expect(
|
||||
eventSchemas.profile_state_authority_selected.safeParse({
|
||||
backend: 'sqlite',
|
||||
classification: 'sqlite-only',
|
||||
authority_mode: 'sqlite-established',
|
||||
runtime: 'desktop',
|
||||
migrated: false,
|
||||
database_path: '/private/profile.sqlite'
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -1,4 +1,3 @@
|
||||
import './telemetry-orca-cli-feature-tip.spec'
|
||||
import './telemetry-events-feature-education.spec'
|
||||
import './telemetry-feature-wall-events.spec'
|
||||
import './profile-state-telemetry-schema.spec'
|
||||
|
||||
Reference in New Issue
Block a user