From 7658aebc4d00f9c5abf4ae8aca8ddf4328dbd3d2 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Fri, 13 Nov 2026 03:24:50 -0800 Subject: [PATCH] test: remove repeated readiness and telemetry schema cases (#41983) Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com> --- .../scripts/ci/sharding/ci-shard-timings.json | 1 - ...agent-screen-readiness-transcripts.test.ts | 31 ++- .../screen-ruled-agent-transcript-suite.ts | 249 ------------------ .../nested-repo-telemetry-schema.test.ts | 97 ------- .../profile-state-telemetry-schema.spec.ts | 17 -- .../telemetry-payload-contracts.test.ts | 1 - 6 files changed, 15 insertions(+), 381 deletions(-) delete mode 100644 src/main/runtime/terminal/wait/screen-ruled-agent-transcript-suite.ts delete mode 100644 src/shared/nested-repo-telemetry-schema.test.ts delete mode 100644 src/shared/profile-state-telemetry-schema.spec.ts diff --git a/config/scripts/ci/sharding/ci-shard-timings.json b/config/scripts/ci/sharding/ci-shard-timings.json index fcbafaeab0f..62734da3d05 100644 --- a/config/scripts/ci/sharding/ci-shard-timings.json +++ b/config/scripts/ci/sharding/ci-shard-timings.json @@ -6401,7 +6401,6 @@ "src/shared/native-chat/transcript/native-chat-unverifiable-turn-status.test.ts": 150, "src/shared/native-file-drop.test.ts": 59, "src/shared/ndjson-retained-chunk-memory.test.ts": 327, - "src/shared/nested-repo-telemetry-schema.test.ts": 196, "src/shared/nested-repo-telemetry.test.ts": 55, "src/shared/network-proxy.test.ts": 44, "src/shared/network/manual-address.test.ts": 68, diff --git a/src/main/runtime/terminal/wait/prime-agent-screen-readiness-transcripts.test.ts b/src/main/runtime/terminal/wait/prime-agent-screen-readiness-transcripts.test.ts index e3b7efea742..e44a6ebe0c5 100644 --- a/src/main/runtime/terminal/wait/prime-agent-screen-readiness-transcripts.test.ts +++ b/src/main/runtime/terminal/wait/prime-agent-screen-readiness-transcripts.test.ts @@ -1,13 +1,6 @@ -import { describe, vi } from 'vitest' -import './agent-transcript-replay-test-harness' -import { describeScreenRuledAgentTranscripts } from './screen-ruled-agent-transcript-suite' - -vi.mock('electron', () => ({ - BrowserWindow: { fromId: vi.fn(() => null) }, - webContents: { fromId: vi.fn(() => null) }, - ipcMain: { on: vi.fn(), removeListener: vi.fn() }, - app: { getPath: vi.fn(() => '/tmp') } -})) +import { describe, expect, it } from 'vitest' +import { finalReplayFrame } from './agent-transcript-replay-test-harness' +import { evaluateAgentStateRules } from '../../agent-state-rules/agent-state-rules-engine' // prime-agent 0.9.8 on macOS, OpenRouter, isolated HOME; 0.9.5 recorded for PR #22154 (see // each .meta.json); STA-8741. @@ -30,11 +23,17 @@ const NOT_READY = [ ] describe('Prime Agent readiness from captured bytes', () => { - describeScreenRuledAgentTranscripts({ - agent: 'prime-agent', - foregroundProcess: 'prime-agent', - ready: READY, - notReady: NOT_READY, - readyWithoutScreen: READY.map(({ name }) => name) + it.each(READY)('$name ($what) reads ready', async ({ name, cols, rows }) => { + const { ruledScreenLines } = await finalReplayFrame(name, cols, rows) + expect( + evaluateAgentStateRules('prime-agent', { readScreenLines: () => ruledScreenLines })?.state + ).toBe('idle') + }) + + it.each(NOT_READY)('$name ($what) does not read ready', async ({ name, cols, rows }) => { + const { ruledScreenLines } = await finalReplayFrame(name, cols, rows) + expect( + evaluateAgentStateRules('prime-agent', { readScreenLines: () => ruledScreenLines })?.state + ).not.toBe('idle') }) }) diff --git a/src/main/runtime/terminal/wait/screen-ruled-agent-transcript-suite.ts b/src/main/runtime/terminal/wait/screen-ruled-agent-transcript-suite.ts deleted file mode 100644 index 43f4d71c54d..00000000000 --- a/src/main/runtime/terminal/wait/screen-ruled-agent-transcript-suite.ts +++ /dev/null @@ -1,249 +0,0 @@ -// One replay suite for every agent whose readiness its live screen decides (terminal-wait-detection.ts). -import { describe, expect, it, vi } from 'vitest' -import { - createTranscriptPane, - TRANSCRIPT_PANE_PTY_ID, - waitForTranscriptIdle -} from '../../orca-runtime-tests/agent-transcript-pane-test-harness' -import { - finalReadProjection, - finalReplayFrame, - readRuntimeFixture, - type TranscriptReplayResize -} from './agent-transcript-replay-test-harness' -import { isQuietReadyScreenBody } from './terminal-wait-detection' -import { evaluateAgentStateRules } from '../../agent-state-rules/agent-state-rules-engine' -import type { TuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent' - -export type ScreenRuledFixture = { name: string; cols: number; rows: number; what: string } - -export type ScreenRuledAgentSuite = { - agent: TuiAgent - foregroundProcess: string - ready: readonly ScreenRuledFixture[] - notReady: readonly ScreenRuledFixture[] - /** Ready recordings the text rules or the quiet-process lane settle with no screen. */ - readyWithoutScreen: readonly string[] -} - -// Mismatched grids garble cursor-addressed chrome (#23475 review). -const MISMATCHED_GRIDS = [ - [80, 24], - [100, 30], - [60, 20], - [150, 50] -] as const - -// 8s covers 3s quiescence and the 2s poll; 5s refusal allows both plus at least one poll tick. -const READY_TIMEOUT_MS = 8_000 -const REFUSAL_TIMEOUT_MS = 5_000 -// Allow for multi-MB animated recordings taking seconds in onPtyData. -const PANE_SETUP_SLACK_MS = 30_000 - -function chunkCount(name: string): number { - return Math.ceil(readRuntimeFixture(name).length / 64) -} - -/** Whether the agent's own rules read `screenLines` as an idle composer. */ -export function readsIdleComposer(agent: TuiAgent, screenLines: readonly string[]): boolean { - return evaluateAgentStateRules(agent, { readScreenLines: () => screenLines })?.state === 'idle' -} - -export function describeScreenRuledAgentTranscripts(suite: ScreenRuledAgentSuite): void { - const { agent } = suite - const rule = (screenLines: readonly string[]): boolean => readsIdleComposer(agent, screenLines) - const [firstReady] = suite.ready - if (!firstReady) { - throw new Error(`${agent}: a suite needs a ready recording`) - } - - describe('the screen rule on the final recorded frame', () => { - it.each(suite.ready)('$name ($what) reads ready', async ({ name, cols, rows }) => { - const { ruledScreenLines } = await finalReplayFrame(name, cols, rows) - expect(rule(ruledScreenLines)).toBe(true) - }) - it.each(suite.notReady)('$name ($what) does not', async ({ name, cols, rows }) => { - const { ruledScreenLines } = await finalReplayFrame(name, cols, rows) - expect(rule(ruledScreenLines)).toBe(false) - }) - }) - - describe('fails safe on a garbled grid', () => { - it.each(suite.notReady)('$name never reads ready', async ({ name, cols, rows }) => { - const resize: TranscriptReplayResize = { - atChunk: Math.floor(chunkCount(name) / 2), - cols: 100, - rows: 30 - } - const frames = [ - ...(await Promise.all(MISMATCHED_GRIDS.map(([c, r]) => finalReplayFrame(name, c, r)))), - await finalReplayFrame(name, cols, rows, resize) - ] - for (const { ruledScreenLines, waitText } of frames) { - expect(isQuietReadyScreenBody(waitText, agent, () => ruledScreenLines)).toBe(false) - } - }) - }) - - describe('through the runtime', () => { - async function pane(fixture: ScreenRuledFixture) { - const created = await createTranscriptPane({ - paneTitle: 'Terminal', - foregroundProcess: suite.foregroundProcess, - launchAgent: agent, - data: readRuntimeFixture(fixture.name), - size: { cols: fixture.cols, rows: fixture.rows } - }) - // Drain emulator parsing: one multi-MB chunk can outlive the output quiet clock unlike a live PTY's silence. - await created.runtime.readTerminal(created.handle, { screen: true }) - return created - } - - it.each(suite.ready)( - '$name: a tui-idle wait settles once the pane is quiet', - async (fixture) => { - const { runtime, handle } = await pane(fixture) - await expect( - waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS) - ).resolves.toMatchObject({ - condition: 'tui-idle', - satisfied: true - }) - }, - READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS - ) - - it.each(suite.notReady)( - '$name: a tui-idle wait does not settle ready', - async (fixture) => { - const { runtime, handle } = await pane(fixture) - const result = await waitForTranscriptIdle({ runtime, handle }, REFUSAL_TIMEOUT_MS).catch( - (error: unknown) => ({ satisfied: false, error: String(error) }) - ) - expect(result.satisfied).toBe(false) - }, - REFUSAL_TIMEOUT_MS + PANE_SETUP_SLACK_MS - ) - - // Why 100x30: a grid no recording was painted for. The PTY reports its real size. - it.each(suite.ready)( - '$name: on a grid the PTY does not have, the lanes the screen would shut decide', - async (fixture) => { - const options = { - paneTitle: 'Terminal', - foregroundProcess: suite.foregroundProcess, - launchAgent: agent, - data: readRuntimeFixture(fixture.name), - size: { cols: 100, rows: 30 } - } - const { runtime, handle } = await createTranscriptPane(options) - await runtime.readTerminal(handle, { screen: true }) - options.size = { cols: fixture.cols, rows: fixture.rows } - const result = await waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS).catch( - () => ({ satisfied: false }) - ) - expect(result.satisfied).toBe(suite.readyWithoutScreen.includes(fixture.name)) - }, - READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS - ) - - // Reattach reflows without a TUI repaint. - it( - 'on a grid reflowed to the PTY size without a repaint, the lanes the screen would shut decide', - async () => { - const options = { - paneTitle: 'Terminal', - foregroundProcess: suite.foregroundProcess, - launchAgent: agent, - data: readRuntimeFixture(firstReady.name), - size: { cols: 100, rows: 30 } - } - const { runtime, handle } = await createTranscriptPane(options) - options.size = { cols: firstReady.cols, rows: firstReady.rows } - runtime.reflowHeadlessTerminalToPtyGrid( - TRANSCRIPT_PANE_PTY_ID, - firstReady.cols, - firstReady.rows - ) - // Echoing the already-reflowed size sends no SIGWINCH, so the TUI never repaints. - runtime.onExternalPtyResize(TRANSCRIPT_PANE_PTY_ID, firstReady.cols, firstReady.rows) - await runtime.readTerminal(handle, { screen: true }) - const result = await waitForTranscriptIdle({ runtime, handle }, READY_TIMEOUT_MS).catch( - () => ({ satisfied: false }) - ) - expect(result.satisfied).toBe(suite.readyWithoutScreen.includes(firstReady.name)) - }, - READY_TIMEOUT_MS + PANE_SETUP_SLACK_MS - ) - - // Why no bytes: a restored pane has no output clock, so the provider's screen is all it has. - // Why the read projection: it is what the probe's screen read returns, draft blanked. - async function probedVerdict(fixture: ScreenRuledFixture): Promise { - const { lines, draft } = await finalReadProjection(fixture.name, fixture.cols, fixture.rows) - const { runtime, handle } = await createTranscriptPane({ - paneTitle: 'Terminal', - foregroundProcess: suite.foregroundProcess, - launchAgent: agent, - data: '' - }) - const read = vi.spyOn(runtime, 'readTerminal').mockResolvedValue({ - handle, - status: 'running', - tail: lines, - ...(draft === undefined ? {} : { draft }), - truncated: false, - nextCursor: null, - source: 'screen' - }) - const result = await waitForTranscriptIdle({ runtime, handle }, 2_500).catch(() => ({ - satisfied: false - })) - // Presence precondition: the visible-screen probe actually ran. - expect(read).toHaveBeenCalled() - return result.satisfied === true - } - - it.each(suite.ready)( - '$name: a restored pane settles from a visible-screen read', - async (fixture) => { - expect(await probedVerdict(fixture)).toBe(true) - }, - 15_000 - ) - - it.each(suite.notReady)( - '$name: a restored pane does not settle from a visible-screen read', - async (fixture) => { - expect(await probedVerdict(fixture)).toBe(false) - }, - 15_000 - ) - - it('waits for quiet again after a ready pane repaints', async () => { - const { runtime, handle } = await pane(firstReady) - vi.useFakeTimers({ - toFake: ['Date', 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval'] - }) - try { - const settled = vi.fn() - const waiting = runtime.waitForTerminal(handle, { - condition: 'tui-idle', - timeoutMs: READY_TIMEOUT_MS - }) - void waiting.then(settled, () => {}) - await vi.advanceTimersByTimeAsync(2_000) - expect(settled).not.toHaveBeenCalled() - - runtime.onPtyData(TRANSCRIPT_PANE_PTY_ID, readRuntimeFixture(firstReady.name), Date.now()) - await runtime.readTerminal(handle, { screen: true }) - await vi.advanceTimersByTimeAsync(2_000) - expect(settled).not.toHaveBeenCalled() - - await vi.advanceTimersByTimeAsync(2_000) - await expect(waiting).resolves.toMatchObject({ condition: 'tui-idle', satisfied: true }) - } finally { - vi.useRealTimers() - } - }) - }) -} diff --git a/src/shared/nested-repo-telemetry-schema.test.ts b/src/shared/nested-repo-telemetry-schema.test.ts deleted file mode 100644 index 0caf6601dcc..00000000000 --- a/src/shared/nested-repo-telemetry-schema.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { eventSchemas } from './telemetry-events' - -const attemptId = '2fbac1e3-5094-45b4-80a6-90281e6e9e09' - -describe('nested repo import telemetry schemas', () => { - it('rejects raw paths, repo names, and raw errors via .strict()', () => { - expect( - eventSchemas.add_repo_nested_scan_result.safeParse({ - attempt_id: attemptId, - surface: 'onboarding', - runtime_kind: 'local', - result: 'review_shown', - selected_path_kind: 'non_git_folder', - found_count: 1, - found_count_bucket: '1', - truncated: false, - timed_out: false, - selected_path: '/Users/alice/work/platform' - }).success - ).toBe(false) - - expect( - eventSchemas.add_repo_nested_import_action.safeParse({ - attempt_id: attemptId, - surface: 'sidebar', - runtime_kind: 'local', - action: 'import_group', - found_count: 1, - found_count_bucket: '1', - selected_count: 1, - selected_count_bucket: '1', - all_selected: true, - repo_name: 'secret-service' - }).success - ).toBe(false) - - expect( - eventSchemas.add_repo_nested_import_result.safeParse({ - attempt_id: attemptId, - surface: 'sidebar', - runtime_kind: 'ssh', - mode: 'group', - outcome: 'failed', - found_count: 1, - found_count_bucket: '1', - selected_count: 1, - selected_count_bucket: '1', - imported_count: 0, - imported_count_bucket: '0', - already_known_count: 0, - already_known_count_bucket: '0', - failed_count: 1, - failed_count_bucket: '1', - all_selected: true, - error_message: 'failed at /Users/alice/work/platform' - }).success - ).toBe(false) - }) - - it('rejects mismatched exact counts and buckets', () => { - expect( - eventSchemas.add_repo_nested_import_action.safeParse({ - attempt_id: attemptId, - surface: 'sidebar', - runtime_kind: 'local', - action: 'import_group', - found_count: 1, - found_count_bucket: '16+', - selected_count: 1, - selected_count_bucket: '1', - all_selected: true - }).success - ).toBe(false) - - expect( - eventSchemas.add_repo_nested_import_result.safeParse({ - attempt_id: attemptId, - surface: 'sidebar', - runtime_kind: 'local', - mode: 'group', - outcome: 'success', - found_count: 2, - found_count_bucket: '2-3', - selected_count: 2, - selected_count_bucket: '2-3', - imported_count: 2, - imported_count_bucket: '16+', - already_known_count: 0, - already_known_count_bucket: '0', - failed_count: 0, - failed_count_bucket: '0', - all_selected: true - }).success - ).toBe(false) - }) -}) diff --git a/src/shared/profile-state-telemetry-schema.spec.ts b/src/shared/profile-state-telemetry-schema.spec.ts deleted file mode 100644 index 57718096e8d..00000000000 --- a/src/shared/profile-state-telemetry-schema.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { eventSchemas } from './telemetry-events' - -describe('profile state authority telemetry', () => { - it('rejects paths or other unbounded diagnostic fields', () => { - expect( - eventSchemas.profile_state_authority_selected.safeParse({ - backend: 'sqlite', - classification: 'sqlite-only', - authority_mode: 'sqlite-established', - runtime: 'desktop', - migrated: false, - database_path: '/private/profile.sqlite' - }).success - ).toBe(false) - }) -}) diff --git a/src/shared/telemetry-payload-contracts.test.ts b/src/shared/telemetry-payload-contracts.test.ts index 9a4cf8ea3e8..c83bd727095 100644 --- a/src/shared/telemetry-payload-contracts.test.ts +++ b/src/shared/telemetry-payload-contracts.test.ts @@ -1,4 +1,3 @@ import './telemetry-orca-cli-feature-tip.spec' import './telemetry-events-feature-education.spec' import './telemetry-feature-wall-events.spec' -import './profile-state-telemetry-schema.spec'