// Schema round-trip coverage for the event map. Fail-closed invariants that // must hold: agent_error is enum-only (error_message / error_stack rejected // by `.strict()`), unknown enum values fail, and any well-formed payload // round-trips without coercion. import { describe, expect, it } from 'vitest' import { addRepoSetupStepActionSchema, AGENT_KIND_VALUES, agentKindSchema, errorClassSchema, eventSchemas, isCohortExtendedEvent, SETTINGS_CHANGED_WHITELIST, settingsChangedKeySchema } from './telemetry-events' import { FEATURE_INTERACTION_IDS, getFeatureInteractionCategory } from './feature-interactions' import { appStarSourceSchema } from './gh-star-source' describe('feature_interaction_usage_bucket_reached schema', () => { it('accepts a valid bucket payload', () => { const parsed = eventSchemas.feature_interaction_usage_bucket_reached.safeParse({ feature_id: 'browser-tab-created', feature_category: 'browser', count_bucket: 'count_3_4', bucket_source: 'crossed_now', nth_repo_added: 2 }) expect(parsed.success).toBe(true) }) it('is in the runtime cohort-injection roster', () => { expect(isCohortExtendedEvent('feature_interaction_usage_bucket_reached')).toBe(true) }) it('keeps the feature id enum in sync with the catalog', () => { const schema = eventSchemas.feature_interaction_usage_bucket_reached for (const feature_id of FEATURE_INTERACTION_IDS) { expect( schema.safeParse({ feature_id, feature_category: getFeatureInteractionCategory(feature_id), count_bucket: 'count_1', bucket_source: 'crossed_now' }).success ).toBe(true) } }) it('rejects unknown enum values and mismatched categories', () => { const valid = { feature_id: 'github-tasks', feature_category: 'task_management', count_bucket: 'count_1', bucket_source: 'observed_existing' } expect( eventSchemas.feature_interaction_usage_bucket_reached.safeParse({ ...valid, feature_id: 'unknown-feature' }).success ).toBe(false) expect( eventSchemas.feature_interaction_usage_bucket_reached.safeParse({ ...valid, feature_category: 'browser' }).success ).toBe(false) expect( eventSchemas.feature_interaction_usage_bucket_reached.safeParse({ ...valid, count_bucket: 'count_4' }).success ).toBe(false) expect( eventSchemas.feature_interaction_usage_bucket_reached.safeParse({ ...valid, bucket_source: 'renderer' }).success ).toBe(false) }) it('rejects raw privacy fields via .strict()', () => { const rawFields = [ 'prompt', 'command', 'path', 'repo', 'branch', 'url', 'hostname', 'error', 'text', 'query', 'result_label', 'workspace_name', 'setting_name', 'target_id', 'annotation_text', 'dom_snippet', 'screenshot', 'page_title', 'trusted_directory', 'trigger_x', 'trigger_y', 'focus_state', 'minimize_state', 'audio', 'transcript', 'model', 'device', 'error_detail' ] for (const field of rawFields) { const parsed = eventSchemas.feature_interaction_usage_bucket_reached.safeParse({ feature_id: 'browser-annotations-sent-to-agent', feature_category: 'browser', count_bucket: 'count_1', bucket_source: 'crossed_now', [field]: 'raw' }) expect(parsed.success).toBe(false) } }) }) describe('app_starred_orca schema', () => { it('accepts every declared app star source', () => { for (const source of appStarSourceSchema.options) { const parsed = eventSchemas.app_starred_orca.safeParse({ source }) expect(parsed.success).toBe(true) } }) it('accepts cohort context on successful app star telemetry', () => { const parsed = eventSchemas.app_starred_orca.safeParse({ source: 'settings', nth_repo_added: 2 }) expect(parsed.success).toBe(true) }) it('rejects unknown app star source values', () => { const parsed = eventSchemas.app_starred_orca.safeParse({ source: 'github_website' }) expect(parsed.success).toBe(false) }) it('rejects extra keys via .strict()', () => { const parsed = eventSchemas.app_starred_orca.safeParse({ source: 'landing', repo: 'stablyai/orca' }) expect(parsed.success).toBe(false) }) }) describe('star_nag_outcome schema', () => { const valid = { outcome: 'shown', source: 'threshold', mode: 'gh', threshold: 35, agents_since_baseline: 35, agents_since_baseline_bucket: '35-69', nth_repo_added: 2 } it('accepts a strict valid payload with cohort context', () => { expect(eventSchemas.star_nag_outcome.safeParse(valid).success).toBe(true) }) it('is in the runtime cohort-injection roster', () => { expect(isCohortExtendedEvent('star_nag_outcome')).toBe(true) }) it('accepts next_threshold only as a positive integer for deferrals', () => { expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'dismissed', next_threshold: 70 }).success ).toBe(true) expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'later', next_threshold: 70 }).success ).toBe(true) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, next_threshold: 0 }).success).toBe( false ) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, next_threshold: 1.5 }).success).toBe( false ) expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'shown', next_threshold: 70 }) .success ).toBe(false) }) it('accepts cooldown_days only for deferrals', () => { expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'later', cooldown_days: 30 }).success ).toBe(true) expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'dismissed', cooldown_days: 30 }).success ).toBe(true) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, cooldown_days: 0 }).success).toBe( false ) expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'opened_repo', cooldown_days: 30 }).success ).toBe(false) }) it('accepts new star nag action outcomes', () => { for (const outcome of [ 'star_clicked', 'direct_star_succeeded', 'direct_star_failed', 'opened_repo', 'later' ]) { expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome }).success).toBe(true) } }) it('rejects unknown outcome source mode and bucket values', () => { expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'ignored' }).success).toBe( false ) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, source: 'renderer' }).success).toBe( false ) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, mode: 'desktop' }).success).toBe( false ) expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, agents_since_baseline_bucket: '35+' }).success ).toBe(false) }) it('rejects malformed numeric fields and raw extra fields', () => { expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, threshold: -1 }).success).toBe(false) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, threshold: 1.5 }).success).toBe( false ) expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, agents_since_baseline: -1 }).success ).toBe(false) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, nth_repo_added: -1 }).success).toBe( false ) expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, error: 'gh failed' }).success).toBe( false ) expect( eventSchemas.star_nag_outcome.safeParse({ ...valid, url: 'https://github.com' }).success ).toBe(false) }) }) describe('agent_error schema', () => { it('round-trips a minimal {error_class, agent_kind} payload', () => { const parsed = eventSchemas.agent_error.safeParse({ error_class: 'unknown', agent_kind: 'claude-code' }) expect(parsed.success).toBe(true) }) // Core invariant: `.strict()` rejects raw error strings. If this test ever // flips, the analytics lane is leaking UGC — revert the offending schema // change. it('rejects error_message via .strict()', () => { const parsed = eventSchemas.agent_error.safeParse({ error_class: 'unknown', agent_kind: 'claude-code', error_message: 'boom at /Users/alice/secret/path' }) expect(parsed.success).toBe(false) }) it('rejects error_stack via .strict()', () => { const parsed = eventSchemas.agent_error.safeParse({ error_class: 'unknown', agent_kind: 'claude-code', error_stack: 'Error: boom\n at /Users/alice/...' }) expect(parsed.success).toBe(false) }) it('rejects error_name (deferred — schema is enum-only)', () => { // `error_name` was part of an earlier draft. The trimmed schema is // enum-only; if a future PR re-introduces it as additive-optional, // this test should be replaced rather than relaxed silently. const parsed = eventSchemas.agent_error.safeParse({ error_class: 'unknown', agent_kind: 'claude-code', error_name: 'BinaryNotFound' }) expect(parsed.success).toBe(false) }) it('rejects unknown error_class enum values', () => { const parsed = eventSchemas.agent_error.safeParse({ error_class: 'made_up_class', agent_kind: 'claude-code' }) expect(parsed.success).toBe(false) }) it('rejects unknown agent_kind enum values', () => { const parsed = eventSchemas.agent_error.safeParse({ error_class: 'unknown', agent_kind: 'made_up_agent' }) expect(parsed.success).toBe(false) }) }) describe('daemon_lifecycle schema', () => { it('round-trips a startup replace payload', () => { const parsed = eventSchemas.daemon_lifecycle.safeParse({ transition: 'replaced', reason: 'stale_bundle', live_session_count_bucket: '0' }) expect(parsed.success).toBe(true) }) it('round-trips a retirement payload', () => { const parsed = eventSchemas.daemon_lifecycle.safeParse({ transition: 'retired', reason: 'died_respawn', live_session_count_bucket: 'unknown' }) expect(parsed.success).toBe(true) }) // Core privacy invariant: enum-only + bucketed counts. If this flips, the lane is leaking // paths/versions/exact counts — revert the offending schema change (STA-2376). // Both union members, so neither can lose .strict() unnoticed. it('rejects raw paths, versions, and unbucketed counts via .strict()', () => { const bases = [ { transition: 'replaced', reason: 'failed_health_check', live_session_count_bucket: '2-5' }, { transition: 'retired', reason: 'died_respawn', live_session_count_bucket: 'unknown' } ] for (const base of bases) { for (const leak of [ { daemon_path: '/Users/alice/Orca.app' }, { daemon_app_version: '1.4.129' }, { live_session_count: 3 } ]) { const parsed = eventSchemas.daemon_lifecycle.safeParse({ ...base, ...leak }) expect(parsed.success).toBe(false) } // Sanity: the base itself must be valid, so the rejections above are the leak, not the base. expect(eventSchemas.daemon_lifecycle.safeParse(base).success).toBe(true) } }) it('rejects unknown reason and bucket enum values', () => { expect( eventSchemas.daemon_lifecycle.safeParse({ transition: 'replaced', reason: 'made_up_reason', live_session_count_bucket: '0' }).success ).toBe(false) expect( eventSchemas.daemon_lifecycle.safeParse({ transition: 'replaced', reason: 'stale_bundle', live_session_count_bucket: '99' }).success ).toBe(false) }) it('rejects reasons and fields that do not belong to the transition', () => { expect( eventSchemas.daemon_lifecycle.safeParse({ transition: 'replaced', reason: 'died_respawn', live_session_count_bucket: 'unknown' }).success ).toBe(false) expect( eventSchemas.daemon_lifecycle.safeParse({ transition: 'retired', reason: 'failed_health_check', live_session_count_bucket: 'unknown' }).success ).toBe(false) }) }) describe('workspace_created schema', () => { it('rejects unknown source', () => { const parsed = eventSchemas.workspace_created.safeParse({ source: 'carrier_pigeon', from_existing_branch: false }) expect(parsed.success).toBe(false) }) it('accepts a valid payload', () => { const parsed = eventSchemas.workspace_created.safeParse({ source: 'command_palette', from_existing_branch: true }) expect(parsed.success).toBe(true) }) it('rejects extra keys via .strict()', () => { const parsed = eventSchemas.workspace_created.safeParse({ source: 'command_palette', from_existing_branch: true, branch: 'refs/heads/main' // raw branch name is UGC — rejected by .strict() }) expect(parsed.success).toBe(false) }) }) describe('agent_started schema', () => { it('requires all three keys', () => { const parsed = eventSchemas.agent_started.safeParse({ agent_kind: 'claude-code', launch_source: 'sidebar' }) expect(parsed.success).toBe(false) }) }) describe('agent_prompt_sent schema', () => { it('accepts a hook-confirmed prompt-send payload with cohort context', () => { const parsed = eventSchemas.agent_prompt_sent.safeParse({ agent_kind: 'codex', launch_source: 'unknown', request_kind: 'followup', nth_repo_added: 1 }) expect(parsed.success).toBe(true) }) it('rejects prompt text via .strict()', () => { const parsed = eventSchemas.agent_prompt_sent.safeParse({ agent_kind: 'claude-code', launch_source: 'unknown', request_kind: 'followup', prompt: 'please inspect /Users/alice/private-repo' }) expect(parsed.success).toBe(false) }) }) describe('agent_hook_unattributed schema', () => { it('accepts the two bounded attribution failure reasons', () => { for (const reason of ['empty_pane_key', 'unknown_tab_id'] as const) { expect(eventSchemas.agent_hook_unattributed.safeParse({ reason }).success).toBe(true) } }) it('rejects extra payload fields via .strict()', () => { const parsed = eventSchemas.agent_hook_unattributed.safeParse({ reason: 'unknown_tab_id', pane_key: 'tab-secret:1' }) expect(parsed.success).toBe(false) }) }) describe('add_repo_setup_step_action schema', () => { it('accepts every Setup-step action declared in the schema', () => { for (const action of addRepoSetupStepActionSchema.options) { const parsed = eventSchemas.add_repo_setup_step_action.safeParse({ action }) expect(parsed.success).toBe(true) } }) it('rejects unknown action enum values', () => { const parsed = eventSchemas.add_repo_setup_step_action.safeParse({ action: 'export_to_pdf' }) expect(parsed.success).toBe(false) }) it('rejects extra keys via .strict()', () => { const parsed = eventSchemas.add_repo_setup_step_action.safeParse({ action: 'skip', repo_name: 'orca' // raw repo names are UGC — must not cross the wire }) expect(parsed.success).toBe(false) }) }) describe('add_repo_default_checkout_handoff schema', () => { it('accepts bounded handoff outcome enums', () => { const parsed = eventSchemas.add_repo_default_checkout_handoff.safeParse({ source: 'clone_url', result: 'opened_default_checkout', reason: 'detected_default_checkout', nth_repo_added: 1 }) expect(parsed.success).toBe(true) }) it('rejects raw repo/path context via .strict()', () => { const parsed = eventSchemas.add_repo_default_checkout_handoff.safeParse({ source: 'local_folder_picker', result: 'revealed_project', reason: 'no_default_checkout', repo_name: 'secret-repo', path: '/Users/alice/secret-repo' }) expect(parsed.success).toBe(false) }) }) describe('workspace_create_failed schema', () => { it('accepts a valid payload', () => { const parsed = eventSchemas.workspace_create_failed.safeParse({ source: 'sidebar', error_class: 'git_failed' }) expect(parsed.success).toBe(true) }) it('rejects unknown error_class values', () => { const parsed = eventSchemas.workspace_create_failed.safeParse({ source: 'sidebar', error_class: 'cosmic_ray' }) expect(parsed.success).toBe(false) }) // Core invariant mirroring agent_error: raw error strings never cross the // wire. If this test ever flips, the failure-rate lane is leaking UGC — // revert the offending schema change. it('rejects error_message via .strict()', () => { const parsed = eventSchemas.workspace_create_failed.safeParse({ source: 'sidebar', error_class: 'git_failed', error_message: 'fatal: cannot create work tree at /Users/alice/secret' }) expect(parsed.success).toBe(false) }) it('rejects error_stack via .strict()', () => { const parsed = eventSchemas.workspace_create_failed.safeParse({ source: 'sidebar', error_class: 'git_failed', error_stack: 'Error: cannot create work tree\n at /Users/alice/...' }) expect(parsed.success).toBe(false) }) }) describe('settings_changed schema', () => { it('accepts whitelisted setting keys', () => { for (const key of SETTINGS_CHANGED_WHITELIST) { const parsed = eventSchemas.settings_changed.safeParse({ setting_key: key, value_kind: 'bool' }) expect(parsed.success).toBe(true) } }) it('rejects non-whitelisted setting keys', () => { const parsed = eventSchemas.settings_changed.safeParse({ setting_key: 'telemetryOptIn', // deliberately excluded from the whitelist value_kind: 'bool' }) expect(parsed.success).toBe(false) }) }) describe('exported enum schemas', () => { it('agentKindSchema accepts the known product IDs', () => { for (const kind of AGENT_KIND_VALUES) { expect(agentKindSchema.safeParse(kind).success).toBe(true) } }) it('errorClassSchema rejects novel classes', () => { expect(errorClassSchema.safeParse('kernel_panic').success).toBe(false) }) it('settingsChangedKeySchema membership matches SETTINGS_CHANGED_WHITELIST', () => { for (const key of SETTINGS_CHANGED_WHITELIST) { expect(settingsChangedKeySchema.safeParse(key).success).toBe(true) } }) })