orca/src/shared/telemetry-events.test.ts

// Schema round-trip coverage for the event map. Fail-closed invariants that
// must hold: agent_error is enum-only (error_message / error_stack rejected
// by `.strict()`), unknown enum values fail, and any well-formed payload
// round-trips without coercion.

import { describe, expect, it } from 'vitest'
import { errorClassSchema, eventSchemas, isCohortExtendedEvent } from './telemetry-events'

describe('feature_interaction_usage_bucket_reached schema', () => {
  it('accepts a valid bucket payload', () => {
    const parsed = eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
      feature_id: 'browser-tab-created',
      feature_category: 'browser',
      count_bucket: 'count_3_4',
      bucket_source: 'crossed_now',
      nth_repo_added: 2
    })
    expect(parsed.success).toBe(true)
  })

  it('is in the runtime cohort-injection roster', () => {
    expect(isCohortExtendedEvent('feature_interaction_usage_bucket_reached')).toBe(true)
  })

  it('rejects unknown enum values and mismatched categories', () => {
    const valid = {
      feature_id: 'github-tasks',
      feature_category: 'task_management',
      count_bucket: 'count_1',
      bucket_source: 'observed_existing'
    }
    expect(
      eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
        ...valid,
        feature_id: 'unknown-feature'
      }).success
    ).toBe(false)
    expect(
      eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
        ...valid,
        feature_category: 'browser'
      }).success
    ).toBe(false)
    expect(
      eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
        ...valid,
        count_bucket: 'count_4'
      }).success
    ).toBe(false)
    expect(
      eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
        ...valid,
        bucket_source: 'renderer'
      }).success
    ).toBe(false)
  })

  it('rejects raw privacy fields via .strict()', () => {
    const rawFields = [
      'prompt',
      'command',
      'path',
      'repo',
      'branch',
      'url',
      'hostname',
      'error',
      'text',
      'query',
      'result_label',
      'workspace_name',
      'setting_name',
      'target_id',
      'annotation_text',
      'dom_snippet',
      'screenshot',
      'page_title',
      'trusted_directory',
      'trigger_x',
      'trigger_y',
      'focus_state',
      'minimize_state',
      'audio',
      'transcript',
      'model',
      'device',
      'error_detail'
    ]
    for (const field of rawFields) {
      const parsed = eventSchemas.feature_interaction_usage_bucket_reached.safeParse({
        feature_id: 'browser-annotations-sent-to-agent',
        feature_category: 'browser',
        count_bucket: 'count_1',
        bucket_source: 'crossed_now',
        [field]: 'raw'
      })
      expect(parsed.success).toBe(false)
    }
  })
})

describe('app_starred_orca schema', () => {
  it('accepts cohort context on successful app star telemetry', () => {
    const parsed = eventSchemas.app_starred_orca.safeParse({
      source: 'settings',
      nth_repo_added: 2
    })
    expect(parsed.success).toBe(true)
  })

  it('rejects unknown app star source values', () => {
    const parsed = eventSchemas.app_starred_orca.safeParse({
      source: 'github_website'
    })
    expect(parsed.success).toBe(false)
  })

  it('rejects extra keys via .strict()', () => {
    const parsed = eventSchemas.app_starred_orca.safeParse({
      source: 'landing',
      repo: 'stablyai/orca'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('star_nag_outcome schema', () => {
  const valid = {
    outcome: 'shown',
    source: 'threshold',
    mode: 'gh',
    threshold: 35,
    agents_since_baseline: 35,
    agents_since_baseline_bucket: '35-69',
    nth_repo_added: 2
  }

  it('accepts a strict valid payload with cohort context', () => {
    expect(eventSchemas.star_nag_outcome.safeParse(valid).success).toBe(true)
  })

  it('is in the runtime cohort-injection roster', () => {
    expect(isCohortExtendedEvent('star_nag_outcome')).toBe(true)
  })

  it('accepts next_threshold only as a positive integer for deferrals', () => {
    expect(
      eventSchemas.star_nag_outcome.safeParse({
        ...valid,
        outcome: 'dismissed',
        next_threshold: 70
      }).success
    ).toBe(true)
    expect(
      eventSchemas.star_nag_outcome.safeParse({
        ...valid,
        outcome: 'later',
        next_threshold: 70
      }).success
    ).toBe(true)
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, next_threshold: 0 }).success).toBe(
      false
    )
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, next_threshold: 1.5 }).success).toBe(
      false
    )
    expect(
      eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'shown', next_threshold: 70 })
        .success
    ).toBe(false)
  })

  it('accepts cooldown_days only for deferrals', () => {
    expect(
      eventSchemas.star_nag_outcome.safeParse({
        ...valid,
        outcome: 'later',
        cooldown_days: 30
      }).success
    ).toBe(true)
    expect(
      eventSchemas.star_nag_outcome.safeParse({
        ...valid,
        outcome: 'dismissed',
        cooldown_days: 30
      }).success
    ).toBe(true)
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, cooldown_days: 0 }).success).toBe(
      false
    )
    expect(
      eventSchemas.star_nag_outcome.safeParse({
        ...valid,
        outcome: 'opened_repo',
        cooldown_days: 30
      }).success
    ).toBe(false)
  })

  it('accepts new star nag action outcomes', () => {
    for (const outcome of [
      'star_clicked',
      'direct_star_succeeded',
      'direct_star_failed',
      'opened_repo',
      'later'
    ]) {
      expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome }).success).toBe(true)
    }
  })

  it('rejects unknown outcome source mode and bucket values', () => {
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, outcome: 'ignored' }).success).toBe(
      false
    )
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, source: 'renderer' }).success).toBe(
      false
    )
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, mode: 'desktop' }).success).toBe(
      false
    )
    expect(
      eventSchemas.star_nag_outcome.safeParse({
        ...valid,
        agents_since_baseline_bucket: '35+'
      }).success
    ).toBe(false)
  })

  it('rejects malformed numeric fields and raw extra fields', () => {
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, threshold: -1 }).success).toBe(false)
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, threshold: 1.5 }).success).toBe(
      false
    )
    expect(
      eventSchemas.star_nag_outcome.safeParse({ ...valid, agents_since_baseline: -1 }).success
    ).toBe(false)
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, nth_repo_added: -1 }).success).toBe(
      false
    )
    expect(eventSchemas.star_nag_outcome.safeParse({ ...valid, error: 'gh failed' }).success).toBe(
      false
    )
    expect(
      eventSchemas.star_nag_outcome.safeParse({ ...valid, url: 'https://github.com' }).success
    ).toBe(false)
  })
})

describe('agent_error schema', () => {
  it('round-trips a minimal {error_class, agent_kind} payload', () => {
    const parsed = eventSchemas.agent_error.safeParse({
      error_class: 'unknown',
      agent_kind: 'claude-code'
    })
    expect(parsed.success).toBe(true)
  })

  // Core invariant: `.strict()` rejects raw error strings. If this test ever
  // flips, the analytics lane is leaking UGC — revert the offending schema
  // change.
  it('rejects error_message via .strict()', () => {
    const parsed = eventSchemas.agent_error.safeParse({
      error_class: 'unknown',
      agent_kind: 'claude-code',
      error_message: 'boom at /Users/alice/secret/path'
    })
    expect(parsed.success).toBe(false)
  })

  it('rejects error_stack via .strict()', () => {
    const parsed = eventSchemas.agent_error.safeParse({
      error_class: 'unknown',
      agent_kind: 'claude-code',
      error_stack: 'Error: boom\n    at /Users/alice/...'
    })
    expect(parsed.success).toBe(false)
  })

  it('rejects error_name (deferred — schema is enum-only)', () => {
    // `error_name` was part of an earlier draft. The trimmed schema is
    // enum-only; if a future PR re-introduces it as additive-optional,
    // this test should be replaced rather than relaxed silently.
    const parsed = eventSchemas.agent_error.safeParse({
      error_class: 'unknown',
      agent_kind: 'claude-code',
      error_name: 'BinaryNotFound'
    })
    expect(parsed.success).toBe(false)
  })

  it('rejects unknown error_class enum values', () => {
    const parsed = eventSchemas.agent_error.safeParse({
      error_class: 'made_up_class',
      agent_kind: 'claude-code'
    })
    expect(parsed.success).toBe(false)
  })

  it('rejects unknown agent_kind enum values', () => {
    const parsed = eventSchemas.agent_error.safeParse({
      error_class: 'unknown',
      agent_kind: 'made_up_agent'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('daemon_lifecycle schema', () => {
  it('round-trips a startup replace payload', () => {
    const parsed = eventSchemas.daemon_lifecycle.safeParse({
      transition: 'replaced',
      reason: 'stale_bundle',
      live_session_count_bucket: '0'
    })
    expect(parsed.success).toBe(true)
  })

  it('round-trips a retirement payload', () => {
    const parsed = eventSchemas.daemon_lifecycle.safeParse({
      transition: 'retired',
      reason: 'died_respawn',
      live_session_count_bucket: 'unknown'
    })
    expect(parsed.success).toBe(true)
  })

  // Core privacy invariant: enum-only + bucketed counts. If this flips, the lane is leaking
  // paths/versions/exact counts — revert the offending schema change (STA-2376).
  // Both union members, so neither can lose .strict() unnoticed.
  it('rejects raw paths, versions, and unbucketed counts via .strict()', () => {
    const bases = [
      { transition: 'replaced', reason: 'failed_health_check', live_session_count_bucket: '2-5' },
      { transition: 'retired', reason: 'died_respawn', live_session_count_bucket: 'unknown' }
    ]
    for (const base of bases) {
      for (const leak of [
        { daemon_path: '/Users/alice/Orca.app' },
        { daemon_app_version: '1.4.129' },
        { live_session_count: 3 }
      ]) {
        const parsed = eventSchemas.daemon_lifecycle.safeParse({ ...base, ...leak })
        expect(parsed.success).toBe(false)
      }
      // Sanity: the base itself must be valid, so the rejections above are the leak, not the base.
      expect(eventSchemas.daemon_lifecycle.safeParse(base).success).toBe(true)
    }
  })

  it('rejects unknown reason and bucket enum values', () => {
    expect(
      eventSchemas.daemon_lifecycle.safeParse({
        transition: 'replaced',
        reason: 'made_up_reason',
        live_session_count_bucket: '0'
      }).success
    ).toBe(false)
    expect(
      eventSchemas.daemon_lifecycle.safeParse({
        transition: 'replaced',
        reason: 'stale_bundle',
        live_session_count_bucket: '99'
      }).success
    ).toBe(false)
  })

  it('rejects reasons and fields that do not belong to the transition', () => {
    expect(
      eventSchemas.daemon_lifecycle.safeParse({
        transition: 'replaced',
        reason: 'died_respawn',
        live_session_count_bucket: 'unknown'
      }).success
    ).toBe(false)
    expect(
      eventSchemas.daemon_lifecycle.safeParse({
        transition: 'retired',
        reason: 'failed_health_check',
        live_session_count_bucket: 'unknown'
      }).success
    ).toBe(false)
  })
})

describe('workspace_created schema', () => {
  it('rejects unknown source', () => {
    const parsed = eventSchemas.workspace_created.safeParse({
      source: 'carrier_pigeon',
      from_existing_branch: false
    })
    expect(parsed.success).toBe(false)
  })

  it('accepts a valid payload', () => {
    const parsed = eventSchemas.workspace_created.safeParse({
      source: 'command_palette',
      from_existing_branch: true
    })
    expect(parsed.success).toBe(true)
  })

  it('rejects extra keys via .strict()', () => {
    const parsed = eventSchemas.workspace_created.safeParse({
      source: 'command_palette',
      from_existing_branch: true,
      branch: 'refs/heads/main' // raw branch name is UGC — rejected by .strict()
    })
    expect(parsed.success).toBe(false)
  })
})

describe('agent_started schema', () => {
  it('requires all three keys', () => {
    const parsed = eventSchemas.agent_started.safeParse({
      agent_kind: 'claude-code',
      launch_source: 'sidebar'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('agent_prompt_sent schema', () => {
  it('accepts a hook-confirmed prompt-send payload with cohort context', () => {
    const parsed = eventSchemas.agent_prompt_sent.safeParse({
      agent_kind: 'codex',
      launch_source: 'unknown',
      request_kind: 'followup',
      nth_repo_added: 1
    })
    expect(parsed.success).toBe(true)
  })

  it('rejects prompt text via .strict()', () => {
    const parsed = eventSchemas.agent_prompt_sent.safeParse({
      agent_kind: 'claude-code',
      launch_source: 'unknown',
      request_kind: 'followup',
      prompt: 'please inspect /Users/alice/private-repo'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('agent_hook_unattributed schema', () => {
  it('accepts the two bounded attribution failure reasons', () => {
    for (const reason of ['empty_pane_key', 'unknown_tab_id'] as const) {
      expect(eventSchemas.agent_hook_unattributed.safeParse({ reason }).success).toBe(true)
    }
  })

  it('rejects extra payload fields via .strict()', () => {
    const parsed = eventSchemas.agent_hook_unattributed.safeParse({
      reason: 'unknown_tab_id',
      pane_key: 'tab-secret:1'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('add_repo_setup_step_action schema', () => {
  it('rejects unknown action enum values', () => {
    const parsed = eventSchemas.add_repo_setup_step_action.safeParse({
      action: 'export_to_pdf'
    })
    expect(parsed.success).toBe(false)
  })

  it('rejects extra keys via .strict()', () => {
    const parsed = eventSchemas.add_repo_setup_step_action.safeParse({
      action: 'skip',
      repo_name: 'orca' // raw repo names are UGC — must not cross the wire
    })
    expect(parsed.success).toBe(false)
  })
})

describe('add_repo_default_checkout_handoff schema', () => {
  it('accepts bounded handoff outcome enums', () => {
    const parsed = eventSchemas.add_repo_default_checkout_handoff.safeParse({
      source: 'clone_url',
      result: 'opened_default_checkout',
      reason: 'detected_default_checkout',
      nth_repo_added: 1
    })
    expect(parsed.success).toBe(true)
  })

  it('rejects raw repo/path context via .strict()', () => {
    const parsed = eventSchemas.add_repo_default_checkout_handoff.safeParse({
      source: 'local_folder_picker',
      result: 'revealed_project',
      reason: 'no_default_checkout',
      repo_name: 'secret-repo',
      path: '/Users/alice/secret-repo'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('workspace_create_failed schema', () => {
  it('accepts a valid payload', () => {
    const parsed = eventSchemas.workspace_create_failed.safeParse({
      source: 'sidebar',
      error_class: 'git_failed'
    })
    expect(parsed.success).toBe(true)
  })

  it('rejects unknown error_class values', () => {
    const parsed = eventSchemas.workspace_create_failed.safeParse({
      source: 'sidebar',
      error_class: 'cosmic_ray'
    })
    expect(parsed.success).toBe(false)
  })

  // Core invariant mirroring agent_error: raw error strings never cross the
  // wire. If this test ever flips, the failure-rate lane is leaking UGC —
  // revert the offending schema change.
  it('rejects error_message via .strict()', () => {
    const parsed = eventSchemas.workspace_create_failed.safeParse({
      source: 'sidebar',
      error_class: 'git_failed',
      error_message: 'fatal: cannot create work tree at /Users/alice/secret'
    })
    expect(parsed.success).toBe(false)
  })

  it('rejects error_stack via .strict()', () => {
    const parsed = eventSchemas.workspace_create_failed.safeParse({
      source: 'sidebar',
      error_class: 'git_failed',
      error_stack: 'Error: cannot create work tree\n    at /Users/alice/...'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('settings_changed schema', () => {
  it('accepts the Chat UI switch as a boolean adoption signal', () => {
    expect(
      eventSchemas.settings_changed.safeParse({
        setting_key: 'experimentalNativeChat',
        value_kind: 'bool'
      }).success
    ).toBe(true)
  })

  it.each(['experimentalStructuredNativeChat', 'openAgentTabsInChatByDefault'])(
    'rejects the retired %s key',
    (settingKey) => {
      expect(
        eventSchemas.settings_changed.safeParse({ setting_key: settingKey, value_kind: 'bool' })
          .success
      ).toBe(false)
    }
  )

  it('rejects non-whitelisted setting keys', () => {
    const parsed = eventSchemas.settings_changed.safeParse({
      setting_key: 'telemetryOptIn', // deliberately excluded from the whitelist
      value_kind: 'bool'
    })
    expect(parsed.success).toBe(false)
  })
})

describe('exported enum schemas', () => {
  it('errorClassSchema rejects novel classes', () => {
    expect(errorClassSchema.safeParse('kernel_panic').success).toBe(false)
  })
})

describe('remote_outbound_budget_close schema', () => {
  it('round-trips every emitter', () => {
    for (const emitter of ['size', 'queue']) {
      expect(eventSchemas.remote_outbound_budget_close.safeParse({ emitter }).success).toBe(true)
    }
  })

  it('rejects an unknown emitter and any payload-describing extra key', () => {
    expect(eventSchemas.remote_outbound_budget_close.safeParse({ emitter: 'other' }).success).toBe(
      false
    )
    expect(
      eventSchemas.remote_outbound_budget_close.safeParse({
        emitter: 'size',
        byte_length: 4194305
      }).success
    ).toBe(false)
  })
})