From 4eccb8416772f420ecae611d74aff750eaaf2028 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:08:13 -0700 Subject: [PATCH] Chat UI launch (#36891) * feat(telemetry): A/B holdout for the Chat UI launch tip Freeze a 50/50 chat_ui_launch_tip_v1 cohort for profiles still due the tip on official builds with telemetry not disabled; control never sees the tip. Emit chat_ui_feature_tip_exposure in both arms and agent_default_surface_changed from the tip and from Settings -> Agents. Co-Authored-By: Claude * fix(telemetry): hold the Chat UI launch tip until the telemetry banner resolves Enroll only when consent is enabled. While the banner is pending, the tip is skipped for that launch without being marked seen, so its exposure is not spent unmeasured and the arm cannot influence the consent choice. Co-Authored-By: Claude * test(telemetry): document best-effort Chat UI tip exposure on rejected sends Co-Authored-By: Claude * test(telemetry): pass the exposure test through a hoisted mock instead of in-test importActual Co-Authored-By: Claude * test(telemetry): justify the fake Store assertion in the telemetry harness Co-Authored-By: Claude --------- Co-authored-by: Claude --- src/main/ipc/onboarding.ts | 4 +- .../chat-ui-launch-tip-cohort.test.ts | 86 ++++++++++++ .../telemetry/chat-ui-launch-tip-cohort.ts | 49 +++++++ src/main/telemetry/client-test-harness.ts | 3 + src/main/telemetry/client.ts | 3 + .../use-onboarding-and-feature-tips.test.tsx | 131 ++++++++++++++++++ .../use-onboarding-and-feature-tips.ts | 11 ++ .../feature-tips/ChatUiHereTip.test.tsx | 8 +- .../feature-tips/FeatureTipsModal.tsx | 9 ++ .../feature-tips/feature-tip-telemetry.ts | 37 +++++ .../AgentDefaultSurfaceSection.test.tsx | 34 +++++ .../settings/AgentDefaultSurfaceSection.tsx | 17 ++- src/shared/telemetry-event-classification.ts | 5 +- src/shared/telemetry-event-registry.ts | 4 + .../telemetry-native-feature-event-schemas.ts | 18 +++ src/shared/telemetry-settings-types.ts | 2 + 16 files changed, 417 insertions(+), 4 deletions(-) create mode 100644 src/main/telemetry/chat-ui-launch-tip-cohort.test.ts create mode 100644 src/main/telemetry/chat-ui-launch-tip-cohort.ts create mode 100644 src/renderer/src/app-shell/use-onboarding-and-feature-tips.test.tsx diff --git a/src/main/ipc/onboarding.ts b/src/main/ipc/onboarding.ts index 6582140c15b..6438e56f0ca 100644 --- a/src/main/ipc/onboarding.ts +++ b/src/main/ipc/onboarding.ts @@ -2,6 +2,7 @@ import { ipcMain } from 'electron' import { sanitizeOnboardingUpdate, type Store } from '../persistence' import type { OnboardingState } from '../../shared/onboarding-state-types' import type { ChatUiLaunchTipVariant } from '../../shared/native-chat/chat-ui-launch-tip-audience' +import { isChatUiLaunchTipDeferredForConsent } from '../telemetry/chat-ui-launch-tip-cohort' export function registerOnboardingHandlers(store: Store): void { ipcMain.removeHandler('onboarding:get') @@ -13,7 +14,8 @@ export function registerOnboardingHandlers(store: Store): void { ipcMain.handle('onboarding:update', (_event, updates: unknown): OnboardingState => { return store.updateOnboarding(sanitizeOnboardingUpdate(updates)) }) + // 'none' skips the tip this launch without marking it seen, so it returns once consent resolves. ipcMain.handle('onboarding:getChatUiLaunchTipVariant', (): ChatUiLaunchTipVariant => - store.getChatUiLaunchTipVariant() + isChatUiLaunchTipDeferredForConsent() ? 'none' : store.getChatUiLaunchTipVariant() ) } diff --git a/src/main/telemetry/chat-ui-launch-tip-cohort.test.ts b/src/main/telemetry/chat-ui-launch-tip-cohort.test.ts new file mode 100644 index 00000000000..ed9dcd7b836 --- /dev/null +++ b/src/main/telemetry/chat-ui-launch-tip-cohort.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ChatUiLaunchTipVariant } from '../../shared/native-chat/chat-ui-launch-tip-audience' +import type { TelemetrySettings } from '../../shared/telemetry-settings-types' +import type { Store } from '../persistence' +import { + ensureChatUiLaunchTipCohort, + isChatUiLaunchTipDeferredForConsent +} from './chat-ui-launch-tip-cohort' +import { makeFakeSettings, restoreConsentEnv, stashAndClearConsentEnv } from './client-test-harness' + +function makeStore( + telemetry: Partial, + variant: ChatUiLaunchTipVariant = 'chat-ui-off', + seen: string[] = [] +): Store { + const settings = makeFakeSettings({ + optedIn: true, + installId: 'install', + existedBeforeTelemetryRelease: true, + ...telemetry + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: supplies every Store method the cohort assignment reads or writes. + return { + getSettings: () => settings, + getUI: () => ({ featureTipsSeenIds: seen }), + getChatUiLaunchTipVariant: () => variant, + updateSettings: vi.fn((updates: { telemetry: TelemetrySettings }) => { + settings.telemetry = updates.telemetry + }) + } as unknown as Store +} + +describe('ensureChatUiLaunchTipCohort', () => { + let env: Record + beforeEach(() => { + env = stashAndClearConsentEnv() + }) + afterEach(() => { + restoreConsentEnv(env) + }) + + it('enrolls a profile still due the tip once and keeps the frozen cohort', () => { + const store = makeStore({}) + const cohort = ensureChatUiLaunchTipCohort(store) + expect(['control', 'treatment']).toContain(cohort) + expect(store.getSettings().telemetry?.chatUiLaunchTipCohort).toBe(cohort) + expect(store.getSettings().telemetry?.installId).toBe('install') + expect(ensureChatUiLaunchTipCohort(store)).toBe(cohort) + expect(store.updateSettings).toHaveBeenCalledTimes(1) + }) + + it('holds the tip unenrolled while the banner is pending, then enrolls after acknowledgement', () => { + const pending = makeStore({ optedIn: null }) + expect(ensureChatUiLaunchTipCohort(pending)).toBeUndefined() + expect(isChatUiLaunchTipDeferredForConsent()).toBe(true) + expect(pending.updateSettings).not.toHaveBeenCalled() + + // Acknowledging the banner persists optedIn=true; the next launch enrolls the still-unseen tip. + pending.getSettings().telemetry!.optedIn = true + const cohort = ensureChatUiLaunchTipCohort(pending) + expect(['control', 'treatment']).toContain(cohort) + expect(isChatUiLaunchTipDeferredForConsent()).toBe(false) + expect(pending.getSettings().telemetry?.chatUiLaunchTipCohort).toBe(cohort) + }) + + it('releases a pending profile unenrolled after the banner is declined, so the tip shows as before', () => { + const pending = makeStore({ optedIn: null }) + ensureChatUiLaunchTipCohort(pending) + expect(isChatUiLaunchTipDeferredForConsent()).toBe(true) + + pending.getSettings().telemetry!.optedIn = false + expect(ensureChatUiLaunchTipCohort(pending)).toBeUndefined() + expect(isChatUiLaunchTipDeferredForConsent()).toBe(false) + expect(pending.updateSettings).not.toHaveBeenCalled() + }) + + it.each([ + ['outside the audience', makeStore({}, 'none')], + ['already shown the tip', makeStore({}, 'chat-ui-off', ['chat-ui-here'])], + ['opted out', makeStore({ optedIn: false })] + ])('leaves a profile %s unenrolled, so it keeps today’s behaviour', (_label, store) => { + expect(ensureChatUiLaunchTipCohort(store)).toBeUndefined() + expect(isChatUiLaunchTipDeferredForConsent()).toBe(false) + expect(store.updateSettings).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/telemetry/chat-ui-launch-tip-cohort.ts b/src/main/telemetry/chat-ui-launch-tip-cohort.ts new file mode 100644 index 00000000000..ac2e2dd542f --- /dev/null +++ b/src/main/telemetry/chat-ui-launch-tip-cohort.ts @@ -0,0 +1,49 @@ +import { randomInt } from 'node:crypto' +import type { Store } from '../persistence' +import { resolveConsent } from './consent' + +export type ChatUiLaunchTipCohort = 'control' | 'treatment' + +// Decided once per launch with the cohort, so the tip and its enrollment read the same consent; contributor builds never set it. +let deferredForConsent = false + +/** + * Enrolls once, only profiles still due the tip whose events can be measured. While the telemetry + * banner is unresolved the tip waits, so the exposure is never spent unmeasured; opted-out and + * contributor builds stay unenrolled and see the tip as before. + */ +export function ensureChatUiLaunchTipCohort(store: Store): ChatUiLaunchTipCohort | undefined { + deferredForConsent = false + try { + const settings = store.getSettings() + const telemetry = settings.telemetry + const saved = telemetry?.chatUiLaunchTipCohort + if (saved === 'control' || saved === 'treatment') { + return saved + } + if ( + !telemetry || + store.getChatUiLaunchTipVariant() === 'none' || + store.getUI().featureTipsSeenIds?.includes('chat-ui-here') + ) { + return undefined + } + const consent = resolveConsent(settings).effective + if (consent !== 'enabled') { + deferredForConsent = consent === 'pending_banner' + return undefined + } + const cohort: ChatUiLaunchTipCohort = randomInt(2) === 0 ? 'control' : 'treatment' + store.updateSettings({ telemetry: { ...telemetry, chatUiLaunchTipCohort: cohort } }) + return cohort + } catch (err) { + // Startup must not fail over an experiment; unenrolled profiles see the tip as before. + console.warn('[telemetry] chat UI launch tip cohort assignment failed', err) + return undefined + } +} + +/** True when this launch holds the tip back until the telemetry banner is resolved. */ +export function isChatUiLaunchTipDeferredForConsent(): boolean { + return deferredForConsent +} diff --git a/src/main/telemetry/client-test-harness.ts b/src/main/telemetry/client-test-harness.ts index 8d6bc7b70bd..055d9472731 100644 --- a/src/main/telemetry/client-test-harness.ts +++ b/src/main/telemetry/client-test-harness.ts @@ -79,8 +79,11 @@ export function makeFakeSettings(telemetry: GlobalSettings['telemetry']): Global } export function makeFakeStore(settings: GlobalSettings): Store { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: fakes every Store method telemetry reads or writes; other methods are never called. return { getSettings: vi.fn(() => settings), + getUI: vi.fn(() => ({ featureTipsSeenIds: [] })), + getChatUiLaunchTipVariant: vi.fn(() => 'none'), updateSettings: vi.fn((updates: Partial) => { if (updates.telemetry) { settings.telemetry = { diff --git a/src/main/telemetry/client.ts b/src/main/telemetry/client.ts index 8e1fbadb1e2..5f24887bdae 100644 --- a/src/main/telemetry/client.ts +++ b/src/main/telemetry/client.ts @@ -6,6 +6,7 @@ import type { CommonProps, EventName, EventProps, OptInVia } from '../../shared/ import type { AgentDashboardSurface } from '../../shared/telemetry-agent-dashboard-event-schemas' import type { Store } from '../persistence' import { consumeBurstToken, resetBurstCapsForSession } from './burst-cap' +import { ensureChatUiLaunchTipCohort } from './chat-ui-launch-tip-cohort' import { getCohortAtEmit } from './cohort-classifier' import { isConsentEnvOverridden, resolveConsent } from './consent' import * as postHogStart from './posthog-client-start' @@ -73,6 +74,8 @@ export function initTelemetry(store: Store, scheduleStart: postHogStart.StartSch if (languageCohort === 'control' || languageCohort === 'treatment') { commonProps.onboarding_language_v1 = languageCohort } + // Official builds only, so contributor builds always show the Chat UI launch tip. + commonProps.chat_ui_launch_tip_v1 = ensureChatUiLaunchTipCohort(store) // Validate singleton common properties once; invalid identity would collapse installs into one subject. const parsedCommon = commonPropsSchema.safeParse(commonProps) diff --git a/src/renderer/src/app-shell/use-onboarding-and-feature-tips.test.tsx b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.test.tsx new file mode 100644 index 00000000000..5279ffc8ca8 --- /dev/null +++ b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.test.tsx @@ -0,0 +1,131 @@ +// @vitest-environment happy-dom + +import { act, type JSX } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { FeatureTipId } from '../../../shared/feature-tips' +import { getDefaultOnboardingState } from '../../../shared/onboarding-defaults' +import { createGlobalSettingsFixture } from '../../../shared/global-settings-test-fixture' +import type { TelemetrySettings } from '../../../shared/telemetry-settings-types' +import type * as telemetry from '@/lib/telemetry' +import { track } from '@/lib/telemetry' +import { useAppStore } from '@/store' +import { useOnboardingAndFeatureTips } from './use-onboarding-and-feature-tips' + +const telemetryMock = vi.hoisted(() => ({ passThrough: false })) +// Pass-through lets one test drive the real bridge wrapper against a rejecting main. +vi.mock('@/lib/telemetry', async (importOriginal) => { + const actual = await importOriginal() + return { + track: vi.fn((...args: Parameters) => { + if (telemetryMock.passThrough) { + actual.track(...args) + } + }) + } +}) + +globalThis.IS_REACT_ACT_ENVIRONMENT = true + +let container: HTMLDivElement +let root: Root + +function Gate(): JSX.Element | null { + const gate = useOnboardingAndFeatureTips() + if (gate.onboarding === null) { + gate.applyStartupOnboardingState({ ...getDefaultOnboardingState(), closedAt: 1 }) + } + return null +} + +async function launch(cohort: TelemetrySettings['chatUiLaunchTipCohort']): Promise { + const markFeatureTipsSeen = vi.fn((ids: FeatureTipId[]) => + useAppStore.setState((state) => ({ featureTipsSeenIds: [...state.featureTipsSeenIds, ...ids] })) + ) + useAppStore.setState({ + persistedUIReady: true, + featureTipsSeenIds: [], + settings: createGlobalSettingsFixture({ + aiVaultSearch: { enabled: true, historyDays: null }, + telemetry: { + optedIn: true, + installId: 'install', + existedBeforeTelemetryRelease: true, + ...(cohort ? { chatUiLaunchTipCohort: cohort } : {}) + } + }), + markFeatureTipsSeen + }) + await act(async () => root.render()) +} + +beforeEach(() => { + useAppStore.setState(useAppStore.getInitialState(), true) + vi.mocked(track).mockClear() + telemetryMock.passThrough = false + Object.defineProperty(window, 'api', { + configurable: true, + value: { + ui: { set: vi.fn().mockResolvedValue(undefined) }, + cli: { getInstallStatus: vi.fn().mockResolvedValue({ supported: false }) }, + onboarding: { getChatUiLaunchTipVariant: vi.fn().mockResolvedValue('chat-ui-off') } + } + }) + container = document.createElement('div') + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) +}) + +describe('Chat UI launch tip holdout', () => { + it.each([ + [undefined, 'unassigned'], + ['treatment', 'treatment'] + ] as const)('shows the tip to %s profiles after recording exposure', async (cohort, label) => { + await launch(cohort) + expect(useAppStore.getState().activeModal).toBe('feature-tips') + expect(useAppStore.getState().modalData).toMatchObject({ tipId: 'chat-ui-here' }) + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', { + variant: 'chat_ui_off', + launch_tip_cohort: label, + shown: true + }) + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_shown', { variant: 'chat_ui_off' }) + }) + + it('records control exposure without showing the tip, then lets the next tip open', async () => { + await launch('control') + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', { + variant: 'chat_ui_off', + launch_tip_cohort: 'control', + shown: false + }) + expect(track).not.toHaveBeenCalledWith('chat_ui_feature_tip_shown', expect.anything()) + expect(useAppStore.getState().featureTipsSeenIds).toContain('chat-ui-here') + expect(useAppStore.getState().activeModal).toBe('feature-tips') + expect(useAppStore.getState().modalData).toMatchObject({ tipId: 'cmd-j-palette' }) + }) + + // Exposure is best-effort: a rejected capture (consent flip, session budget, IPC failure) is not + // retried, and the tip stays consumed so a failed send never shows it twice. + it.each(['treatment', 'control'] as const)( + 'keeps the %s tip consumed when main rejects the exposure event', + async (cohort) => { + telemetryMock.passThrough = true + const telemetryTrack = vi.fn().mockResolvedValue(false) + Object.assign(window.api, { telemetryTrack }) + await launch(cohort) + expect(telemetryTrack).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', { + variant: 'chat_ui_off', + launch_tip_cohort: cohort, + shown: cohort === 'treatment' + }) + expect(useAppStore.getState().featureTipsSeenIds).toContain('chat-ui-here') + expect(useAppStore.getState().modalData).toMatchObject({ + tipId: cohort === 'treatment' ? 'chat-ui-here' : 'cmd-j-palette' + }) + } + ) +}) diff --git a/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts index 8769740bd75..c3bd0432ea4 100644 --- a/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts +++ b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts @@ -7,6 +7,8 @@ import { isCliFeatureTipCompleted } from '../components/feature-tips/feature-tip-startup-gate' import { + isChatUiLaunchTipHeldOut, + trackChatUiFeatureTipExposure, trackChatUiFeatureTipShown, trackCmdJPaletteFeatureTipShown, trackOrcaCliFeatureTipShown @@ -140,6 +142,15 @@ export function useOnboardingAndFeatureTips() { return } + if (featureTipsDecision.tipId === 'chat-ui-here' && chatUiLaunchTipVariant) { + trackChatUiFeatureTipExposure(chatUiLaunchTipVariant, settings ?? null) + if (isChatUiLaunchTipHeldOut(settings ?? null)) { + // Control behaves as if the tip never existed, so the next pending tip may take this launch. + actions.markFeatureTipsSeen(['chat-ui-here']) + return + } + } + promptedThisSessionRef.current = true if (featureTipsDecision.tipId === 'orca-cli') { trackOrcaCliFeatureTipShown('app_open') diff --git a/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx b/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx index 47677e69e89..58b24224be0 100644 --- a/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx +++ b/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx @@ -200,10 +200,16 @@ describe('Chat UI is here tip', () => { expect(useAppStore.getState().activeModal).toBe('none') expect(updateSettingsOrThrow).toHaveBeenCalledExactlyOnceWith({ agentDefaultSurface: 'chat' }) expect(updateSettings).not.toHaveBeenCalled() - expect(track).toHaveBeenLastCalledWith('chat_ui_feature_tip_action', { + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_action', { variant: 'chat_ui_off', action: 'use_chat_ui' }) + expect(track).toHaveBeenLastCalledWith('agent_default_surface_changed', { + from_mode: 'terminal', + to_mode: 'chat', + source: 'launch_card', + launch_tip_shown: true + }) }) it('offers chat users a single Got it that closes without saving', async () => { diff --git a/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx b/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx index 91bf40a49eb..954ebb497c1 100644 --- a/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx +++ b/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState, type JSX } from 'react' import { toast } from 'sonner' import { getDefaultVoiceSettings } from '../../../../shared/constants' +import { normalizeAgentDefaultSurface } from '../../../../shared/native-chat/native-chat-providers' import { ORCHESTRATION_ENABLED_STORAGE_KEY, ORCHESTRATION_SETUP_DISMISSED_STORAGE_KEY, @@ -15,6 +16,7 @@ import { installCliFromFeatureTip } from './feature-tip-cli-install-action' import { getFeatureTipForModal } from './feature-tip-modal-state' import { getOrcaCliFeatureTipTelemetrySource, + trackAgentDefaultSurfaceChanged, trackChatUiFeatureTipAction, trackCmdJPaletteFeatureTipAcknowledged, trackOrcaCliFeatureTipSetupClicked, @@ -155,10 +157,17 @@ export default function FeatureTipsModal(): JSX.Element | null { trackChatUiFeatureTipAction(chatUiLaunchTipVariant, 'dismissed') break } + const fromSurface = normalizeAgentDefaultSurface(settings?.agentDefaultSurface) try { // Only the default moves; an agent the user set for itself keeps its own choice. await updateSettingsOrThrow({ agentDefaultSurface: 'chat' }) trackChatUiFeatureTipAction(chatUiLaunchTipVariant, 'use_chat_ui') + trackAgentDefaultSurfaceChanged({ + from_mode: fromSurface, + to_mode: 'chat', + source: 'launch_card', + launch_tip_shown: true + }) } catch { toast.error( translate( diff --git a/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts b/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts index 818a44c3078..bb20e9d3393 100644 --- a/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts +++ b/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts @@ -1,6 +1,8 @@ import { track } from '@/lib/telemetry' import type { EventProps } from '../../../../shared/telemetry-events' import type { ChatUiLaunchTipVariant } from '../../../../shared/native-chat/chat-ui-launch-tip-audience' +import type { FeatureTipId } from '../../../../shared/feature-tips' +import type { GlobalSettings } from '../../../../shared/global-settings-types' export type OrcaCliFeatureTipSource = EventProps<'orca_cli_feature_tip_shown'>['source'] export type OrcaCliFeatureTipSetupResult = EventProps<'orca_cli_feature_tip_setup_result'>['result'] @@ -58,3 +60,38 @@ export function trackChatUiFeatureTipAction( track('chat_ui_feature_tip_action', { variant: CHAT_UI_TIP_TELEMETRY_VARIANT[variant], action }) } } + +/** Control profiles never see the Chat UI launch tip; unenrolled ones (no consent, contributor builds) do. */ +export function isChatUiLaunchTipHeldOut( + settings: Pick | null +): boolean { + return settings?.telemetry?.chatUiLaunchTipCohort === 'control' +} + +export function wasChatUiLaunchTipShown( + seenTipIds: readonly FeatureTipId[], + settings: Pick | null +): boolean { + return seenTipIds.includes('chat-ui-here') && !isChatUiLaunchTipHeldOut(settings) +} + +export function trackChatUiFeatureTipExposure( + variant: ChatUiLaunchTipVariant, + settings: Pick | null +): void { + if (variant !== 'none') { + track('chat_ui_feature_tip_exposure', { + variant: CHAT_UI_TIP_TELEMETRY_VARIANT[variant], + launch_tip_cohort: settings?.telemetry?.chatUiLaunchTipCohort ?? 'unassigned', + shown: !isChatUiLaunchTipHeldOut(settings) + }) + } +} + +export function trackAgentDefaultSurfaceChanged( + props: EventProps<'agent_default_surface_changed'> +): void { + if (props.from_mode !== props.to_mode) { + track('agent_default_surface_changed', props) + } +} diff --git a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx index 8d9567278dd..543afb80924 100644 --- a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx +++ b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx @@ -4,6 +4,7 @@ import { toast } from 'sonner' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createGlobalSettingsFixture } from '../../../../shared/global-settings-test-fixture' import type { GlobalSettings } from '../../../../shared/global-settings-types' +import { track } from '@/lib/telemetry' import { useAppStore } from '@/store' import { saveNativeChatProviderChoice } from './native-chat-provider-choice' import { AgentDefaultSurfaceSection } from './AgentDefaultSurfaceSection' @@ -11,6 +12,7 @@ import { AgentDefaultSurfaceSection } from './AgentDefaultSurfaceSection' vi.mock('@/lib/chat-provider-agents', () => ({ chatProviderAgents: () => ['claude', 'codex', 'grok', 'opencode'] })) +vi.mock('@/lib/telemetry', () => ({ track: vi.fn() })) vi.mock('sonner', () => ({ toast: { success: vi.fn(), dismiss: vi.fn(), error: vi.fn() } })) let persisted: GlobalSettings @@ -33,6 +35,7 @@ function renderSection(settings: Partial): void { beforeEach(() => { useAppStore.setState(useAppStore.getInitialState(), true) settingsSet.mockClear() + vi.mocked(track).mockClear() vi.mocked(toast.success).mockClear() vi.mocked(toast.error).mockClear() Object.defineProperty(window, 'api', { @@ -86,6 +89,37 @@ describe('Open agents as', () => { }) }) +describe('Chat UI launch tip experiment', () => { + it.each([ + [[], 'treatment', false], + [['chat-ui-here'], 'treatment', true], + [['chat-ui-here'], 'control', false] + ] as const)( + 'reports a switch to Chat with seen tips %j in %s as launch_tip_shown=%s', + async (seen, cohort, shown) => { + useAppStore.setState({ featureTipsSeenIds: [...seen] }) + renderSection({ + agentDefaultSurface: 'terminal', + telemetry: { + optedIn: true, + installId: 'install', + existedBeforeTelemetryRelease: true, + chatUiLaunchTipCohort: cohort + } + }) + fireEvent.click(screen.getByRole('radio', { name: /^Chat/ })) + await waitFor(() => + expect(track).toHaveBeenCalledExactlyOnceWith('agent_default_surface_changed', { + from_mode: 'terminal', + to_mode: 'chat', + source: 'settings', + launch_tip_shown: shown + }) + ) + } + ) +}) + describe('acknowledged default writes', () => { it('reports a rejected tile save once, preserves the choice, and accepts its successor', async () => { renderSection({ agentDefaultSurface: 'terminal' }) diff --git a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx index 1dbec87f6c3..f9856bd0fc6 100644 --- a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx +++ b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx @@ -12,6 +12,10 @@ import { getIntlLocale, translate } from '@/i18n/i18n' import { getAgentLabel } from '@/lib/agent-catalog' import { cn } from '@/lib/utils' import { useAppStore } from '@/store' +import { + trackAgentDefaultSurfaceChanged, + wasChatUiLaunchTipShown +} from '../feature-tips/feature-tip-telemetry' import { ChatSurfacePreview, TerminalSurfacePreview } from './AgentDefaultSurfacePreviews' import { AGENT_DEFAULT_SURFACE_SECTION_ID, AGENTS_INSTALLED_SECTION_ID } from './agents-search' import { @@ -175,7 +179,18 @@ export function AgentDefaultSurfaceSection({ const writer = { fallbackSettings: settings, updateSettings } void mutation.run('default-surface', { mutate: () => saveAgentDefaultSurface({ surface: next, ...writer }), - onSuccess: () => notify(next, writer), + onSuccess: () => { + trackAgentDefaultSurfaceChanged({ + from_mode: surface, + to_mode: next, + source: 'settings', + launch_tip_shown: wasChatUiLaunchTipShown( + useAppStore.getState().featureTipsSeenIds, + settings + ) + }) + notify(next, writer) + }, onError: saveFailed }) } diff --git a/src/shared/telemetry-event-classification.ts b/src/shared/telemetry-event-classification.ts index 5fbb85c41cf..d0fe0690c1d 100644 --- a/src/shared/telemetry-event-classification.ts +++ b/src/shared/telemetry-event-classification.ts @@ -60,6 +60,8 @@ type _CohortExtendedRoster = | 'cmd_j_palette_feature_tip_acknowledged' | 'chat_ui_feature_tip_shown' | 'chat_ui_feature_tip_action' + | 'chat_ui_feature_tip_exposure' + | 'agent_default_surface_changed' // Strict empty payloads infer string index signatures; exclude those from keyed rosters. type _KnownPayloadKeys = string extends keyof T ? never : keyof T type _DerivedCohortExtendedEvents = { @@ -128,7 +130,8 @@ export const commonPropsSchema = z install_id: z.string().min(1).max(64), session_id: z.string().min(1).max(64), orca_channel: z.enum(['stable', 'rc']), - onboarding_language_v1: z.enum(['control', 'treatment']).optional() + onboarding_language_v1: z.enum(['control', 'treatment']).optional(), + chat_ui_launch_tip_v1: z.enum(['control', 'treatment']).optional() }) .strict() export type CommonProps = z.infer diff --git a/src/shared/telemetry-event-registry.ts b/src/shared/telemetry-event-registry.ts index e4af5dcf974..74b4953c58b 100644 --- a/src/shared/telemetry-event-registry.ts +++ b/src/shared/telemetry-event-registry.ts @@ -50,6 +50,8 @@ import { } from './telemetry-feature-education-event-schemas' import { chatUiFeatureTipActionSchema, + chatUiFeatureTipExposureSchema, + agentDefaultSurfaceChangedSchema, chatUiFeatureTipShownSchema, cmdJPaletteFeatureTipAcknowledgedSchema, cmdJPaletteFeatureTipShownSchema, @@ -185,6 +187,8 @@ export const eventSchemas = { cmd_j_palette_feature_tip_acknowledged: cmdJPaletteFeatureTipAcknowledgedSchema, chat_ui_feature_tip_shown: chatUiFeatureTipShownSchema, chat_ui_feature_tip_action: chatUiFeatureTipActionSchema, + chat_ui_feature_tip_exposure: chatUiFeatureTipExposureSchema, + agent_default_surface_changed: agentDefaultSurfaceChangedSchema, feature_wall_opened: featureWallOpenedSchema, feature_wall_closed: featureWallClosedSchema, diff --git a/src/shared/telemetry-native-feature-event-schemas.ts b/src/shared/telemetry-native-feature-event-schemas.ts index b06d998ec20..fdff66ce843 100644 --- a/src/shared/telemetry-native-feature-event-schemas.ts +++ b/src/shared/telemetry-native-feature-event-schemas.ts @@ -96,6 +96,24 @@ export const chatUiFeatureTipActionSchema = z nth_repo_added: nthRepoAddedSchema }) .strict() +// Fires in both arms when the tip would open, so control has the same exposure moment as treatment. +export const chatUiFeatureTipExposureSchema = z + .object({ + variant: chatUiFeatureTipVariantSchema, + launch_tip_cohort: z.enum(['control', 'treatment', 'unassigned']), + shown: z.boolean(), + nth_repo_added: nthRepoAddedSchema + }) + .strict() +export const agentDefaultSurfaceChangedSchema = z + .object({ + from_mode: nativeChatViewModeSchema, + to_mode: nativeChatViewModeSchema, + source: z.enum(['launch_card', 'settings']), + launch_tip_shown: z.boolean(), + nth_repo_added: nthRepoAddedSchema + }) + .strict() export const featureWallOpenedSchema = z .object({ diff --git a/src/shared/telemetry-settings-types.ts b/src/shared/telemetry-settings-types.ts index b668c6814d4..52225808dfd 100644 --- a/src/shared/telemetry-settings-types.ts +++ b/src/shared/telemetry-settings-types.ts @@ -9,6 +9,8 @@ export type TelemetrySettings = { existedBeforeTelemetryRelease: boolean /** Frozen 50/50 assignment for fresh installs; absent for profiles predating the experiment. */ onboardingLanguageCohort?: 'control' | 'treatment' + /** Frozen 50/50 holdout for the Chat UI launch tip; control never sees it. Absent means not enrolled, so the tip shows. */ + chatUiLaunchTipCohort?: 'control' | 'treatment' /** Local UTC-day cursors cap dashboard observations across restarts; at most two writes a day. */ agentDashboardActiveDays?: Partial> }