diff --git a/src/main/ipc/onboarding.ts b/src/main/ipc/onboarding.ts index 6582140c15b..6438e56f0ca 100644 --- a/src/main/ipc/onboarding.ts +++ b/src/main/ipc/onboarding.ts @@ -2,6 +2,7 @@ import { ipcMain } from 'electron' import { sanitizeOnboardingUpdate, type Store } from '../persistence' import type { OnboardingState } from '../../shared/onboarding-state-types' import type { ChatUiLaunchTipVariant } from '../../shared/native-chat/chat-ui-launch-tip-audience' +import { isChatUiLaunchTipDeferredForConsent } from '../telemetry/chat-ui-launch-tip-cohort' export function registerOnboardingHandlers(store: Store): void { ipcMain.removeHandler('onboarding:get') @@ -13,7 +14,8 @@ export function registerOnboardingHandlers(store: Store): void { ipcMain.handle('onboarding:update', (_event, updates: unknown): OnboardingState => { return store.updateOnboarding(sanitizeOnboardingUpdate(updates)) }) + // 'none' skips the tip this launch without marking it seen, so it returns once consent resolves. ipcMain.handle('onboarding:getChatUiLaunchTipVariant', (): ChatUiLaunchTipVariant => - store.getChatUiLaunchTipVariant() + isChatUiLaunchTipDeferredForConsent() ? 'none' : store.getChatUiLaunchTipVariant() ) } diff --git a/src/main/telemetry/chat-ui-launch-tip-cohort.test.ts b/src/main/telemetry/chat-ui-launch-tip-cohort.test.ts new file mode 100644 index 00000000000..ed9dcd7b836 --- /dev/null +++ b/src/main/telemetry/chat-ui-launch-tip-cohort.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ChatUiLaunchTipVariant } from '../../shared/native-chat/chat-ui-launch-tip-audience' +import type { TelemetrySettings } from '../../shared/telemetry-settings-types' +import type { Store } from '../persistence' +import { + ensureChatUiLaunchTipCohort, + isChatUiLaunchTipDeferredForConsent +} from './chat-ui-launch-tip-cohort' +import { makeFakeSettings, restoreConsentEnv, stashAndClearConsentEnv } from './client-test-harness' + +function makeStore( + telemetry: Partial, + variant: ChatUiLaunchTipVariant = 'chat-ui-off', + seen: string[] = [] +): Store { + const settings = makeFakeSettings({ + optedIn: true, + installId: 'install', + existedBeforeTelemetryRelease: true, + ...telemetry + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: supplies every Store method the cohort assignment reads or writes. + return { + getSettings: () => settings, + getUI: () => ({ featureTipsSeenIds: seen }), + getChatUiLaunchTipVariant: () => variant, + updateSettings: vi.fn((updates: { telemetry: TelemetrySettings }) => { + settings.telemetry = updates.telemetry + }) + } as unknown as Store +} + +describe('ensureChatUiLaunchTipCohort', () => { + let env: Record + beforeEach(() => { + env = stashAndClearConsentEnv() + }) + afterEach(() => { + restoreConsentEnv(env) + }) + + it('enrolls a profile still due the tip once and keeps the frozen cohort', () => { + const store = makeStore({}) + const cohort = ensureChatUiLaunchTipCohort(store) + expect(['control', 'treatment']).toContain(cohort) + expect(store.getSettings().telemetry?.chatUiLaunchTipCohort).toBe(cohort) + expect(store.getSettings().telemetry?.installId).toBe('install') + expect(ensureChatUiLaunchTipCohort(store)).toBe(cohort) + expect(store.updateSettings).toHaveBeenCalledTimes(1) + }) + + it('holds the tip unenrolled while the banner is pending, then enrolls after acknowledgement', () => { + const pending = makeStore({ optedIn: null }) + expect(ensureChatUiLaunchTipCohort(pending)).toBeUndefined() + expect(isChatUiLaunchTipDeferredForConsent()).toBe(true) + expect(pending.updateSettings).not.toHaveBeenCalled() + + // Acknowledging the banner persists optedIn=true; the next launch enrolls the still-unseen tip. + pending.getSettings().telemetry!.optedIn = true + const cohort = ensureChatUiLaunchTipCohort(pending) + expect(['control', 'treatment']).toContain(cohort) + expect(isChatUiLaunchTipDeferredForConsent()).toBe(false) + expect(pending.getSettings().telemetry?.chatUiLaunchTipCohort).toBe(cohort) + }) + + it('releases a pending profile unenrolled after the banner is declined, so the tip shows as before', () => { + const pending = makeStore({ optedIn: null }) + ensureChatUiLaunchTipCohort(pending) + expect(isChatUiLaunchTipDeferredForConsent()).toBe(true) + + pending.getSettings().telemetry!.optedIn = false + expect(ensureChatUiLaunchTipCohort(pending)).toBeUndefined() + expect(isChatUiLaunchTipDeferredForConsent()).toBe(false) + expect(pending.updateSettings).not.toHaveBeenCalled() + }) + + it.each([ + ['outside the audience', makeStore({}, 'none')], + ['already shown the tip', makeStore({}, 'chat-ui-off', ['chat-ui-here'])], + ['opted out', makeStore({ optedIn: false })] + ])('leaves a profile %s unenrolled, so it keeps today’s behaviour', (_label, store) => { + expect(ensureChatUiLaunchTipCohort(store)).toBeUndefined() + expect(isChatUiLaunchTipDeferredForConsent()).toBe(false) + expect(store.updateSettings).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/telemetry/chat-ui-launch-tip-cohort.ts b/src/main/telemetry/chat-ui-launch-tip-cohort.ts new file mode 100644 index 00000000000..ac2e2dd542f --- /dev/null +++ b/src/main/telemetry/chat-ui-launch-tip-cohort.ts @@ -0,0 +1,49 @@ +import { randomInt } from 'node:crypto' +import type { Store } from '../persistence' +import { resolveConsent } from './consent' + +export type ChatUiLaunchTipCohort = 'control' | 'treatment' + +// Decided once per launch with the cohort, so the tip and its enrollment read the same consent; contributor builds never set it. +let deferredForConsent = false + +/** + * Enrolls once, only profiles still due the tip whose events can be measured. While the telemetry + * banner is unresolved the tip waits, so the exposure is never spent unmeasured; opted-out and + * contributor builds stay unenrolled and see the tip as before. + */ +export function ensureChatUiLaunchTipCohort(store: Store): ChatUiLaunchTipCohort | undefined { + deferredForConsent = false + try { + const settings = store.getSettings() + const telemetry = settings.telemetry + const saved = telemetry?.chatUiLaunchTipCohort + if (saved === 'control' || saved === 'treatment') { + return saved + } + if ( + !telemetry || + store.getChatUiLaunchTipVariant() === 'none' || + store.getUI().featureTipsSeenIds?.includes('chat-ui-here') + ) { + return undefined + } + const consent = resolveConsent(settings).effective + if (consent !== 'enabled') { + deferredForConsent = consent === 'pending_banner' + return undefined + } + const cohort: ChatUiLaunchTipCohort = randomInt(2) === 0 ? 'control' : 'treatment' + store.updateSettings({ telemetry: { ...telemetry, chatUiLaunchTipCohort: cohort } }) + return cohort + } catch (err) { + // Startup must not fail over an experiment; unenrolled profiles see the tip as before. + console.warn('[telemetry] chat UI launch tip cohort assignment failed', err) + return undefined + } +} + +/** True when this launch holds the tip back until the telemetry banner is resolved. */ +export function isChatUiLaunchTipDeferredForConsent(): boolean { + return deferredForConsent +} diff --git a/src/main/telemetry/client-test-harness.ts b/src/main/telemetry/client-test-harness.ts index 8d6bc7b70bd..055d9472731 100644 --- a/src/main/telemetry/client-test-harness.ts +++ b/src/main/telemetry/client-test-harness.ts @@ -79,8 +79,11 @@ export function makeFakeSettings(telemetry: GlobalSettings['telemetry']): Global } export function makeFakeStore(settings: GlobalSettings): Store { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: fakes every Store method telemetry reads or writes; other methods are never called. return { getSettings: vi.fn(() => settings), + getUI: vi.fn(() => ({ featureTipsSeenIds: [] })), + getChatUiLaunchTipVariant: vi.fn(() => 'none'), updateSettings: vi.fn((updates: Partial) => { if (updates.telemetry) { settings.telemetry = { diff --git a/src/main/telemetry/client.ts b/src/main/telemetry/client.ts index 8e1fbadb1e2..5f24887bdae 100644 --- a/src/main/telemetry/client.ts +++ b/src/main/telemetry/client.ts @@ -6,6 +6,7 @@ import type { CommonProps, EventName, EventProps, OptInVia } from '../../shared/ import type { AgentDashboardSurface } from '../../shared/telemetry-agent-dashboard-event-schemas' import type { Store } from '../persistence' import { consumeBurstToken, resetBurstCapsForSession } from './burst-cap' +import { ensureChatUiLaunchTipCohort } from './chat-ui-launch-tip-cohort' import { getCohortAtEmit } from './cohort-classifier' import { isConsentEnvOverridden, resolveConsent } from './consent' import * as postHogStart from './posthog-client-start' @@ -73,6 +74,8 @@ export function initTelemetry(store: Store, scheduleStart: postHogStart.StartSch if (languageCohort === 'control' || languageCohort === 'treatment') { commonProps.onboarding_language_v1 = languageCohort } + // Official builds only, so contributor builds always show the Chat UI launch tip. + commonProps.chat_ui_launch_tip_v1 = ensureChatUiLaunchTipCohort(store) // Validate singleton common properties once; invalid identity would collapse installs into one subject. const parsedCommon = commonPropsSchema.safeParse(commonProps) diff --git a/src/renderer/src/app-shell/use-onboarding-and-feature-tips.test.tsx b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.test.tsx new file mode 100644 index 00000000000..5279ffc8ca8 --- /dev/null +++ b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.test.tsx @@ -0,0 +1,131 @@ +// @vitest-environment happy-dom + +import { act, type JSX } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { FeatureTipId } from '../../../shared/feature-tips' +import { getDefaultOnboardingState } from '../../../shared/onboarding-defaults' +import { createGlobalSettingsFixture } from '../../../shared/global-settings-test-fixture' +import type { TelemetrySettings } from '../../../shared/telemetry-settings-types' +import type * as telemetry from '@/lib/telemetry' +import { track } from '@/lib/telemetry' +import { useAppStore } from '@/store' +import { useOnboardingAndFeatureTips } from './use-onboarding-and-feature-tips' + +const telemetryMock = vi.hoisted(() => ({ passThrough: false })) +// Pass-through lets one test drive the real bridge wrapper against a rejecting main. +vi.mock('@/lib/telemetry', async (importOriginal) => { + const actual = await importOriginal() + return { + track: vi.fn((...args: Parameters) => { + if (telemetryMock.passThrough) { + actual.track(...args) + } + }) + } +}) + +globalThis.IS_REACT_ACT_ENVIRONMENT = true + +let container: HTMLDivElement +let root: Root + +function Gate(): JSX.Element | null { + const gate = useOnboardingAndFeatureTips() + if (gate.onboarding === null) { + gate.applyStartupOnboardingState({ ...getDefaultOnboardingState(), closedAt: 1 }) + } + return null +} + +async function launch(cohort: TelemetrySettings['chatUiLaunchTipCohort']): Promise { + const markFeatureTipsSeen = vi.fn((ids: FeatureTipId[]) => + useAppStore.setState((state) => ({ featureTipsSeenIds: [...state.featureTipsSeenIds, ...ids] })) + ) + useAppStore.setState({ + persistedUIReady: true, + featureTipsSeenIds: [], + settings: createGlobalSettingsFixture({ + aiVaultSearch: { enabled: true, historyDays: null }, + telemetry: { + optedIn: true, + installId: 'install', + existedBeforeTelemetryRelease: true, + ...(cohort ? { chatUiLaunchTipCohort: cohort } : {}) + } + }), + markFeatureTipsSeen + }) + await act(async () => root.render()) +} + +beforeEach(() => { + useAppStore.setState(useAppStore.getInitialState(), true) + vi.mocked(track).mockClear() + telemetryMock.passThrough = false + Object.defineProperty(window, 'api', { + configurable: true, + value: { + ui: { set: vi.fn().mockResolvedValue(undefined) }, + cli: { getInstallStatus: vi.fn().mockResolvedValue({ supported: false }) }, + onboarding: { getChatUiLaunchTipVariant: vi.fn().mockResolvedValue('chat-ui-off') } + } + }) + container = document.createElement('div') + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) +}) + +describe('Chat UI launch tip holdout', () => { + it.each([ + [undefined, 'unassigned'], + ['treatment', 'treatment'] + ] as const)('shows the tip to %s profiles after recording exposure', async (cohort, label) => { + await launch(cohort) + expect(useAppStore.getState().activeModal).toBe('feature-tips') + expect(useAppStore.getState().modalData).toMatchObject({ tipId: 'chat-ui-here' }) + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', { + variant: 'chat_ui_off', + launch_tip_cohort: label, + shown: true + }) + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_shown', { variant: 'chat_ui_off' }) + }) + + it('records control exposure without showing the tip, then lets the next tip open', async () => { + await launch('control') + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', { + variant: 'chat_ui_off', + launch_tip_cohort: 'control', + shown: false + }) + expect(track).not.toHaveBeenCalledWith('chat_ui_feature_tip_shown', expect.anything()) + expect(useAppStore.getState().featureTipsSeenIds).toContain('chat-ui-here') + expect(useAppStore.getState().activeModal).toBe('feature-tips') + expect(useAppStore.getState().modalData).toMatchObject({ tipId: 'cmd-j-palette' }) + }) + + // Exposure is best-effort: a rejected capture (consent flip, session budget, IPC failure) is not + // retried, and the tip stays consumed so a failed send never shows it twice. + it.each(['treatment', 'control'] as const)( + 'keeps the %s tip consumed when main rejects the exposure event', + async (cohort) => { + telemetryMock.passThrough = true + const telemetryTrack = vi.fn().mockResolvedValue(false) + Object.assign(window.api, { telemetryTrack }) + await launch(cohort) + expect(telemetryTrack).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', { + variant: 'chat_ui_off', + launch_tip_cohort: cohort, + shown: cohort === 'treatment' + }) + expect(useAppStore.getState().featureTipsSeenIds).toContain('chat-ui-here') + expect(useAppStore.getState().modalData).toMatchObject({ + tipId: cohort === 'treatment' ? 'chat-ui-here' : 'cmd-j-palette' + }) + } + ) +}) diff --git a/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts index 8769740bd75..c3bd0432ea4 100644 --- a/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts +++ b/src/renderer/src/app-shell/use-onboarding-and-feature-tips.ts @@ -7,6 +7,8 @@ import { isCliFeatureTipCompleted } from '../components/feature-tips/feature-tip-startup-gate' import { + isChatUiLaunchTipHeldOut, + trackChatUiFeatureTipExposure, trackChatUiFeatureTipShown, trackCmdJPaletteFeatureTipShown, trackOrcaCliFeatureTipShown @@ -140,6 +142,15 @@ export function useOnboardingAndFeatureTips() { return } + if (featureTipsDecision.tipId === 'chat-ui-here' && chatUiLaunchTipVariant) { + trackChatUiFeatureTipExposure(chatUiLaunchTipVariant, settings ?? null) + if (isChatUiLaunchTipHeldOut(settings ?? null)) { + // Control behaves as if the tip never existed, so the next pending tip may take this launch. + actions.markFeatureTipsSeen(['chat-ui-here']) + return + } + } + promptedThisSessionRef.current = true if (featureTipsDecision.tipId === 'orca-cli') { trackOrcaCliFeatureTipShown('app_open') diff --git a/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx b/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx index 47677e69e89..58b24224be0 100644 --- a/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx +++ b/src/renderer/src/components/feature-tips/ChatUiHereTip.test.tsx @@ -200,10 +200,16 @@ describe('Chat UI is here tip', () => { expect(useAppStore.getState().activeModal).toBe('none') expect(updateSettingsOrThrow).toHaveBeenCalledExactlyOnceWith({ agentDefaultSurface: 'chat' }) expect(updateSettings).not.toHaveBeenCalled() - expect(track).toHaveBeenLastCalledWith('chat_ui_feature_tip_action', { + expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_action', { variant: 'chat_ui_off', action: 'use_chat_ui' }) + expect(track).toHaveBeenLastCalledWith('agent_default_surface_changed', { + from_mode: 'terminal', + to_mode: 'chat', + source: 'launch_card', + launch_tip_shown: true + }) }) it('offers chat users a single Got it that closes without saving', async () => { diff --git a/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx b/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx index 91bf40a49eb..954ebb497c1 100644 --- a/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx +++ b/src/renderer/src/components/feature-tips/FeatureTipsModal.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState, type JSX } from 'react' import { toast } from 'sonner' import { getDefaultVoiceSettings } from '../../../../shared/constants' +import { normalizeAgentDefaultSurface } from '../../../../shared/native-chat/native-chat-providers' import { ORCHESTRATION_ENABLED_STORAGE_KEY, ORCHESTRATION_SETUP_DISMISSED_STORAGE_KEY, @@ -15,6 +16,7 @@ import { installCliFromFeatureTip } from './feature-tip-cli-install-action' import { getFeatureTipForModal } from './feature-tip-modal-state' import { getOrcaCliFeatureTipTelemetrySource, + trackAgentDefaultSurfaceChanged, trackChatUiFeatureTipAction, trackCmdJPaletteFeatureTipAcknowledged, trackOrcaCliFeatureTipSetupClicked, @@ -155,10 +157,17 @@ export default function FeatureTipsModal(): JSX.Element | null { trackChatUiFeatureTipAction(chatUiLaunchTipVariant, 'dismissed') break } + const fromSurface = normalizeAgentDefaultSurface(settings?.agentDefaultSurface) try { // Only the default moves; an agent the user set for itself keeps its own choice. await updateSettingsOrThrow({ agentDefaultSurface: 'chat' }) trackChatUiFeatureTipAction(chatUiLaunchTipVariant, 'use_chat_ui') + trackAgentDefaultSurfaceChanged({ + from_mode: fromSurface, + to_mode: 'chat', + source: 'launch_card', + launch_tip_shown: true + }) } catch { toast.error( translate( diff --git a/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts b/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts index 818a44c3078..bb20e9d3393 100644 --- a/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts +++ b/src/renderer/src/components/feature-tips/feature-tip-telemetry.ts @@ -1,6 +1,8 @@ import { track } from '@/lib/telemetry' import type { EventProps } from '../../../../shared/telemetry-events' import type { ChatUiLaunchTipVariant } from '../../../../shared/native-chat/chat-ui-launch-tip-audience' +import type { FeatureTipId } from '../../../../shared/feature-tips' +import type { GlobalSettings } from '../../../../shared/global-settings-types' export type OrcaCliFeatureTipSource = EventProps<'orca_cli_feature_tip_shown'>['source'] export type OrcaCliFeatureTipSetupResult = EventProps<'orca_cli_feature_tip_setup_result'>['result'] @@ -58,3 +60,38 @@ export function trackChatUiFeatureTipAction( track('chat_ui_feature_tip_action', { variant: CHAT_UI_TIP_TELEMETRY_VARIANT[variant], action }) } } + +/** Control profiles never see the Chat UI launch tip; unenrolled ones (no consent, contributor builds) do. */ +export function isChatUiLaunchTipHeldOut( + settings: Pick | null +): boolean { + return settings?.telemetry?.chatUiLaunchTipCohort === 'control' +} + +export function wasChatUiLaunchTipShown( + seenTipIds: readonly FeatureTipId[], + settings: Pick | null +): boolean { + return seenTipIds.includes('chat-ui-here') && !isChatUiLaunchTipHeldOut(settings) +} + +export function trackChatUiFeatureTipExposure( + variant: ChatUiLaunchTipVariant, + settings: Pick | null +): void { + if (variant !== 'none') { + track('chat_ui_feature_tip_exposure', { + variant: CHAT_UI_TIP_TELEMETRY_VARIANT[variant], + launch_tip_cohort: settings?.telemetry?.chatUiLaunchTipCohort ?? 'unassigned', + shown: !isChatUiLaunchTipHeldOut(settings) + }) + } +} + +export function trackAgentDefaultSurfaceChanged( + props: EventProps<'agent_default_surface_changed'> +): void { + if (props.from_mode !== props.to_mode) { + track('agent_default_surface_changed', props) + } +} diff --git a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx index 8d9567278dd..543afb80924 100644 --- a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx +++ b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.test.tsx @@ -4,6 +4,7 @@ import { toast } from 'sonner' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createGlobalSettingsFixture } from '../../../../shared/global-settings-test-fixture' import type { GlobalSettings } from '../../../../shared/global-settings-types' +import { track } from '@/lib/telemetry' import { useAppStore } from '@/store' import { saveNativeChatProviderChoice } from './native-chat-provider-choice' import { AgentDefaultSurfaceSection } from './AgentDefaultSurfaceSection' @@ -11,6 +12,7 @@ import { AgentDefaultSurfaceSection } from './AgentDefaultSurfaceSection' vi.mock('@/lib/chat-provider-agents', () => ({ chatProviderAgents: () => ['claude', 'codex', 'grok', 'opencode'] })) +vi.mock('@/lib/telemetry', () => ({ track: vi.fn() })) vi.mock('sonner', () => ({ toast: { success: vi.fn(), dismiss: vi.fn(), error: vi.fn() } })) let persisted: GlobalSettings @@ -33,6 +35,7 @@ function renderSection(settings: Partial): void { beforeEach(() => { useAppStore.setState(useAppStore.getInitialState(), true) settingsSet.mockClear() + vi.mocked(track).mockClear() vi.mocked(toast.success).mockClear() vi.mocked(toast.error).mockClear() Object.defineProperty(window, 'api', { @@ -86,6 +89,37 @@ describe('Open agents as', () => { }) }) +describe('Chat UI launch tip experiment', () => { + it.each([ + [[], 'treatment', false], + [['chat-ui-here'], 'treatment', true], + [['chat-ui-here'], 'control', false] + ] as const)( + 'reports a switch to Chat with seen tips %j in %s as launch_tip_shown=%s', + async (seen, cohort, shown) => { + useAppStore.setState({ featureTipsSeenIds: [...seen] }) + renderSection({ + agentDefaultSurface: 'terminal', + telemetry: { + optedIn: true, + installId: 'install', + existedBeforeTelemetryRelease: true, + chatUiLaunchTipCohort: cohort + } + }) + fireEvent.click(screen.getByRole('radio', { name: /^Chat/ })) + await waitFor(() => + expect(track).toHaveBeenCalledExactlyOnceWith('agent_default_surface_changed', { + from_mode: 'terminal', + to_mode: 'chat', + source: 'settings', + launch_tip_shown: shown + }) + ) + } + ) +}) + describe('acknowledged default writes', () => { it('reports a rejected tile save once, preserves the choice, and accepts its successor', async () => { renderSection({ agentDefaultSurface: 'terminal' }) diff --git a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx index 1dbec87f6c3..f9856bd0fc6 100644 --- a/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx +++ b/src/renderer/src/components/settings/AgentDefaultSurfaceSection.tsx @@ -12,6 +12,10 @@ import { getIntlLocale, translate } from '@/i18n/i18n' import { getAgentLabel } from '@/lib/agent-catalog' import { cn } from '@/lib/utils' import { useAppStore } from '@/store' +import { + trackAgentDefaultSurfaceChanged, + wasChatUiLaunchTipShown +} from '../feature-tips/feature-tip-telemetry' import { ChatSurfacePreview, TerminalSurfacePreview } from './AgentDefaultSurfacePreviews' import { AGENT_DEFAULT_SURFACE_SECTION_ID, AGENTS_INSTALLED_SECTION_ID } from './agents-search' import { @@ -175,7 +179,18 @@ export function AgentDefaultSurfaceSection({ const writer = { fallbackSettings: settings, updateSettings } void mutation.run('default-surface', { mutate: () => saveAgentDefaultSurface({ surface: next, ...writer }), - onSuccess: () => notify(next, writer), + onSuccess: () => { + trackAgentDefaultSurfaceChanged({ + from_mode: surface, + to_mode: next, + source: 'settings', + launch_tip_shown: wasChatUiLaunchTipShown( + useAppStore.getState().featureTipsSeenIds, + settings + ) + }) + notify(next, writer) + }, onError: saveFailed }) } diff --git a/src/shared/telemetry-event-classification.ts b/src/shared/telemetry-event-classification.ts index 5fbb85c41cf..d0fe0690c1d 100644 --- a/src/shared/telemetry-event-classification.ts +++ b/src/shared/telemetry-event-classification.ts @@ -60,6 +60,8 @@ type _CohortExtendedRoster = | 'cmd_j_palette_feature_tip_acknowledged' | 'chat_ui_feature_tip_shown' | 'chat_ui_feature_tip_action' + | 'chat_ui_feature_tip_exposure' + | 'agent_default_surface_changed' // Strict empty payloads infer string index signatures; exclude those from keyed rosters. type _KnownPayloadKeys = string extends keyof T ? never : keyof T type _DerivedCohortExtendedEvents = { @@ -128,7 +130,8 @@ export const commonPropsSchema = z install_id: z.string().min(1).max(64), session_id: z.string().min(1).max(64), orca_channel: z.enum(['stable', 'rc']), - onboarding_language_v1: z.enum(['control', 'treatment']).optional() + onboarding_language_v1: z.enum(['control', 'treatment']).optional(), + chat_ui_launch_tip_v1: z.enum(['control', 'treatment']).optional() }) .strict() export type CommonProps = z.infer diff --git a/src/shared/telemetry-event-registry.ts b/src/shared/telemetry-event-registry.ts index e4af5dcf974..74b4953c58b 100644 --- a/src/shared/telemetry-event-registry.ts +++ b/src/shared/telemetry-event-registry.ts @@ -50,6 +50,8 @@ import { } from './telemetry-feature-education-event-schemas' import { chatUiFeatureTipActionSchema, + chatUiFeatureTipExposureSchema, + agentDefaultSurfaceChangedSchema, chatUiFeatureTipShownSchema, cmdJPaletteFeatureTipAcknowledgedSchema, cmdJPaletteFeatureTipShownSchema, @@ -185,6 +187,8 @@ export const eventSchemas = { cmd_j_palette_feature_tip_acknowledged: cmdJPaletteFeatureTipAcknowledgedSchema, chat_ui_feature_tip_shown: chatUiFeatureTipShownSchema, chat_ui_feature_tip_action: chatUiFeatureTipActionSchema, + chat_ui_feature_tip_exposure: chatUiFeatureTipExposureSchema, + agent_default_surface_changed: agentDefaultSurfaceChangedSchema, feature_wall_opened: featureWallOpenedSchema, feature_wall_closed: featureWallClosedSchema, diff --git a/src/shared/telemetry-native-feature-event-schemas.ts b/src/shared/telemetry-native-feature-event-schemas.ts index b06d998ec20..fdff66ce843 100644 --- a/src/shared/telemetry-native-feature-event-schemas.ts +++ b/src/shared/telemetry-native-feature-event-schemas.ts @@ -96,6 +96,24 @@ export const chatUiFeatureTipActionSchema = z nth_repo_added: nthRepoAddedSchema }) .strict() +// Fires in both arms when the tip would open, so control has the same exposure moment as treatment. +export const chatUiFeatureTipExposureSchema = z + .object({ + variant: chatUiFeatureTipVariantSchema, + launch_tip_cohort: z.enum(['control', 'treatment', 'unassigned']), + shown: z.boolean(), + nth_repo_added: nthRepoAddedSchema + }) + .strict() +export const agentDefaultSurfaceChangedSchema = z + .object({ + from_mode: nativeChatViewModeSchema, + to_mode: nativeChatViewModeSchema, + source: z.enum(['launch_card', 'settings']), + launch_tip_shown: z.boolean(), + nth_repo_added: nthRepoAddedSchema + }) + .strict() export const featureWallOpenedSchema = z .object({ diff --git a/src/shared/telemetry-settings-types.ts b/src/shared/telemetry-settings-types.ts index b668c6814d4..52225808dfd 100644 --- a/src/shared/telemetry-settings-types.ts +++ b/src/shared/telemetry-settings-types.ts @@ -9,6 +9,8 @@ export type TelemetrySettings = { existedBeforeTelemetryRelease: boolean /** Frozen 50/50 assignment for fresh installs; absent for profiles predating the experiment. */ onboardingLanguageCohort?: 'control' | 'treatment' + /** Frozen 50/50 holdout for the Chat UI launch tip; control never sees it. Absent means not enrolled, so the tip shows. */ + chatUiLaunchTipCohort?: 'control' | 'treatment' /** Local UTC-day cursors cap dashboard observations across restarts; at most two writes a day. */ agentDashboardActiveDays?: Partial> }