Chat UI launch (#36891)

* feat(telemetry): A/B holdout for the Chat UI launch tip

Freeze a 50/50 chat_ui_launch_tip_v1 cohort for profiles still due the tip on
official builds with telemetry not disabled; control never sees the tip. Emit
chat_ui_feature_tip_exposure in both arms and agent_default_surface_changed
from the tip and from Settings -> Agents.

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(telemetry): hold the Chat UI launch tip until the telemetry banner resolves

Enroll only when consent is enabled. While the banner is pending, the tip is
skipped for that launch without being marked seen, so its exposure is not
spent unmeasured and the arm cannot influence the consent choice.

Co-Authored-By: Claude <noreply@anthropic.com>

* test(telemetry): document best-effort Chat UI tip exposure on rejected sends

Co-Authored-By: Claude <noreply@anthropic.com>

* test(telemetry): pass the exposure test through a hoisted mock instead of in-test importActual

Co-Authored-By: Claude <noreply@anthropic.com>

* test(telemetry): justify the fake Store assertion in the telemetry harness

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Neil
2026-11-01 00:08:13 -07:00
committed by GitHub
co-authored by Claude
parent 82c7b0c8b2
commit 4eccb84167
16 changed files with 417 additions and 4 deletions
+3 -1
View File
@@ -2,6 +2,7 @@ import { ipcMain } from 'electron'
import { sanitizeOnboardingUpdate, type Store } from '../persistence'
import type { OnboardingState } from '../../shared/onboarding-state-types'
import type { ChatUiLaunchTipVariant } from '../../shared/native-chat/chat-ui-launch-tip-audience'
import { isChatUiLaunchTipDeferredForConsent } from '../telemetry/chat-ui-launch-tip-cohort'
export function registerOnboardingHandlers(store: Store): void {
ipcMain.removeHandler('onboarding:get')
@@ -13,7 +14,8 @@ export function registerOnboardingHandlers(store: Store): void {
ipcMain.handle('onboarding:update', (_event, updates: unknown): OnboardingState => {
return store.updateOnboarding(sanitizeOnboardingUpdate(updates))
})
// 'none' skips the tip this launch without marking it seen, so it returns once consent resolves.
ipcMain.handle('onboarding:getChatUiLaunchTipVariant', (): ChatUiLaunchTipVariant =>
store.getChatUiLaunchTipVariant()
isChatUiLaunchTipDeferredForConsent() ? 'none' : store.getChatUiLaunchTipVariant()
)
}
@@ -0,0 +1,86 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { ChatUiLaunchTipVariant } from '../../shared/native-chat/chat-ui-launch-tip-audience'
import type { TelemetrySettings } from '../../shared/telemetry-settings-types'
import type { Store } from '../persistence'
import {
ensureChatUiLaunchTipCohort,
isChatUiLaunchTipDeferredForConsent
} from './chat-ui-launch-tip-cohort'
import { makeFakeSettings, restoreConsentEnv, stashAndClearConsentEnv } from './client-test-harness'
function makeStore(
telemetry: Partial<TelemetrySettings>,
variant: ChatUiLaunchTipVariant = 'chat-ui-off',
seen: string[] = []
): Store {
const settings = makeFakeSettings({
optedIn: true,
installId: 'install',
existedBeforeTelemetryRelease: true,
...telemetry
})
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: supplies every Store method the cohort assignment reads or writes.
return {
getSettings: () => settings,
getUI: () => ({ featureTipsSeenIds: seen }),
getChatUiLaunchTipVariant: () => variant,
updateSettings: vi.fn((updates: { telemetry: TelemetrySettings }) => {
settings.telemetry = updates.telemetry
})
} as unknown as Store
}
describe('ensureChatUiLaunchTipCohort', () => {
let env: Record<string, string | undefined>
beforeEach(() => {
env = stashAndClearConsentEnv()
})
afterEach(() => {
restoreConsentEnv(env)
})
it('enrolls a profile still due the tip once and keeps the frozen cohort', () => {
const store = makeStore({})
const cohort = ensureChatUiLaunchTipCohort(store)
expect(['control', 'treatment']).toContain(cohort)
expect(store.getSettings().telemetry?.chatUiLaunchTipCohort).toBe(cohort)
expect(store.getSettings().telemetry?.installId).toBe('install')
expect(ensureChatUiLaunchTipCohort(store)).toBe(cohort)
expect(store.updateSettings).toHaveBeenCalledTimes(1)
})
it('holds the tip unenrolled while the banner is pending, then enrolls after acknowledgement', () => {
const pending = makeStore({ optedIn: null })
expect(ensureChatUiLaunchTipCohort(pending)).toBeUndefined()
expect(isChatUiLaunchTipDeferredForConsent()).toBe(true)
expect(pending.updateSettings).not.toHaveBeenCalled()
// Acknowledging the banner persists optedIn=true; the next launch enrolls the still-unseen tip.
pending.getSettings().telemetry!.optedIn = true
const cohort = ensureChatUiLaunchTipCohort(pending)
expect(['control', 'treatment']).toContain(cohort)
expect(isChatUiLaunchTipDeferredForConsent()).toBe(false)
expect(pending.getSettings().telemetry?.chatUiLaunchTipCohort).toBe(cohort)
})
it('releases a pending profile unenrolled after the banner is declined, so the tip shows as before', () => {
const pending = makeStore({ optedIn: null })
ensureChatUiLaunchTipCohort(pending)
expect(isChatUiLaunchTipDeferredForConsent()).toBe(true)
pending.getSettings().telemetry!.optedIn = false
expect(ensureChatUiLaunchTipCohort(pending)).toBeUndefined()
expect(isChatUiLaunchTipDeferredForConsent()).toBe(false)
expect(pending.updateSettings).not.toHaveBeenCalled()
})
it.each([
['outside the audience', makeStore({}, 'none')],
['already shown the tip', makeStore({}, 'chat-ui-off', ['chat-ui-here'])],
['opted out', makeStore({ optedIn: false })]
])('leaves a profile %s unenrolled, so it keeps today’s behaviour', (_label, store) => {
expect(ensureChatUiLaunchTipCohort(store)).toBeUndefined()
expect(isChatUiLaunchTipDeferredForConsent()).toBe(false)
expect(store.updateSettings).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,49 @@
import { randomInt } from 'node:crypto'
import type { Store } from '../persistence'
import { resolveConsent } from './consent'
export type ChatUiLaunchTipCohort = 'control' | 'treatment'
// Decided once per launch with the cohort, so the tip and its enrollment read the same consent; contributor builds never set it.
let deferredForConsent = false
/**
* Enrolls once, only profiles still due the tip whose events can be measured. While the telemetry
* banner is unresolved the tip waits, so the exposure is never spent unmeasured; opted-out and
* contributor builds stay unenrolled and see the tip as before.
*/
export function ensureChatUiLaunchTipCohort(store: Store): ChatUiLaunchTipCohort | undefined {
deferredForConsent = false
try {
const settings = store.getSettings()
const telemetry = settings.telemetry
const saved = telemetry?.chatUiLaunchTipCohort
if (saved === 'control' || saved === 'treatment') {
return saved
}
if (
!telemetry ||
store.getChatUiLaunchTipVariant() === 'none' ||
store.getUI().featureTipsSeenIds?.includes('chat-ui-here')
) {
return undefined
}
const consent = resolveConsent(settings).effective
if (consent !== 'enabled') {
deferredForConsent = consent === 'pending_banner'
return undefined
}
const cohort: ChatUiLaunchTipCohort = randomInt(2) === 0 ? 'control' : 'treatment'
store.updateSettings({ telemetry: { ...telemetry, chatUiLaunchTipCohort: cohort } })
return cohort
} catch (err) {
// Startup must not fail over an experiment; unenrolled profiles see the tip as before.
console.warn('[telemetry] chat UI launch tip cohort assignment failed', err)
return undefined
}
}
/** True when this launch holds the tip back until the telemetry banner is resolved. */
export function isChatUiLaunchTipDeferredForConsent(): boolean {
return deferredForConsent
}
@@ -79,8 +79,11 @@ export function makeFakeSettings(telemetry: GlobalSettings['telemetry']): Global
}
export function makeFakeStore(settings: GlobalSettings): Store {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: fakes every Store method telemetry reads or writes; other methods are never called.
return {
getSettings: vi.fn(() => settings),
getUI: vi.fn(() => ({ featureTipsSeenIds: [] })),
getChatUiLaunchTipVariant: vi.fn(() => 'none'),
updateSettings: vi.fn((updates: Partial<GlobalSettings>) => {
if (updates.telemetry) {
settings.telemetry = {
+3
View File
@@ -6,6 +6,7 @@ import type { CommonProps, EventName, EventProps, OptInVia } from '../../shared/
import type { AgentDashboardSurface } from '../../shared/telemetry-agent-dashboard-event-schemas'
import type { Store } from '../persistence'
import { consumeBurstToken, resetBurstCapsForSession } from './burst-cap'
import { ensureChatUiLaunchTipCohort } from './chat-ui-launch-tip-cohort'
import { getCohortAtEmit } from './cohort-classifier'
import { isConsentEnvOverridden, resolveConsent } from './consent'
import * as postHogStart from './posthog-client-start'
@@ -73,6 +74,8 @@ export function initTelemetry(store: Store, scheduleStart: postHogStart.StartSch
if (languageCohort === 'control' || languageCohort === 'treatment') {
commonProps.onboarding_language_v1 = languageCohort
}
// Official builds only, so contributor builds always show the Chat UI launch tip.
commonProps.chat_ui_launch_tip_v1 = ensureChatUiLaunchTipCohort(store)
// Validate singleton common properties once; invalid identity would collapse installs into one subject.
const parsedCommon = commonPropsSchema.safeParse(commonProps)
@@ -0,0 +1,131 @@
// @vitest-environment happy-dom
import { act, type JSX } from 'react'
import { createRoot, type Root } from 'react-dom/client'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { FeatureTipId } from '../../../shared/feature-tips'
import { getDefaultOnboardingState } from '../../../shared/onboarding-defaults'
import { createGlobalSettingsFixture } from '../../../shared/global-settings-test-fixture'
import type { TelemetrySettings } from '../../../shared/telemetry-settings-types'
import type * as telemetry from '@/lib/telemetry'
import { track } from '@/lib/telemetry'
import { useAppStore } from '@/store'
import { useOnboardingAndFeatureTips } from './use-onboarding-and-feature-tips'
const telemetryMock = vi.hoisted(() => ({ passThrough: false }))
// Pass-through lets one test drive the real bridge wrapper against a rejecting main.
vi.mock('@/lib/telemetry', async (importOriginal) => {
const actual = await importOriginal<typeof telemetry>()
return {
track: vi.fn((...args: Parameters<typeof actual.track>) => {
if (telemetryMock.passThrough) {
actual.track(...args)
}
})
}
})
globalThis.IS_REACT_ACT_ENVIRONMENT = true
let container: HTMLDivElement
let root: Root
function Gate(): JSX.Element | null {
const gate = useOnboardingAndFeatureTips()
if (gate.onboarding === null) {
gate.applyStartupOnboardingState({ ...getDefaultOnboardingState(), closedAt: 1 })
}
return null
}
async function launch(cohort: TelemetrySettings['chatUiLaunchTipCohort']): Promise<void> {
const markFeatureTipsSeen = vi.fn((ids: FeatureTipId[]) =>
useAppStore.setState((state) => ({ featureTipsSeenIds: [...state.featureTipsSeenIds, ...ids] }))
)
useAppStore.setState({
persistedUIReady: true,
featureTipsSeenIds: [],
settings: createGlobalSettingsFixture({
aiVaultSearch: { enabled: true, historyDays: null },
telemetry: {
optedIn: true,
installId: 'install',
existedBeforeTelemetryRelease: true,
...(cohort ? { chatUiLaunchTipCohort: cohort } : {})
}
}),
markFeatureTipsSeen
})
await act(async () => root.render(<Gate />))
}
beforeEach(() => {
useAppStore.setState(useAppStore.getInitialState(), true)
vi.mocked(track).mockClear()
telemetryMock.passThrough = false
Object.defineProperty(window, 'api', {
configurable: true,
value: {
ui: { set: vi.fn().mockResolvedValue(undefined) },
cli: { getInstallStatus: vi.fn().mockResolvedValue({ supported: false }) },
onboarding: { getChatUiLaunchTipVariant: vi.fn().mockResolvedValue('chat-ui-off') }
}
})
container = document.createElement('div')
root = createRoot(container)
})
afterEach(() => {
act(() => root.unmount())
})
describe('Chat UI launch tip holdout', () => {
it.each([
[undefined, 'unassigned'],
['treatment', 'treatment']
] as const)('shows the tip to %s profiles after recording exposure', async (cohort, label) => {
await launch(cohort)
expect(useAppStore.getState().activeModal).toBe('feature-tips')
expect(useAppStore.getState().modalData).toMatchObject({ tipId: 'chat-ui-here' })
expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', {
variant: 'chat_ui_off',
launch_tip_cohort: label,
shown: true
})
expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_shown', { variant: 'chat_ui_off' })
})
it('records control exposure without showing the tip, then lets the next tip open', async () => {
await launch('control')
expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', {
variant: 'chat_ui_off',
launch_tip_cohort: 'control',
shown: false
})
expect(track).not.toHaveBeenCalledWith('chat_ui_feature_tip_shown', expect.anything())
expect(useAppStore.getState().featureTipsSeenIds).toContain('chat-ui-here')
expect(useAppStore.getState().activeModal).toBe('feature-tips')
expect(useAppStore.getState().modalData).toMatchObject({ tipId: 'cmd-j-palette' })
})
// Exposure is best-effort: a rejected capture (consent flip, session budget, IPC failure) is not
// retried, and the tip stays consumed so a failed send never shows it twice.
it.each(['treatment', 'control'] as const)(
'keeps the %s tip consumed when main rejects the exposure event',
async (cohort) => {
telemetryMock.passThrough = true
const telemetryTrack = vi.fn().mockResolvedValue(false)
Object.assign(window.api, { telemetryTrack })
await launch(cohort)
expect(telemetryTrack).toHaveBeenCalledWith('chat_ui_feature_tip_exposure', {
variant: 'chat_ui_off',
launch_tip_cohort: cohort,
shown: cohort === 'treatment'
})
expect(useAppStore.getState().featureTipsSeenIds).toContain('chat-ui-here')
expect(useAppStore.getState().modalData).toMatchObject({
tipId: cohort === 'treatment' ? 'chat-ui-here' : 'cmd-j-palette'
})
}
)
})
@@ -7,6 +7,8 @@ import {
isCliFeatureTipCompleted
} from '../components/feature-tips/feature-tip-startup-gate'
import {
isChatUiLaunchTipHeldOut,
trackChatUiFeatureTipExposure,
trackChatUiFeatureTipShown,
trackCmdJPaletteFeatureTipShown,
trackOrcaCliFeatureTipShown
@@ -140,6 +142,15 @@ export function useOnboardingAndFeatureTips() {
return
}
if (featureTipsDecision.tipId === 'chat-ui-here' && chatUiLaunchTipVariant) {
trackChatUiFeatureTipExposure(chatUiLaunchTipVariant, settings ?? null)
if (isChatUiLaunchTipHeldOut(settings ?? null)) {
// Control behaves as if the tip never existed, so the next pending tip may take this launch.
actions.markFeatureTipsSeen(['chat-ui-here'])
return
}
}
promptedThisSessionRef.current = true
if (featureTipsDecision.tipId === 'orca-cli') {
trackOrcaCliFeatureTipShown('app_open')
@@ -200,10 +200,16 @@ describe('Chat UI is here tip', () => {
expect(useAppStore.getState().activeModal).toBe('none')
expect(updateSettingsOrThrow).toHaveBeenCalledExactlyOnceWith({ agentDefaultSurface: 'chat' })
expect(updateSettings).not.toHaveBeenCalled()
expect(track).toHaveBeenLastCalledWith('chat_ui_feature_tip_action', {
expect(track).toHaveBeenCalledWith('chat_ui_feature_tip_action', {
variant: 'chat_ui_off',
action: 'use_chat_ui'
})
expect(track).toHaveBeenLastCalledWith('agent_default_surface_changed', {
from_mode: 'terminal',
to_mode: 'chat',
source: 'launch_card',
launch_tip_shown: true
})
})
it('offers chat users a single Got it that closes without saving', async () => {
@@ -1,6 +1,7 @@
import { useEffect, useRef, useState, type JSX } from 'react'
import { toast } from 'sonner'
import { getDefaultVoiceSettings } from '../../../../shared/constants'
import { normalizeAgentDefaultSurface } from '../../../../shared/native-chat/native-chat-providers'
import {
ORCHESTRATION_ENABLED_STORAGE_KEY,
ORCHESTRATION_SETUP_DISMISSED_STORAGE_KEY,
@@ -15,6 +16,7 @@ import { installCliFromFeatureTip } from './feature-tip-cli-install-action'
import { getFeatureTipForModal } from './feature-tip-modal-state'
import {
getOrcaCliFeatureTipTelemetrySource,
trackAgentDefaultSurfaceChanged,
trackChatUiFeatureTipAction,
trackCmdJPaletteFeatureTipAcknowledged,
trackOrcaCliFeatureTipSetupClicked,
@@ -155,10 +157,17 @@ export default function FeatureTipsModal(): JSX.Element | null {
trackChatUiFeatureTipAction(chatUiLaunchTipVariant, 'dismissed')
break
}
const fromSurface = normalizeAgentDefaultSurface(settings?.agentDefaultSurface)
try {
// Only the default moves; an agent the user set for itself keeps its own choice.
await updateSettingsOrThrow({ agentDefaultSurface: 'chat' })
trackChatUiFeatureTipAction(chatUiLaunchTipVariant, 'use_chat_ui')
trackAgentDefaultSurfaceChanged({
from_mode: fromSurface,
to_mode: 'chat',
source: 'launch_card',
launch_tip_shown: true
})
} catch {
toast.error(
translate(
@@ -1,6 +1,8 @@
import { track } from '@/lib/telemetry'
import type { EventProps } from '../../../../shared/telemetry-events'
import type { ChatUiLaunchTipVariant } from '../../../../shared/native-chat/chat-ui-launch-tip-audience'
import type { FeatureTipId } from '../../../../shared/feature-tips'
import type { GlobalSettings } from '../../../../shared/global-settings-types'
export type OrcaCliFeatureTipSource = EventProps<'orca_cli_feature_tip_shown'>['source']
export type OrcaCliFeatureTipSetupResult = EventProps<'orca_cli_feature_tip_setup_result'>['result']
@@ -58,3 +60,38 @@ export function trackChatUiFeatureTipAction(
track('chat_ui_feature_tip_action', { variant: CHAT_UI_TIP_TELEMETRY_VARIANT[variant], action })
}
}
/** Control profiles never see the Chat UI launch tip; unenrolled ones (no consent, contributor builds) do. */
export function isChatUiLaunchTipHeldOut(
settings: Pick<GlobalSettings, 'telemetry'> | null
): boolean {
return settings?.telemetry?.chatUiLaunchTipCohort === 'control'
}
export function wasChatUiLaunchTipShown(
seenTipIds: readonly FeatureTipId[],
settings: Pick<GlobalSettings, 'telemetry'> | null
): boolean {
return seenTipIds.includes('chat-ui-here') && !isChatUiLaunchTipHeldOut(settings)
}
export function trackChatUiFeatureTipExposure(
variant: ChatUiLaunchTipVariant,
settings: Pick<GlobalSettings, 'telemetry'> | null
): void {
if (variant !== 'none') {
track('chat_ui_feature_tip_exposure', {
variant: CHAT_UI_TIP_TELEMETRY_VARIANT[variant],
launch_tip_cohort: settings?.telemetry?.chatUiLaunchTipCohort ?? 'unassigned',
shown: !isChatUiLaunchTipHeldOut(settings)
})
}
}
export function trackAgentDefaultSurfaceChanged(
props: EventProps<'agent_default_surface_changed'>
): void {
if (props.from_mode !== props.to_mode) {
track('agent_default_surface_changed', props)
}
}
@@ -4,6 +4,7 @@ import { toast } from 'sonner'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createGlobalSettingsFixture } from '../../../../shared/global-settings-test-fixture'
import type { GlobalSettings } from '../../../../shared/global-settings-types'
import { track } from '@/lib/telemetry'
import { useAppStore } from '@/store'
import { saveNativeChatProviderChoice } from './native-chat-provider-choice'
import { AgentDefaultSurfaceSection } from './AgentDefaultSurfaceSection'
@@ -11,6 +12,7 @@ import { AgentDefaultSurfaceSection } from './AgentDefaultSurfaceSection'
vi.mock('@/lib/chat-provider-agents', () => ({
chatProviderAgents: () => ['claude', 'codex', 'grok', 'opencode']
}))
vi.mock('@/lib/telemetry', () => ({ track: vi.fn() }))
vi.mock('sonner', () => ({ toast: { success: vi.fn(), dismiss: vi.fn(), error: vi.fn() } }))
let persisted: GlobalSettings
@@ -33,6 +35,7 @@ function renderSection(settings: Partial<GlobalSettings>): void {
beforeEach(() => {
useAppStore.setState(useAppStore.getInitialState(), true)
settingsSet.mockClear()
vi.mocked(track).mockClear()
vi.mocked(toast.success).mockClear()
vi.mocked(toast.error).mockClear()
Object.defineProperty(window, 'api', {
@@ -86,6 +89,37 @@ describe('Open agents as', () => {
})
})
describe('Chat UI launch tip experiment', () => {
it.each([
[[], 'treatment', false],
[['chat-ui-here'], 'treatment', true],
[['chat-ui-here'], 'control', false]
] as const)(
'reports a switch to Chat with seen tips %j in %s as launch_tip_shown=%s',
async (seen, cohort, shown) => {
useAppStore.setState({ featureTipsSeenIds: [...seen] })
renderSection({
agentDefaultSurface: 'terminal',
telemetry: {
optedIn: true,
installId: 'install',
existedBeforeTelemetryRelease: true,
chatUiLaunchTipCohort: cohort
}
})
fireEvent.click(screen.getByRole('radio', { name: /^Chat/ }))
await waitFor(() =>
expect(track).toHaveBeenCalledExactlyOnceWith('agent_default_surface_changed', {
from_mode: 'terminal',
to_mode: 'chat',
source: 'settings',
launch_tip_shown: shown
})
)
}
)
})
describe('acknowledged default writes', () => {
it('reports a rejected tile save once, preserves the choice, and accepts its successor', async () => {
renderSection({ agentDefaultSurface: 'terminal' })
@@ -12,6 +12,10 @@ import { getIntlLocale, translate } from '@/i18n/i18n'
import { getAgentLabel } from '@/lib/agent-catalog'
import { cn } from '@/lib/utils'
import { useAppStore } from '@/store'
import {
trackAgentDefaultSurfaceChanged,
wasChatUiLaunchTipShown
} from '../feature-tips/feature-tip-telemetry'
import { ChatSurfacePreview, TerminalSurfacePreview } from './AgentDefaultSurfacePreviews'
import { AGENT_DEFAULT_SURFACE_SECTION_ID, AGENTS_INSTALLED_SECTION_ID } from './agents-search'
import {
@@ -175,7 +179,18 @@ export function AgentDefaultSurfaceSection({
const writer = { fallbackSettings: settings, updateSettings }
void mutation.run('default-surface', {
mutate: () => saveAgentDefaultSurface({ surface: next, ...writer }),
onSuccess: () => notify(next, writer),
onSuccess: () => {
trackAgentDefaultSurfaceChanged({
from_mode: surface,
to_mode: next,
source: 'settings',
launch_tip_shown: wasChatUiLaunchTipShown(
useAppStore.getState().featureTipsSeenIds,
settings
)
})
notify(next, writer)
},
onError: saveFailed
})
}
+4 -1
View File
@@ -60,6 +60,8 @@ type _CohortExtendedRoster =
| 'cmd_j_palette_feature_tip_acknowledged'
| 'chat_ui_feature_tip_shown'
| 'chat_ui_feature_tip_action'
| 'chat_ui_feature_tip_exposure'
| 'agent_default_surface_changed'
// Strict empty payloads infer string index signatures; exclude those from keyed rosters.
type _KnownPayloadKeys<T> = string extends keyof T ? never : keyof T
type _DerivedCohortExtendedEvents = {
@@ -128,7 +130,8 @@ export const commonPropsSchema = z
install_id: z.string().min(1).max(64),
session_id: z.string().min(1).max(64),
orca_channel: z.enum(['stable', 'rc']),
onboarding_language_v1: z.enum(['control', 'treatment']).optional()
onboarding_language_v1: z.enum(['control', 'treatment']).optional(),
chat_ui_launch_tip_v1: z.enum(['control', 'treatment']).optional()
})
.strict()
export type CommonProps = z.infer<typeof commonPropsSchema>
+4
View File
@@ -50,6 +50,8 @@ import {
} from './telemetry-feature-education-event-schemas'
import {
chatUiFeatureTipActionSchema,
chatUiFeatureTipExposureSchema,
agentDefaultSurfaceChangedSchema,
chatUiFeatureTipShownSchema,
cmdJPaletteFeatureTipAcknowledgedSchema,
cmdJPaletteFeatureTipShownSchema,
@@ -185,6 +187,8 @@ export const eventSchemas = {
cmd_j_palette_feature_tip_acknowledged: cmdJPaletteFeatureTipAcknowledgedSchema,
chat_ui_feature_tip_shown: chatUiFeatureTipShownSchema,
chat_ui_feature_tip_action: chatUiFeatureTipActionSchema,
chat_ui_feature_tip_exposure: chatUiFeatureTipExposureSchema,
agent_default_surface_changed: agentDefaultSurfaceChangedSchema,
feature_wall_opened: featureWallOpenedSchema,
feature_wall_closed: featureWallClosedSchema,
@@ -96,6 +96,24 @@ export const chatUiFeatureTipActionSchema = z
nth_repo_added: nthRepoAddedSchema
})
.strict()
// Fires in both arms when the tip would open, so control has the same exposure moment as treatment.
export const chatUiFeatureTipExposureSchema = z
.object({
variant: chatUiFeatureTipVariantSchema,
launch_tip_cohort: z.enum(['control', 'treatment', 'unassigned']),
shown: z.boolean(),
nth_repo_added: nthRepoAddedSchema
})
.strict()
export const agentDefaultSurfaceChangedSchema = z
.object({
from_mode: nativeChatViewModeSchema,
to_mode: nativeChatViewModeSchema,
source: z.enum(['launch_card', 'settings']),
launch_tip_shown: z.boolean(),
nth_repo_added: nthRepoAddedSchema
})
.strict()
export const featureWallOpenedSchema = z
.object({
+2
View File
@@ -9,6 +9,8 @@ export type TelemetrySettings = {
existedBeforeTelemetryRelease: boolean
/** Frozen 50/50 assignment for fresh installs; absent for profiles predating the experiment. */
onboardingLanguageCohort?: 'control' | 'treatment'
/** Frozen 50/50 holdout for the Chat UI launch tip; control never sees it. Absent means not enrolled, so the tip shows. */
chatUiLaunchTipCohort?: 'control' | 'treatment'
/** Local UTC-day cursors cap dashboard observations across restarts; at most two writes a day. */
agentDashboardActiveDays?: Partial<Record<AgentDashboardSurface, number>>
}