From 76caab284b0641ba06dcf951efb69b00135be5b6 Mon Sep 17 00:00:00 2001 From: brennanb2025 Date: Mon, 25 May 2026 16:08:25 -0700 Subject: [PATCH] Limit contextual tours to new users Co-authored-by: Orca --- .../feature-education-retention-analytics.md | 19 +++--- docs/reference/feature-education-state.md | 12 ++++ .../runtime/rpc/methods/client-ui.test.ts | 6 +- src/main/runtime/rpc/methods/client-ui.ts | 3 +- src/renderer/src/App.tsx | 11 ++++ .../contextual-tour-gate.test.ts | 21 +++++++ .../contextual-tours/contextual-tour-gate.ts | 5 ++ .../contextual-tours/use-contextual-tour.ts | 3 + src/renderer/src/store/slices/ui.test.ts | 62 +++++++++++++++++-- src/renderer/src/store/slices/ui.ts | 18 ++++++ src/shared/types.ts | 3 + .../e2e-completed-onboarding-profile.ts | 3 +- 12 files changed, 149 insertions(+), 17 deletions(-) diff --git a/docs/reference/feature-education-retention-analytics.md b/docs/reference/feature-education-retention-analytics.md index 5804499a4d9..2bc697a4c5a 100644 --- a/docs/reference/feature-education-retention-analytics.md +++ b/docs/reference/feature-education-retention-analytics.md @@ -13,6 +13,8 @@ Do not add broad telemetry that mirrors local education state, such as "feature Feature adoption analysis should join tour cohorts to existing downstream product events. Add new telemetry only for concrete user actions that are not already represented by an existing event. +Automatic contextual tours are limited to new users for this rollout. The local `contextualToursAutoEligible` flag is set once after persisted UI and onboarding state load: users still in first-run onboarding become eligible; users with closed onboarding become ineligible. This avoids surprising existing users with education for surfaces they may already understand. + ## Product Questions Use the data to answer: @@ -147,7 +149,7 @@ At 1,000 DAU, this PR should emit at most: - One `contextual_tour_outcome` per shown tour - Currently one contextual tour per session -Expected launch volume: under 2,000 events/day at 1,000 DAU, decaying as `contextualToursSeenIds` fills in. +Expected launch volume: under 2,000 events/day at 1,000 DAU and lower for mature cohorts because existing users are not auto-eligible and `contextualToursSeenIds` fills in for eligible new users. Do not add recurring, render-loop, polling, passive snapshot, or local-state-mirror events for this analysis. @@ -168,12 +170,13 @@ Payloads must remain low-cardinality and bounded. Do not include: ## Existing Users -Existing users should not receive a startup blast of education. Tours should remain contextual: +Existing users should not receive automatic contextual tours from this rollout. We cannot reliably know whether they have already viewed or used every covered surface, so the least surprising default is to leave them alone and measure this launch on new-user cohorts. -- at most one tour per session -- only after the user enters the relevant surface -- suppressed during onboarding and blocking dialogs -- suppressed after the tour has already been surfaced locally -- suppressed for surfaces where local state proves the user has already interacted, where practical +Implementation: -For existing-user analysis, compare users by actual tour exposure and outcome, not by install age alone. +- `contextualToursAutoEligible: false` for profiles whose onboarding is already closed when the rollout first runs +- `contextualToursAutoEligible: true` for profiles still in first-run onboarding when the rollout first runs +- automatic tour requests require `contextualToursAutoEligible === true` +- local `featureInteractions` still records supported surface entry for all users after UI hydration + +For existing-user analysis, compare users by actual tour exposure and outcome only when they have exposure. Most existing users should fall into the no-tour baseline for this PR. If a later release adds manual tour entry points or truly new feature education, evaluate that release separately instead of reusing this rollout's eligibility rule. diff --git a/docs/reference/feature-education-state.md b/docs/reference/feature-education-state.md index 547f33054f4..1896c7b8dc4 100644 --- a/docs/reference/feature-education-state.md +++ b/docs/reference/feature-education-state.md @@ -14,6 +14,15 @@ Set this when a restart tip is opened, dismissed, or acted on. It prevents the s Set this after a tour has rendered a measured target, or when the user skips/completes the tour. It prevents the same tour from showing again. It does not mean the user used the feature. +`contextualToursAutoEligible` answers: "May this profile receive automatic contextual tours from this rollout?" + +Missing means Orca has not classified the profile yet. On first hydrated launch after this change, the renderer sets it once: + +- `true` when first-run onboarding is still open, which covers new users from this rollout forward +- `false` when onboarding is already closed, which covers existing users + +Automatic contextual tours require this flag to be `true`. Existing users still get local feature interaction state recorded when they enter supported surfaces, but they are not auto-toured by this rollout. + `featureInteractions` answers: "Has the user actually interacted with this feature?" This is the product-state signal for already-discovered features. It lives in `PersistedUIState.featureInteractions`, keyed by `FeatureInteractionId`, and stores the first local interaction timestamp: @@ -37,6 +46,7 @@ Use this to suppress tips or tours that would teach a feature the user has alrea - Do not rename or reuse ids. If semantics change materially, add a new id and leave the old one readable. - Unknown or malformed persisted ids are ignored during hydration for forward/backward compatibility. - This state is local product state, not telemetry. Analytics should use the bounded feature-education telemetry events (`contextual_tour_shown` and `contextual_tour_outcome`) plus existing downstream action events, rather than reading or uploading the persisted state blob. +- Automatic contextual tours are new-user-only for this rollout. Do not reset `contextualToursAutoEligible` for existing users unless the product decision changes for a later release. ## Adding A Tip @@ -60,3 +70,5 @@ For contextual tours in this branch, `useContextualTour(...)` records the matchi ## Test Profiles Completed-onboarding E2E profiles should preseed both education exposure and feature interaction state. That keeps first-run education from covering unrelated UI under test while preserving production behavior for real profiles. + +Profiles that model existing users should also set `contextualToursAutoEligible: false`; profiles that explicitly exercise contextual tours can set it to `true`. diff --git a/src/main/runtime/rpc/methods/client-ui.test.ts b/src/main/runtime/rpc/methods/client-ui.test.ts index 886df3d4111..321dc43e7a8 100644 --- a/src/main/runtime/rpc/methods/client-ui.test.ts +++ b/src/main/runtime/rpc/methods/client-ui.test.ts @@ -159,7 +159,8 @@ describe('client UI RPC methods', () => { featureInteractions: { tasks: { firstInteractedAt: 100 } }, - contextualToursSeenIds: ['tasks'] + contextualToursSeenIds: ['tasks'], + contextualToursAutoEligible: true } const runtime = { getRuntimeId: () => 'test-runtime', @@ -191,7 +192,8 @@ describe('client UI RPC methods', () => { featureInteractions: { tasks: { firstInteractedAt: 100 } }, - contextualToursSeenIds: ['tasks'] + contextualToursSeenIds: ['tasks'], + contextualToursAutoEligible: true } const response = await dispatcher.dispatch(makeRequest('ui.set', payload)) diff --git a/src/main/runtime/rpc/methods/client-ui.ts b/src/main/runtime/rpc/methods/client-ui.ts index 54989e4a717..ea6feba6d28 100644 --- a/src/main/runtime/rpc/methods/client-ui.ts +++ b/src/main/runtime/rpc/methods/client-ui.ts @@ -178,7 +178,8 @@ const UiUpdate = z workspaceCleanup: WorkspaceCleanup.optional(), featureTipsSeenIds: StringArray.optional(), featureInteractions: FeatureInteractions.optional(), - contextualToursSeenIds: StringArray.optional() + contextualToursSeenIds: StringArray.optional(), + contextualToursAutoEligible: z.boolean().optional() }) .strict() .default({}) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 2d754e99607..088853cd824 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -287,6 +287,7 @@ function App(): React.JSX.Element { openModal: s.openModal, closeModal: s.closeModal, markFeatureTipsSeen: s.markFeatureTipsSeen, + setContextualToursAutoEligible: s.setContextualToursAutoEligible, setContextualToursOnboardingVisible: s.setContextualToursOnboardingVisible, cancelContextualTour: s.cancelContextualTour, toggleRightSidebar: s.toggleRightSidebar, @@ -305,6 +306,7 @@ function App(): React.JSX.Element { const activeModal = useAppStore((s) => s.activeModal) const featureTipsSeenIds = useAppStore((s) => s.featureTipsSeenIds) const featureInteractions = useAppStore((s) => s.featureInteractions) + const contextualToursAutoEligible = useAppStore((s) => s.contextualToursAutoEligible) const activeWorktreeId = useAppStore((s) => s.activeWorktreeId) // Why: App swaps the sidebar between workspace and landing layouts when the // active workspace is slept/deleted. Keep virtualized scroll memory above @@ -463,6 +465,15 @@ function App(): React.JSX.Element { } }, [actions, onboarding, onboardingLoaded]) + useEffect(() => { + if (!persistedUIReady || !onboardingLoaded || contextualToursAutoEligible !== null) { + return + } + // Why: this rollout is for users who are still in first-run onboarding. + // Existing profiles are locally classified once and never auto-toured. + actions.setContextualToursAutoEligible(shouldShowOnboarding(onboarding)) + }, [actions, contextualToursAutoEligible, onboarding, onboardingLoaded, persistedUIReady]) + useEffect(() => { const featureTipsDecision = getFeatureTipsAppOpenDecision({ activeModal, diff --git a/src/renderer/src/components/contextual-tours/contextual-tour-gate.test.ts b/src/renderer/src/components/contextual-tours/contextual-tour-gate.test.ts index 31d16763d94..f3b077392b6 100644 --- a/src/renderer/src/components/contextual-tours/contextual-tour-gate.test.ts +++ b/src/renderer/src/components/contextual-tours/contextual-tour-gate.test.ts @@ -26,6 +26,7 @@ describe('contextual tour gate', () => { const decision = getContextualTourRequestDecision({ tour: getContextualTour('tasks'), persistedUIReady: true, + autoEligible: true, onboardingVisible: false, seenIds: [], sessionConsumed: false, @@ -119,6 +120,7 @@ describe('contextual tour gate', () => { getContextualTourRequestDecision({ tour, persistedUIReady: true, + autoEligible: true, onboardingVisible: false, seenIds: [], sessionConsumed: false, @@ -133,6 +135,7 @@ describe('contextual tour gate', () => { getContextualTourRequestDecision({ tour, persistedUIReady: false, + autoEligible: true, onboardingVisible: false, seenIds: [], sessionConsumed: false, @@ -147,6 +150,22 @@ describe('contextual tour gate', () => { getContextualTourRequestDecision({ tour, persistedUIReady: true, + autoEligible: false, + onboardingVisible: false, + seenIds: [], + sessionConsumed: false, + activeTourId: null, + activeModal: 'none', + blockingSurfaceVisible: false, + targetExists: hasFirstTarget + }) + ).toEqual({ kind: 'blocked', reason: 'auto-disabled' }) + + expect( + getContextualTourRequestDecision({ + tour, + persistedUIReady: true, + autoEligible: true, onboardingVisible: true, seenIds: [], sessionConsumed: false, @@ -161,6 +180,7 @@ describe('contextual tour gate', () => { getContextualTourRequestDecision({ tour, persistedUIReady: true, + autoEligible: true, onboardingVisible: false, seenIds: ['tasks'], sessionConsumed: false, @@ -175,6 +195,7 @@ describe('contextual tour gate', () => { getContextualTourRequestDecision({ tour, persistedUIReady: true, + autoEligible: true, onboardingVisible: false, seenIds: [], sessionConsumed: false, diff --git a/src/renderer/src/components/contextual-tours/contextual-tour-gate.ts b/src/renderer/src/components/contextual-tours/contextual-tour-gate.ts index d927a9950ee..10b53c89677 100644 --- a/src/renderer/src/components/contextual-tours/contextual-tour-gate.ts +++ b/src/renderer/src/components/contextual-tours/contextual-tour-gate.ts @@ -11,6 +11,7 @@ export type ContextualTourRequestDecision = kind: 'blocked' reason: | 'persisted-ui-not-ready' + | 'auto-disabled' | 'onboarding' | 'seen' | 'session-consumed' @@ -125,6 +126,7 @@ export function getNextVisibleContextualTourStepIndex(args: { export function getContextualTourRequestDecision(args: { tour: ContextualTour persistedUIReady: boolean + autoEligible: boolean onboardingVisible: boolean seenIds: readonly string[] sessionConsumed: boolean @@ -136,6 +138,9 @@ export function getContextualTourRequestDecision(args: { if (!args.persistedUIReady) { return { kind: 'blocked', reason: 'persisted-ui-not-ready' } } + if (!args.autoEligible) { + return { kind: 'blocked', reason: 'auto-disabled' } + } if (args.onboardingVisible) { return { kind: 'blocked', reason: 'onboarding' } } diff --git a/src/renderer/src/components/contextual-tours/use-contextual-tour.ts b/src/renderer/src/components/contextual-tours/use-contextual-tour.ts index ae263f05a39..0dd5ee92db3 100644 --- a/src/renderer/src/components/contextual-tours/use-contextual-tour.ts +++ b/src/renderer/src/components/contextual-tours/use-contextual-tour.ts @@ -24,6 +24,7 @@ export function useContextualTour( const activeContextualTourId = useAppStore((s) => s.activeContextualTourId) const activeContextualTourSource = useAppStore((s) => s.activeContextualTourSource) const contextualToursSeenIds = useAppStore((s) => s.contextualToursSeenIds) + const contextualToursAutoEligible = useAppStore((s) => s.contextualToursAutoEligible) const contextualTourShownThisSession = useAppStore((s) => s.contextualTourShownThisSession) const contextualToursOnboardingVisible = useAppStore((s) => s.contextualToursOnboardingVisible) const contextualToursBlockingSurfaceVisible = useAppStore( @@ -57,6 +58,7 @@ export function useContextualTour( typeof window === 'undefined' || typeof document === 'undefined' || !persistedUIReady || + contextualToursAutoEligible !== true || contextualToursOnboardingVisible || contextualToursBlockingSurfaceVisible || activeContextualTourId !== null || @@ -113,6 +115,7 @@ export function useContextualTour( activeContextualTourId, contextualToursBlockingSurfaceVisible, activeModal, + contextualToursAutoEligible, contextualTourShownThisSession, contextualToursOnboardingVisible, contextualToursSeenIds, diff --git a/src/renderer/src/store/slices/ui.test.ts b/src/renderer/src/store/slices/ui.test.ts index b03fba81d7a..9aa238fc469 100644 --- a/src/renderer/src/store/slices/ui.test.ts +++ b/src/renderer/src/store/slices/ui.test.ts @@ -853,6 +853,13 @@ function stubContextualTourTargets(selectors: readonly string[]): void { } describe('createUISlice contextual tours', () => { + function makeAutoTourEligibleUI(overrides: Partial = {}): PersistedUIState { + return makePersistedUI({ + contextualToursAutoEligible: true, + ...overrides + }) + } + it('normalizes persisted contextual tour ids during hydration', () => { const store = createUIStore() @@ -865,6 +872,40 @@ describe('createUISlice contextual tours', () => { expect(store.getState().contextualToursSeenIds).toEqual(['tasks', 'browser']) }) + it('normalizes persisted contextual tour auto eligibility during hydration', () => { + const store = createUIStore() + + store.getState().hydratePersistedUI(makePersistedUI()) + expect(store.getState().contextualToursAutoEligible).toBeNull() + + store.getState().hydratePersistedUI(makePersistedUI({ contextualToursAutoEligible: false })) + expect(store.getState().contextualToursAutoEligible).toBe(false) + + store + .getState() + .hydratePersistedUI(makePersistedUI({ contextualToursAutoEligible: 'yes' as never })) + expect(store.getState().contextualToursAutoEligible).toBeNull() + }) + + it('persists contextual tour auto eligibility once classified', () => { + const setMock = vi.fn(() => Promise.resolve()) + vi.stubGlobal('window', { + api: { + ui: { + set: setMock + } + } + }) + const store = createUIStore() + + store.getState().setContextualToursAutoEligible(false) + store.getState().setContextualToursAutoEligible(false) + + expect(store.getState().contextualToursAutoEligible).toBe(false) + expect(setMock).toHaveBeenCalledTimes(1) + expect(setMock).toHaveBeenCalledWith({ contextualToursAutoEligible: false }) + }) + it('marks contextual tours seen and persists them once', () => { const setMock = vi.fn(() => Promise.resolve()) vi.stubGlobal('window', { @@ -892,7 +933,7 @@ describe('createUISlice contextual tours', () => { store.getState().requestContextualTour('tasks', 'tasks_open') expect(store.getState().activeContextualTourId).toBeNull() - store.getState().hydratePersistedUI(makePersistedUI()) + store.getState().hydratePersistedUI(makeAutoTourEligibleUI()) store.getState().requestContextualTour('tasks', 'tasks_open') expect(store.getState().activeContextualTourId).toBe('tasks') @@ -905,7 +946,7 @@ describe('createUISlice contextual tours', () => { it('does not mark seen when the required first target is absent', () => { const store = createUIStore() stubContextualTourTargets([]) - store.getState().hydratePersistedUI(makePersistedUI()) + store.getState().hydratePersistedUI(makeAutoTourEligibleUI()) store.getState().requestContextualTour('tasks', 'tasks_open') @@ -917,7 +958,7 @@ describe('createUISlice contextual tours', () => { it('does not start while a root confirmation surface is visible', () => { const store = createUIStore() stubContextualTourTargets(['[data-contextual-tour-target="tasks-source-filters"]']) - store.getState().hydratePersistedUI(makePersistedUI()) + store.getState().hydratePersistedUI(makeAutoTourEligibleUI()) store.getState().setContextualToursBlockingSurfaceVisible(true) store.getState().requestContextualTour('tasks', 'tasks_open') @@ -926,13 +967,24 @@ describe('createUISlice contextual tours', () => { expect(store.getState().contextualTourShownThisSession).toBe(false) }) + it('does not auto-start tours for profiles that are not eligible', () => { + const store = createUIStore() + stubContextualTourTargets(['[data-contextual-tour-target="tasks-source-filters"]']) + store.getState().hydratePersistedUI(makePersistedUI({ contextualToursAutoEligible: false })) + + store.getState().requestContextualTour('tasks', 'tasks_open') + + expect(store.getState().activeContextualTourId).toBeNull() + expect(store.getState().contextualTourShownThisSession).toBe(false) + }) + it('allows only workspace creation over its creation modals', () => { const store = createUIStore() stubContextualTourTargets([ '[data-contextual-tour-target="workspace-creation-source"]', '[data-contextual-tour-target="tasks-source-filters"]' ]) - store.getState().hydratePersistedUI(makePersistedUI()) + store.getState().hydratePersistedUI(makeAutoTourEligibleUI()) store.getState().openModal('new-workspace-composer') store.getState().requestContextualTour('tasks', 'tasks_open') @@ -949,7 +1001,7 @@ describe('createUISlice contextual tours', () => { '[data-contextual-tour-target="browser-annotation-control"]' ] stubContextualTourTargets(visibleSelectors) - store.getState().hydratePersistedUI(makePersistedUI()) + store.getState().hydratePersistedUI(makeAutoTourEligibleUI()) store.getState().requestContextualTour('browser', 'browser_visible') store.getState().advanceContextualTour() diff --git a/src/renderer/src/store/slices/ui.ts b/src/renderer/src/store/slices/ui.ts index 50e5636164b..3d67c660cac 100644 --- a/src/renderer/src/store/slices/ui.ts +++ b/src/renderer/src/store/slices/ui.ts @@ -463,12 +463,14 @@ export type UISlice = { featureInteractions: FeatureInteractionState recordFeatureInteraction: (id: FeatureInteractionId) => void contextualToursSeenIds: ContextualTourId[] + contextualToursAutoEligible: boolean | null activeContextualTourId: ContextualTourId | null activeContextualTourStepIndex: number activeContextualTourSource: string | null contextualTourShownThisSession: boolean contextualToursOnboardingVisible: boolean contextualToursBlockingSurfaceVisible: boolean + setContextualToursAutoEligible: (eligible: boolean) => void setContextualToursOnboardingVisible: (visible: boolean) => void setContextualToursBlockingSurfaceVisible: (visible: boolean) => void requestContextualTour: (id: ContextualTourId, source: string) => void @@ -932,12 +934,23 @@ export const createUISlice: StateCreator = (set, get) return { featureInteractions: next } }), contextualToursSeenIds: [], + contextualToursAutoEligible: null, activeContextualTourId: null, activeContextualTourStepIndex: 0, activeContextualTourSource: null, contextualTourShownThisSession: false, contextualToursOnboardingVisible: false, contextualToursBlockingSurfaceVisible: false, + setContextualToursAutoEligible: (eligible) => + set((s) => { + if (s.contextualToursAutoEligible === eligible) { + return s + } + if (typeof window !== 'undefined') { + window.api.ui.set({ contextualToursAutoEligible: eligible }).catch(console.error) + } + return { contextualToursAutoEligible: eligible } + }), setContextualToursOnboardingVisible: (visible) => set((s) => s.contextualToursOnboardingVisible === visible @@ -956,6 +969,7 @@ export const createUISlice: StateCreator = (set, get) const decision = getContextualTourRequestDecision({ tour, persistedUIReady: s.persistedUIReady, + autoEligible: s.contextualToursAutoEligible === true, onboardingVisible: s.contextualToursOnboardingVisible, seenIds: s.contextualToursSeenIds, sessionConsumed: s.contextualTourShownThisSession, @@ -1399,6 +1413,10 @@ export const createUISlice: StateCreator = (set, get) featureTipsSeenIds: normalizeFeatureTipIds(ui.featureTipsSeenIds), featureInteractions: normalizeFeatureInteractions(ui.featureInteractions), contextualToursSeenIds: normalizeContextualTourIds(ui.contextualToursSeenIds), + contextualToursAutoEligible: + typeof ui.contextualToursAutoEligible === 'boolean' + ? ui.contextualToursAutoEligible + : null, trustedOrcaHooks: filterTrustedOrcaHooksToValidRepos( ui.trustedOrcaHooks ?? {}, validRepoIds diff --git a/src/shared/types.ts b/src/shared/types.ts index fb52abee7b0..8fe0dcc5fa6 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -2220,6 +2220,9 @@ export type PersistedUIState = { /** Contextual tours already surfaced to the user. Unknown ids are ignored * during hydration so downgrade/upgrade cycles remain forward-compatible. */ contextualToursSeenIds?: ContextualTourId[] + /** Whether this profile may receive automatic contextual tours from this + * rollout. Missing means the renderer has not classified the profile yet. */ + contextualToursAutoEligible?: boolean } export const PET_SIZE_MIN = 60 diff --git a/tests/e2e/helpers/e2e-completed-onboarding-profile.ts b/tests/e2e/helpers/e2e-completed-onboarding-profile.ts index f9f64ac67fb..0a929cc823d 100644 --- a/tests/e2e/helpers/e2e-completed-onboarding-profile.ts +++ b/tests/e2e/helpers/e2e-completed-onboarding-profile.ts @@ -36,7 +36,8 @@ export function getE2ECompletedOnboardingProfile() { { firstInteractedAt: SEEN_FIRST_RUN_FEATURE_INTERACTION_TIMESTAMP } ]) ), - contextualToursSeenIds: [...SEEN_FIRST_RUN_CONTEXTUAL_TOUR_IDS] + contextualToursSeenIds: [...SEEN_FIRST_RUN_CONTEXTUAL_TOUR_IDS], + contextualToursAutoEligible: false } } }