From 88662806c0e47d8bdde5ef44e03418aeb71b22c0 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:14:39 +0000 Subject: [PATCH] fix(mobile): explain dictation setup before asking for the microphone (#35248) A mic tap opened capture, and with it the OS permission prompt, before anything asked the host whether it could transcribe; the setup sheet only appeared after the host refused dictation.start. The session screen now gates the start on the setup read it already does, opens the setup sheet when setup is missing, and hands the next tap back to the host and re-reads when the sheet closes. A host that never had a model selected now offers the mic. The sheet's switch uses MobileSwitch. Fixes #35203 --- .../components/MobileDictationSetupSheet.tsx | 9 +++- .../dictation/mobile-dictation-setup.test.ts | 50 ++++++------------- .../src/dictation/mobile-dictation-setup.ts | 16 ++++-- mobile/src/session/MobileSessionSheets.tsx | 6 +-- .../mobile-dictation-mode-refresh.test.tsx | 12 +++++ ...mobile-dictation-session-outcomes.test.tsx | 36 ++++++++++++- ...se-mobile-session-native-chat-dictation.ts | 29 ++++++++++- 7 files changed, 111 insertions(+), 47 deletions(-) diff --git a/mobile/src/components/MobileDictationSetupSheet.tsx b/mobile/src/components/MobileDictationSetupSheet.tsx index 89085c68589..3de1b356516 100644 --- a/mobile/src/components/MobileDictationSetupSheet.tsx +++ b/mobile/src/components/MobileDictationSetupSheet.tsx @@ -1,7 +1,8 @@ import { useCallback, useEffect, useState } from 'react' -import { ActivityIndicator, Pressable, Switch, Text, View } from 'react-native' +import { ActivityIndicator, Pressable, Text, View } from 'react-native' import { Check, Download } from 'lucide-react-native' import { BottomDrawer } from './BottomDrawer' +import { MobileSwitch } from './MobileSwitch' import { colors, radii, spacing, typography } from '../theme/mobile-theme' import { themedStyleSheet } from '../theme/themed-styles' import { useMobileColorScheme } from '../theme/use-mobile-color-scheme' @@ -145,7 +146,11 @@ export function MobileDictationSetupSheet({ visible, client, onClose, onReady }: <> Dictation enabled - void handleToggleEnabled(v)} /> + void handleToggleEnabled(v)} + /> {setup.models.map((model) => { diff --git a/mobile/src/dictation/mobile-dictation-setup.test.ts b/mobile/src/dictation/mobile-dictation-setup.test.ts index 921d783d471..750130c8ed0 100644 --- a/mobile/src/dictation/mobile-dictation-setup.test.ts +++ b/mobile/src/dictation/mobile-dictation-setup.test.ts @@ -7,7 +7,7 @@ import { deleteDictationModel, downloadDictationModel, fetchDictationSetup, - isDictationReady, + dictationNeedsSetup, isDictationSetupRequiredError, dictationFailureMessage, isModelInFlight, @@ -228,38 +228,20 @@ describe('state helpers', () => { expect(isModelInFlight(model({ status: 'ready' }))).toBe(false) }) - it('isDictationReady requires enabled + selected + ready', () => { - expect( - isDictationReady({ - enabled: true, - selectedModelId: 'm1', - dictationMode: 'toggle', - models: [model({ status: 'ready' })] - }) - ).toBe(true) - expect( - isDictationReady({ - enabled: false, - selectedModelId: 'm1', - dictationMode: 'toggle', - models: [model({ status: 'ready' })] - }) - ).toBe(false) - expect( - isDictationReady({ - enabled: true, - selectedModelId: 'm1', - dictationMode: 'toggle', - models: [model({ status: 'not-downloaded' })] - }) - ).toBe(false) - expect( - isDictationReady({ - enabled: true, - selectedModelId: '', - dictationMode: 'toggle', - models: [] - }) - ).toBe(false) + it('dictationNeedsSetup mirrors the host start refusals', () => { + const setup = (enabled: boolean, selectedModelId: string, models: MobileSpeechModel[]) => ({ + enabled, + selectedModelId, + dictationMode: 'toggle' as const, + models + }) + expect(dictationNeedsSetup(setup(true, 'm1', [model({ status: 'ready' })]))).toBe(false) + expect(dictationNeedsSetup(setup(false, 'm1', [model({ status: 'ready' })]))).toBe(true) + expect(dictationNeedsSetup(setup(true, 'm1', [model({ status: 'not-downloaded' })]))).toBe(true) + expect(dictationNeedsSetup(setup(true, '', []))).toBe(true) + // A user model is absent from the host catalog; only the host can judge it. + expect(dictationNeedsSetup(setup(true, 'custom', [model({ status: 'ready' })]))).toBe(false) + // Older hosts may omit enabled and the selection. + expect(dictationNeedsSetup({ dictationMode: 'toggle', models: [] })).toBe(false) }) }) diff --git a/mobile/src/dictation/mobile-dictation-setup.ts b/mobile/src/dictation/mobile-dictation-setup.ts index 9796d3bb7d9..316b19d8581 100644 --- a/mobile/src/dictation/mobile-dictation-setup.ts +++ b/mobile/src/dictation/mobile-dictation-setup.ts @@ -118,12 +118,18 @@ export function isModelInFlight(model: MobileSpeechModel): boolean { return model.status === 'downloading' || model.status === 'extracting' } -export function isDictationReady(setup: MobileSpeechSetup): boolean { - if (!setup.enabled || !setup.selectedModelId) { - return false +// Hide the mic only when dictation was turned off with a model chosen; with none chosen a tap explains setup. +export function dictationMicHidden(setup: MobileSpeechSetup): boolean { + return setup.enabled === false && setup.selectedModelId !== '' +} + +// Mirror the host's start refusals on what it reported; absent fields or an uncatalogued (user) model stay the host's call. +export function dictationNeedsSetup(setup: MobileSpeechSetup): boolean { + if (setup.enabled === false || setup.selectedModelId === '') { + return true } - const selected = setup.models.find((m) => m.id === setup.selectedModelId) - return selected?.status === 'ready' + const status = setup.models.find((m) => m.id === setup.selectedModelId)?.status + return status !== undefined && status !== 'ready' } // Keep copy short and host-neutral: the paired computer may be a headless server. diff --git a/mobile/src/session/MobileSessionSheets.tsx b/mobile/src/session/MobileSessionSheets.tsx index 3d3de0104a0..75f8c6b3ce7 100644 --- a/mobile/src/session/MobileSessionSheets.tsx +++ b/mobile/src/session/MobileSessionSheets.tsx @@ -58,7 +58,7 @@ export function MobileSessionSheets({ controller }: { controller: MobileSessionC setDeleteKeyTarget, terminalModes, showDictationSetup, - setShowDictationSetup, + closeDictationSetup, browserScreencastSupported, quickCommandsSupported, quickCommandAgentPromptMaxLength, @@ -369,8 +369,8 @@ export function MobileSessionSheets({ controller }: { controller: MobileSessionC setShowDictationSetup(false)} - onReady={() => setShowDictationSetup(false)} + onClose={closeDictationSetup} + onReady={closeDictationSetup} /> { expect(env.captureEnabled).toBe(false) }) + it('offers the mic on a host that never had a speech model selected', async () => { + env.connected = true + env.deferReply = true + env.slowReply = Promise.resolve({ + id: 'first-run', + ok: true, + result: { enabled: false, selectedModelId: '', dictationMode: 'toggle', models: [] } + }) + await render() + expect(micBinding()).toBe('toggle') + }) + it('keeps legacy controls when the host omits the enabled field', async () => { env.connected = true env.deferReply = true diff --git a/mobile/src/session/mobile-dictation-session-outcomes.test.tsx b/mobile/src/session/mobile-dictation-session-outcomes.test.tsx index 7a6819f4c86..9fcd6163972 100644 --- a/mobile/src/session/mobile-dictation-session-outcomes.test.tsx +++ b/mobile/src/session/mobile-dictation-session-outcomes.test.tsx @@ -9,6 +9,7 @@ const seen = vi.hoisted(() => ({ toasts: new Array(), errorHaptics: 0, setupSheetOpens: 0, + micPermissionRequests: 0, clipboard: new Array(), inputs: new Array(), audioListeners: new Map void>() @@ -24,7 +25,10 @@ vi.mock('@orca/expo-two-way-audio', () => ({ return { remove: () => seen.audioListeners.delete(name) } }, initialize: () => Promise.resolve(true), - requestMicrophonePermissionsAsync: () => Promise.resolve({ granted: true }), + requestMicrophonePermissionsAsync: () => { + seen.micPermissionRequests += 1 + return Promise.resolve({ granted: true }) + }, tearDown: () => {}, toggleRecording: () => true })) @@ -192,6 +196,7 @@ beforeEach(() => { seen.toasts.length = 0 seen.errorHaptics = 0 seen.setupSheetOpens = 0 + seen.micPermissionRequests = 0 seen.clipboard.length = 0 seen.inputs.length = 0 seen.audioListeners.clear() @@ -215,6 +220,35 @@ describe('a mic tap the desktop refuses because dictation is not set up', () => } }) +describe('a mic tap after the host reported dictation is not set up', () => { + it('opens setup without asking for the microphone, then lets the host judge the tap after the sheet closes', async () => { + const rpc = createFakeRpcClient() + const mounted = mount(rpc, { liveInput: false, sendLive: async () => true }) + await act(async () => { + const read = mounted.current().refreshDictationMode() + for (const request of rpc.requests.splice(0)) { + request.resolve({ + id: 'desktop', + ok: true, + result: { enabled: false, selectedModelId: '', dictationMode: 'toggle', models: [] } + }) + } + await read + }) + act(() => mounted.current().handleDictationToggle()) + expect(seen.setupSheetOpens).toBe(1) + expect(seen.micPermissionRequests).toBe(0) + expect(rpc.requests).toEqual([]) + + // Closing the sheet hands the next tap to the host until a fresh read lands. + act(() => mounted.current().closeDictationSetup()) + act(() => mounted.current().handleDictationToggle()) + await settle(rpc, '') + expect(seen.setupSheetOpens).toBe(1) + expect(seen.micPermissionRequests).toBe(1) + }) +}) + describe('a mic tap the desktop refuses for a reason setup cannot fix', () => { it('toasts the host refusal in plain words with the error haptic', async () => { await tapAndRefuseStart('dictation_already_active') diff --git a/mobile/src/session/use-mobile-session-native-chat-dictation.ts b/mobile/src/session/use-mobile-session-native-chat-dictation.ts index c87c21d836a..48e53a7c450 100644 --- a/mobile/src/session/use-mobile-session-native-chat-dictation.ts +++ b/mobile/src/session/use-mobile-session-native-chat-dictation.ts @@ -11,6 +11,8 @@ import { } from '../terminal/terminal-live-dictation-routing' import { dictationFailureMessage, + dictationMicHidden, + dictationNeedsSetup, fetchDictationSetup, isDictationSetupRequiredError } from '../dictation/mobile-dictation-setup' @@ -179,7 +181,15 @@ export function useMobileSessionNativeChatDictation( } }) + // Only a host setup read sets this; while false, the host's start refusal decides (older hosts, first tap before the read). + const dictationNeedsSetupRef = useRef(false) + const startDictation = useCallback(() => { + if (dictationNeedsSetupRef.current) { + // Explain setup before the OS microphone prompt: mic access cannot help until the host can transcribe. + setShowDictationSetup(true) + return + } chatDictationDestination.current = captureMobileNativeChatTextDestination() const routeContext = activeHandle ? { handle: activeHandle, liveInputEnabled: liveInputTerminalHandles.has(activeHandle) } @@ -191,7 +201,13 @@ export function useMobileSessionNativeChatDictation( } reportDictationFailure(err) }) - }, [activeHandle, dictation, liveInputTerminalHandles, reportDictationFailure]) + }, [ + activeHandle, + dictation, + liveInputTerminalHandles, + reportDictationFailure, + setShowDictationSetup + ]) const cancelDictation = useCallback(() => { dictationRouteContextRef.current = null @@ -233,13 +249,21 @@ export function useMobileSessionNativeChatDictation( const setup = await fetchDictationSetup(client) if (read === dictationSettingsReadRef.current) { setDictationMode(setup.dictationMode) - setDictationEnabled(setup.enabled !== false) + setDictationEnabled(!dictationMicHidden(setup)) + dictationNeedsSetupRef.current = dictationNeedsSetup(setup) } } catch { // Keep the last settings when an older host cannot report them. } }, [client, connState, setDictationEnabled, setDictationMode]) + // The sheet may have changed host setup: let the host judge taps until the fresh read lands. + const closeDictationSetup = useCallback(() => { + setShowDictationSetup(false) + dictationNeedsSetupRef.current = false + void refreshDictationMode() + }, [refreshDictationMode, setShowDictationSetup]) + useFocusEffect( useCallback(() => { void refreshDictationMode() @@ -280,6 +304,7 @@ export function useMobileSessionNativeChatDictation( handleDictationToggle, handleDictationPressIn, handleDictationPressOut, + closeDictationSetup, refreshDictationMode } }