diff --git a/mobile/src/dictation/dictation-reply-schema.ts b/mobile/src/dictation/dictation-reply-schema.ts index f50061d06e6..9298651bd11 100644 --- a/mobile/src/dictation/dictation-reply-schema.ts +++ b/mobile/src/dictation/dictation-reply-schema.ts @@ -32,6 +32,8 @@ export const dictationSetupSchema = z.looseObject({ enabled: salvagedOptional('enabled', z.boolean()), selectedModelId: salvagedOptional('selectedModelId', z.string()), dictationMode: salvagedOptional('dictationMode', z.string()), + // Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default. + removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()), models: salvagingArray(speechModelSchema) }) diff --git a/mobile/src/hooks/mobile-dictation-session-state.ts b/mobile/src/hooks/mobile-dictation-session-state.ts index a3272f3fc88..0caaeadc144 100644 --- a/mobile/src/hooks/mobile-dictation-session-state.ts +++ b/mobile/src/hooks/mobile-dictation-session-state.ts @@ -11,6 +11,8 @@ export type UseMobileDictationOptions = { enabled: boolean onTranscript: (text: string) => void onError?: (error: Error) => void + // The computer's filler switch, for text the phone recognizes itself; unknown means on. + removeFillerWords?: boolean } export type UseMobileDictationResult = { diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 348702c8fec..1d0e1746482 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -33,10 +33,10 @@ export type { UseMobileDictationResult } from './mobile-dictation-session-state' export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult { useMobileTranslation() - const { client, enabled, onTranscript, onError } = options + const { client, enabled, onTranscript, onError, removeFillerWords } = options // Both capture seams hold the screen while the microphone is open; native uses the device directly and page uses shell audio verbs. const capture = useDictationCapture() - const speech = useOnDeviceSpeech() + const speech = useOnDeviceSpeech(removeFillerWords) const [status, setStatus] = useState('idle') const [error, setError] = useState(null) const [transcriptSoFar, setTranscriptSoFar, keptTranscript] = useMobileDictationTranscriptSoFar() diff --git a/mobile/src/platform/on-device-speech.ts b/mobile/src/platform/on-device-speech.ts index 06c08ca49ce..143c56c3317 100644 --- a/mobile/src/platform/on-device-speech.ts +++ b/mobile/src/platform/on-device-speech.ts @@ -1,6 +1,6 @@ import type { OnDeviceSpeech } from './on-device-speech-contract' // The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer. -export function useOnDeviceSpeech(): OnDeviceSpeech | null { +export function useOnDeviceSpeech(_removeFillerWords?: boolean): OnDeviceSpeech | null { return null } diff --git a/mobile/src/platform/on-device-speech.web.test.tsx b/mobile/src/platform/on-device-speech.web.test.tsx new file mode 100644 index 00000000000..fc3ee5e654b --- /dev/null +++ b/mobile/src/platform/on-device-speech.web.test.tsx @@ -0,0 +1,67 @@ +// Text the phone recognizes itself gets the computer's filler removal, in the language the recognizer started in. +import { createElement } from 'react' +import { act, create } from 'react-test-renderer' +import { describe, expect, it, vi } from 'vitest' +import type { OnDeviceSpeech } from './on-device-speech-contract' + +vi.mock('../mobile-web-shell/bridge/use-native-verbs', () => { + const verbs = { + canRecognizeSpeech: true, + speechStatus: async () => ({ status: 'ready', locale: 'de-DE', preference: 'automatic' }), + speechStart: async () => ({ started: true, locale: 'de-DE', reason: null }), + speechRead: async () => ({ + revision: 1, + committed: 'Äh, das ist um fünf Uhr.', + tentative: 'ähm', + failure: null + }), + speechFinish: async () => ({ + text: 'Äh, das ist um fünf Uhr, ähm, glaube ich.', + failure: null + }), + speechCancel: async () => ({}) + } + return { useNativeVerbs: () => verbs } +}) + +vi.mock('./on-device-speech-locale', () => ({ onDeviceSpeechLocale: () => 'de-DE' })) + +import { useOnDeviceSpeech } from './on-device-speech.web' + +function mountSpeech(removeFillerWords: boolean): OnDeviceSpeech { + let speech: OnDeviceSpeech | null = null + function Probe(): null { + speech = useOnDeviceSpeech(removeFillerWords) + return null + } + act(() => { + create(createElement(Probe)) + }) + if (speech === null) { + throw new Error('no on-device speech') + } + return speech +} + +describe('on-device dictation text', () => { + it('drops the fillers of the language the recognizer started in', async () => { + const speech = mountSpeech(true) + await speech.start() + + expect(await speech.read()).toMatchObject({ committed: 'Das ist um fünf Uhr.', tentative: '' }) + expect(await speech.finish()).toEqual({ + text: 'Das ist um fünf Uhr, glaube ich.', + failure: null + }) + }) + + it('keeps every word when the computer turned filler removal off', async () => { + const speech = mountSpeech(false) + await speech.start() + + expect(await speech.finish()).toEqual({ + text: 'Äh, das ist um fünf Uhr, ähm, glaube ich.', + failure: null + }) + }) +}) diff --git a/mobile/src/platform/on-device-speech.web.ts b/mobile/src/platform/on-device-speech.web.ts index b11eed5ddda..00a8c777803 100644 --- a/mobile/src/platform/on-device-speech.web.ts +++ b/mobile/src/platform/on-device-speech.web.ts @@ -1,14 +1,43 @@ -import { useMemo } from 'react' +import { useLayoutEffect, useMemo, useRef } from 'react' +import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline' import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs' import type { OnDeviceSpeech } from './on-device-speech-contract' import { onDeviceSpeechLocale } from './on-device-speech-locale' -function createPageOnDeviceSpeech(verbs: NativeVerbs): OnDeviceSpeech { +const keepText = (text: string): string => text + +// The phone's recognizer transcribes only the locale it started in, so that locale's language picks the fillers, as the host does for its model. +function phoneDictationTextCleaner( + locale: string | null, + removeFillerWords: boolean +): (text: string) => string { + const language = locale?.split('-')[0].toLowerCase() + const clean = createDictationTextPipeline(language ? { language } : undefined, { + removeFillerWords + }) + return clean ?? keepText +} + +function createPageOnDeviceSpeech( + verbs: NativeVerbs, + removeFillerWords: () => boolean +): OnDeviceSpeech { + let clean = keepText return { status: () => verbs.speechStatus(onDeviceSpeechLocale()), - start: () => verbs.speechStart(onDeviceSpeechLocale()), - read: verbs.speechRead, - finish: verbs.speechFinish, + start: async () => { + const started = await verbs.speechStart(onDeviceSpeechLocale()) + clean = phoneDictationTextCleaner(started.locale, removeFillerWords()) + return started + }, + read: async () => { + const read = await verbs.speechRead() + return { ...read, committed: clean(read.committed), tentative: clean(read.tentative) } + }, + finish: async () => { + const finished = await verbs.speechFinish() + return finished.text === null ? finished : { ...finished, text: clean(finished.text) } + }, // Cancel has no reply anyone waits on; a shell that lost the session has nothing to stop. cancel: () => { void verbs.speechCancel().catch(() => undefined) @@ -17,7 +46,18 @@ function createPageOnDeviceSpeech(verbs: NativeVerbs): OnDeviceSpeech { } // Null on a shell without the verb, so dictation stays on the computer exactly as before. -export function useOnDeviceSpeech(): OnDeviceSpeech | null { +// removeFillerWords is the computer's switch, read at each start. +export function useOnDeviceSpeech(removeFillerWords = true): OnDeviceSpeech | null { const verbs = useNativeVerbs() - return useMemo(() => (verbs.canRecognizeSpeech ? createPageOnDeviceSpeech(verbs) : null), [verbs]) + const removeFillerWordsRef = useRef(removeFillerWords) + useLayoutEffect(() => { + removeFillerWordsRef.current = removeFillerWords + }, [removeFillerWords]) + return useMemo( + () => + verbs.canRecognizeSpeech + ? createPageOnDeviceSpeech(verbs, () => removeFillerWordsRef.current) + : null, + [verbs] + ) } diff --git a/mobile/src/session/use-mobile-session-native-chat-dictation.ts b/mobile/src/session/use-mobile-session-native-chat-dictation.ts index a82a2592ad5..e6dace40f3c 100644 --- a/mobile/src/session/use-mobile-session-native-chat-dictation.ts +++ b/mobile/src/session/use-mobile-session-native-chat-dictation.ts @@ -1,4 +1,4 @@ -import { useEffect, useCallback, useRef } from 'react' +import { useEffect, useCallback, useRef, useState } from 'react' import { AppState } from 'react-native' import { useFocusEffect } from 'expo-router' import { useMobileDictation } from '../hooks/use-mobile-dictation' @@ -136,9 +136,12 @@ export function useMobileSessionNativeChatDictation( const dictationCanStart = showNativeChat ? connState === 'connected' || phoneDictation.ready : canSend + // The host's filler switch, kept from its last setup read; on until one says otherwise. + const [removeFillerWords, setRemoveFillerWords] = useState(true) const dictation = useMobileDictation({ client, enabled: (showNativeChat || canCompose) && dictationEnabled, + removeFillerWords, onTranscript: (text) => { // Chat dictation is inserted locally; a terminal's words are typed into it live. if (destination.landsInChat(showNativeChatRef.current)) { @@ -246,6 +249,7 @@ export function useMobileSessionNativeChatDictation( const setup = await fetchDictationSetup(client) if (read === dictationSettingsReadRef.current) { setDictationMode(setup.dictationMode) + setRemoveFillerWords(setup.removeFillerWords !== false) setDictationEnabled(!dictationMicHidden(setup)) dictationNeedsSetupRef.current = dictationNeedsSetup(setup) } diff --git a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts index bc21daf1f32..71aac4a9c53 100644 --- a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts +++ b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts @@ -60,6 +60,7 @@ export class RuntimeMobileSpeechCatalog { enabled: voice.enabled === true, selectedModelId: voice.sttModel ?? '', dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle', + removeFillerWords: voice.removeFillerWords !== false, models } } diff --git a/src/shared/runtime-worktree-contracts.ts b/src/shared/runtime-worktree-contracts.ts index ca10376ceb8..23005a62ddb 100644 --- a/src/shared/runtime-worktree-contracts.ts +++ b/src/shared/runtime-worktree-contracts.ts @@ -171,6 +171,8 @@ export type RuntimeSpeechSetupState = { enabled: boolean selectedModelId: string dictationMode: 'toggle' | 'hold' + /** Phones that recognize speech themselves clean their own text by this switch. */ + removeFillerWords: boolean models: RuntimeSpeechModelSummary[] }