From dbfede08bfcd82334e8ff469380cffabd3ea5e11 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:21:18 +0000 Subject: [PATCH] feat(dictation): remove filler words from text the phone recognizes itself Phones that dictate with their own recognizer (Automatic or This phone) never send text through the computer, so its filler pass never saw it. The page's on-device speech adapter now runs the same shared text pipeline on the phone's live and final text, in the language of the locale the recognizer started in, and honours the computer's Remove Filler Words switch, which the host now reports in its dictation setup. An older host that doesn't report it leaves the switch on, its default. --- .../src/dictation/dictation-reply-schema.ts | 2 + .../hooks/mobile-dictation-session-state.ts | 2 + mobile/src/hooks/use-mobile-dictation.ts | 4 +- mobile/src/platform/on-device-speech.ts | 2 +- .../platform/on-device-speech.web.test.tsx | 67 +++++++++++++++++++ mobile/src/platform/on-device-speech.web.ts | 54 +++++++++++++-- ...se-mobile-session-native-chat-dictation.ts | 6 +- .../speech/runtime-mobile-speech-catalog.ts | 1 + src/shared/runtime-worktree-contracts.ts | 2 + 9 files changed, 129 insertions(+), 11 deletions(-) create mode 100644 mobile/src/platform/on-device-speech.web.test.tsx diff --git a/mobile/src/dictation/dictation-reply-schema.ts b/mobile/src/dictation/dictation-reply-schema.ts index f50061d06e6..9298651bd11 100644 --- a/mobile/src/dictation/dictation-reply-schema.ts +++ b/mobile/src/dictation/dictation-reply-schema.ts @@ -32,6 +32,8 @@ export const dictationSetupSchema = z.looseObject({ enabled: salvagedOptional('enabled', z.boolean()), selectedModelId: salvagedOptional('selectedModelId', z.string()), dictationMode: salvagedOptional('dictationMode', z.string()), + // Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default. + removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()), models: salvagingArray(speechModelSchema) }) diff --git a/mobile/src/hooks/mobile-dictation-session-state.ts b/mobile/src/hooks/mobile-dictation-session-state.ts index a3272f3fc88..0caaeadc144 100644 --- a/mobile/src/hooks/mobile-dictation-session-state.ts +++ b/mobile/src/hooks/mobile-dictation-session-state.ts @@ -11,6 +11,8 @@ export type UseMobileDictationOptions = { enabled: boolean onTranscript: (text: string) => void onError?: (error: Error) => void + // The computer's filler switch, for text the phone recognizes itself; unknown means on. + removeFillerWords?: boolean } export type UseMobileDictationResult = { diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 348702c8fec..1d0e1746482 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -33,10 +33,10 @@ export type { UseMobileDictationResult } from './mobile-dictation-session-state' export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult { useMobileTranslation() - const { client, enabled, onTranscript, onError } = options + const { client, enabled, onTranscript, onError, removeFillerWords } = options // Both capture seams hold the screen while the microphone is open; native uses the device directly and page uses shell audio verbs. const capture = useDictationCapture() - const speech = useOnDeviceSpeech() + const speech = useOnDeviceSpeech(removeFillerWords) const [status, setStatus] = useState('idle') const [error, setError] = useState(null) const [transcriptSoFar, setTranscriptSoFar, keptTranscript] = useMobileDictationTranscriptSoFar() diff --git a/mobile/src/platform/on-device-speech.ts b/mobile/src/platform/on-device-speech.ts index 06c08ca49ce..143c56c3317 100644 --- a/mobile/src/platform/on-device-speech.ts +++ b/mobile/src/platform/on-device-speech.ts @@ -1,6 +1,6 @@ import type { OnDeviceSpeech } from './on-device-speech-contract' // The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer. -export function useOnDeviceSpeech(): OnDeviceSpeech | null { +export function useOnDeviceSpeech(_removeFillerWords?: boolean): OnDeviceSpeech | null { return null } diff --git a/mobile/src/platform/on-device-speech.web.test.tsx b/mobile/src/platform/on-device-speech.web.test.tsx new file mode 100644 index 00000000000..fc3ee5e654b --- /dev/null +++ b/mobile/src/platform/on-device-speech.web.test.tsx @@ -0,0 +1,67 @@ +// Text the phone recognizes itself gets the computer's filler removal, in the language the recognizer started in. +import { createElement } from 'react' +import { act, create } from 'react-test-renderer' +import { describe, expect, it, vi } from 'vitest' +import type { OnDeviceSpeech } from './on-device-speech-contract' + +vi.mock('../mobile-web-shell/bridge/use-native-verbs', () => { + const verbs = { + canRecognizeSpeech: true, + speechStatus: async () => ({ status: 'ready', locale: 'de-DE', preference: 'automatic' }), + speechStart: async () => ({ started: true, locale: 'de-DE', reason: null }), + speechRead: async () => ({ + revision: 1, + committed: 'Äh, das ist um fünf Uhr.', + tentative: 'ähm', + failure: null + }), + speechFinish: async () => ({ + text: 'Äh, das ist um fünf Uhr, ähm, glaube ich.', + failure: null + }), + speechCancel: async () => ({}) + } + return { useNativeVerbs: () => verbs } +}) + +vi.mock('./on-device-speech-locale', () => ({ onDeviceSpeechLocale: () => 'de-DE' })) + +import { useOnDeviceSpeech } from './on-device-speech.web' + +function mountSpeech(removeFillerWords: boolean): OnDeviceSpeech { + let speech: OnDeviceSpeech | null = null + function Probe(): null { + speech = useOnDeviceSpeech(removeFillerWords) + return null + } + act(() => { + create(createElement(Probe)) + }) + if (speech === null) { + throw new Error('no on-device speech') + } + return speech +} + +describe('on-device dictation text', () => { + it('drops the fillers of the language the recognizer started in', async () => { + const speech = mountSpeech(true) + await speech.start() + + expect(await speech.read()).toMatchObject({ committed: 'Das ist um fünf Uhr.', tentative: '' }) + expect(await speech.finish()).toEqual({ + text: 'Das ist um fünf Uhr, glaube ich.', + failure: null + }) + }) + + it('keeps every word when the computer turned filler removal off', async () => { + const speech = mountSpeech(false) + await speech.start() + + expect(await speech.finish()).toEqual({ + text: 'Äh, das ist um fünf Uhr, ähm, glaube ich.', + failure: null + }) + }) +}) diff --git a/mobile/src/platform/on-device-speech.web.ts b/mobile/src/platform/on-device-speech.web.ts index b11eed5ddda..00a8c777803 100644 --- a/mobile/src/platform/on-device-speech.web.ts +++ b/mobile/src/platform/on-device-speech.web.ts @@ -1,14 +1,43 @@ -import { useMemo } from 'react' +import { useLayoutEffect, useMemo, useRef } from 'react' +import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline' import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs' import type { OnDeviceSpeech } from './on-device-speech-contract' import { onDeviceSpeechLocale } from './on-device-speech-locale' -function createPageOnDeviceSpeech(verbs: NativeVerbs): OnDeviceSpeech { +const keepText = (text: string): string => text + +// The phone's recognizer transcribes only the locale it started in, so that locale's language picks the fillers, as the host does for its model. +function phoneDictationTextCleaner( + locale: string | null, + removeFillerWords: boolean +): (text: string) => string { + const language = locale?.split('-')[0].toLowerCase() + const clean = createDictationTextPipeline(language ? { language } : undefined, { + removeFillerWords + }) + return clean ?? keepText +} + +function createPageOnDeviceSpeech( + verbs: NativeVerbs, + removeFillerWords: () => boolean +): OnDeviceSpeech { + let clean = keepText return { status: () => verbs.speechStatus(onDeviceSpeechLocale()), - start: () => verbs.speechStart(onDeviceSpeechLocale()), - read: verbs.speechRead, - finish: verbs.speechFinish, + start: async () => { + const started = await verbs.speechStart(onDeviceSpeechLocale()) + clean = phoneDictationTextCleaner(started.locale, removeFillerWords()) + return started + }, + read: async () => { + const read = await verbs.speechRead() + return { ...read, committed: clean(read.committed), tentative: clean(read.tentative) } + }, + finish: async () => { + const finished = await verbs.speechFinish() + return finished.text === null ? finished : { ...finished, text: clean(finished.text) } + }, // Cancel has no reply anyone waits on; a shell that lost the session has nothing to stop. cancel: () => { void verbs.speechCancel().catch(() => undefined) @@ -17,7 +46,18 @@ function createPageOnDeviceSpeech(verbs: NativeVerbs): OnDeviceSpeech { } // Null on a shell without the verb, so dictation stays on the computer exactly as before. -export function useOnDeviceSpeech(): OnDeviceSpeech | null { +// removeFillerWords is the computer's switch, read at each start. +export function useOnDeviceSpeech(removeFillerWords = true): OnDeviceSpeech | null { const verbs = useNativeVerbs() - return useMemo(() => (verbs.canRecognizeSpeech ? createPageOnDeviceSpeech(verbs) : null), [verbs]) + const removeFillerWordsRef = useRef(removeFillerWords) + useLayoutEffect(() => { + removeFillerWordsRef.current = removeFillerWords + }, [removeFillerWords]) + return useMemo( + () => + verbs.canRecognizeSpeech + ? createPageOnDeviceSpeech(verbs, () => removeFillerWordsRef.current) + : null, + [verbs] + ) } diff --git a/mobile/src/session/use-mobile-session-native-chat-dictation.ts b/mobile/src/session/use-mobile-session-native-chat-dictation.ts index a82a2592ad5..e6dace40f3c 100644 --- a/mobile/src/session/use-mobile-session-native-chat-dictation.ts +++ b/mobile/src/session/use-mobile-session-native-chat-dictation.ts @@ -1,4 +1,4 @@ -import { useEffect, useCallback, useRef } from 'react' +import { useEffect, useCallback, useRef, useState } from 'react' import { AppState } from 'react-native' import { useFocusEffect } from 'expo-router' import { useMobileDictation } from '../hooks/use-mobile-dictation' @@ -136,9 +136,12 @@ export function useMobileSessionNativeChatDictation( const dictationCanStart = showNativeChat ? connState === 'connected' || phoneDictation.ready : canSend + // The host's filler switch, kept from its last setup read; on until one says otherwise. + const [removeFillerWords, setRemoveFillerWords] = useState(true) const dictation = useMobileDictation({ client, enabled: (showNativeChat || canCompose) && dictationEnabled, + removeFillerWords, onTranscript: (text) => { // Chat dictation is inserted locally; a terminal's words are typed into it live. if (destination.landsInChat(showNativeChatRef.current)) { @@ -246,6 +249,7 @@ export function useMobileSessionNativeChatDictation( const setup = await fetchDictationSetup(client) if (read === dictationSettingsReadRef.current) { setDictationMode(setup.dictationMode) + setRemoveFillerWords(setup.removeFillerWords !== false) setDictationEnabled(!dictationMicHidden(setup)) dictationNeedsSetupRef.current = dictationNeedsSetup(setup) } diff --git a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts index bc21daf1f32..71aac4a9c53 100644 --- a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts +++ b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts @@ -60,6 +60,7 @@ export class RuntimeMobileSpeechCatalog { enabled: voice.enabled === true, selectedModelId: voice.sttModel ?? '', dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle', + removeFillerWords: voice.removeFillerWords !== false, models } } diff --git a/src/shared/runtime-worktree-contracts.ts b/src/shared/runtime-worktree-contracts.ts index ca10376ceb8..23005a62ddb 100644 --- a/src/shared/runtime-worktree-contracts.ts +++ b/src/shared/runtime-worktree-contracts.ts @@ -171,6 +171,8 @@ export type RuntimeSpeechSetupState = { enabled: boolean selectedModelId: string dictationMode: 'toggle' | 'hold' + /** Phones that recognize speech themselves clean their own text by this switch. */ + removeFillerWords: boolean models: RuntimeSpeechModelSummary[] }