From 09ecfbb90f01f63aa288960cc85be7ed7170800e Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:04:50 +0000 Subject: [PATCH] =?UTF-8?q?feat(dictation):=20remove=20filler=20words=20li?= =?UTF-8?q?ke=20=E2=80=9Cum=E2=80=9D=20and=20=E2=80=9Cuh=E2=80=9D=20from?= =?UTF-8?q?=20dictated=20text=20(#40137)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dictation): remove filler words from dictated text, per language Spoken fillers (um, uh, äh, えーと, 음, 嗯 and their equivalents) are dropped from dictated text before it is inserted, on desktop, the web client and phones that dictate through the computer. The pass runs once in the speech host's event sink, so every route shares it, and lives in a new shared text pipeline that later passes join. Only standalone sounds go, never words with meaning, and the comma after a filler goes with it. The language comes from a single-language model or the dictation language setting; an unknown language uses only fillers that are no word in another language. Voice settings gain a Remove Filler Words switch, on by default. * fix(dictation): take the filler language from what the engine is told, and drop the comma before a final filler * fix(dictation): keep 嗯 as a reply, clean all-caps transcripts, and read the sentence start in constant time * refactor(dictation): one list of sentence-end marks in filler removal, and drop a duplicate off-switch test * fix(dictation): write non-ASCII filler punctuation as escapes so the orcad bundle stays ASCII --- src/main/ipc/speech.ts | 2 +- .../runtime-mobile-dictation-controller.ts | 2 +- src/main/speech/stt-service.test.ts | 43 +++++++- src/main/speech/stt-service.ts | 27 ++++- .../components/settings/voice/VoicePane.tsx | 15 +++ .../settings/voice/voice-pane-search.ts | 21 ++++ ...wser-link-materialization.test-fixture.tsx | 1 + src/renderer/src/i18n/locales/de.json | 9 +- src/renderer/src/i18n/locales/en.json | 9 +- src/renderer/src/i18n/locales/es.json | 9 +- src/renderer/src/i18n/locales/fr.json | 9 +- src/renderer/src/i18n/locales/ja.json | 9 +- src/renderer/src/i18n/locales/ko.json | 9 +- src/renderer/src/i18n/locales/pt.json | 9 +- src/renderer/src/i18n/locales/ru.json | 9 +- src/renderer/src/i18n/locales/tr.json | 9 +- src/renderer/src/i18n/locales/zh-Hant.json | 9 +- src/renderer/src/i18n/locales/zh.json | 9 +- src/shared/constants.ts | 1 + src/shared/dictation-filler-words.test.ts | 73 ++++++++++++ src/shared/dictation-filler-words.ts | 104 ++++++++++++++++++ src/shared/dictation-text-pipeline.ts | 36 ++++++ src/shared/speech-types.ts | 2 + 23 files changed, 409 insertions(+), 17 deletions(-) create mode 100644 src/shared/dictation-filler-words.test.ts create mode 100644 src/shared/dictation-filler-words.ts create mode 100644 src/shared/dictation-text-pipeline.ts diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 0268e64691b..feded50d21b 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -221,7 +221,7 @@ export function registerSpeechHandlers(store: Store): void { }, resolvedHotwordsPath, owner, - store.getSettings().voice?.dictationLanguage + store.getSettings().voice ) // Coalesced starts install no sink; only an accepted start supersedes the close hook. if (sessionReady) { diff --git a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts index a088268891b..8ac411c2da1 100644 --- a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts @@ -87,7 +87,7 @@ export class RuntimeMobileDictationController { (event) => acceptEvent(session, event), undefined, owner, - voice.dictationLanguage + voice ) if (this.sessions.get(clientId) !== session) { throw new Error('dictation_canceled') diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index 0b9076a1eda..b0bcdcb323a 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -159,6 +159,7 @@ vi.mock('./model-catalog', () => ({ id, type: 'openai', provider: 'openai', + language: 'multilingual', streaming: false, sampleRate: 16000 } @@ -167,6 +168,7 @@ vi.mock('./model-catalog', () => ({ id, type: 'whisper', provider: 'local', + language: 'multilingual', streaming: false, sampleRate: 16000, files: ['encoder.onnx', 'decoder.onnx', 'tokens.txt'], @@ -176,6 +178,7 @@ vi.mock('./model-catalog', () => ({ id: 'model-a', type: 'transducer', provider: 'local', + language: 'en', streaming: true, sampleRate: 16000, files: ['encoder.onnx', 'decoder.onnx', 'joiner.onnx', 'tokens.txt'] @@ -233,6 +236,38 @@ describe('SttService', () => { expect(worker.messages.at(-1)).toMatchObject({ type: 'feed', streamId: desktopStream }) }) + it('removes filler words from the text every host route reads, unless they are turned off', async () => { + const service = new SttService(readyModels(), spawnMockWorker) + const cleaned = vi.fn() + const raw = vi.fn() + + await service.startDictation('model-a', cleaned, undefined, 'desktop', { + removeFillerWords: true + }) + await service.startDictation('model-a', raw, undefined, 'mobile:phone-a:1', { + removeFillerWords: false + }) + service.feedAudio(new Float32Array([1]), 16000, 'desktop') + service.feedAudio(new Float32Array([2]), 16000, 'mobile:phone-a:1') + const worker = getLastWorker()! + const [cleanedStream, rawStream] = worker.messages + .filter((message) => message.type === 'feed') + .map((message) => message.streamId) + for (const streamId of [cleanedStream, rawStream]) { + worker.emit('message', { type: 'partial', streamId, text: 'Um, ship' }) + worker.emit('message', { type: 'final', streamId, text: 'Um, ship it, uh, today.' }) + } + + expect(cleaned.mock.calls.slice(1)).toEqual([ + [{ type: 'partial', text: 'Ship' }], + [{ type: 'final', text: 'Ship it, today.' }] + ]) + expect(raw.mock.calls.slice(1)).toEqual([ + [{ type: 'partial', text: 'Um, ship' }], + [{ type: 'final', text: 'Um, ship it, uh, today.' }] + ]) + }) + it("hands a renderer's audio port only to its own on-device stream", async () => { const service = new SttService(readyModels(), spawnMockWorker) await service.startDictation('model-a', vi.fn(), undefined, 'mobile:phone-a:1') @@ -294,9 +329,13 @@ describe('SttService', () => { spawnMockWorker ) - await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', 'ko') + await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', { + dictationLanguage: 'ko' + }) await service.stopDictation('desktop') - await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', 'ja') + await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', { + dictationLanguage: 'ja' + }) expect(getCreatedWorkerCount()).toBe(2) expect(getLastWorker()!.messages.find((message) => message.type === 'init')).toMatchObject({ diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 9db198abdb5..c7228ff6bfd 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -1,4 +1,9 @@ import type { MessagePortMain } from 'electron' +import { + createDictationTextPipeline, + type DictationTextSettings +} from '../../shared/dictation-text-pipeline' +import { getCatalogModel } from './model-catalog' import type { ModelManager } from './model-manager' import type { SpawnSttWorkerProcess } from './stt-worker-process' import { prewarmSttModel } from './stt-session-prewarm' @@ -22,6 +27,16 @@ export type SttEvent = export type SttEventSink = (event: SttEvent) => void +// Cleaned once here, so every route the host transcribes for gets the same text. +function cleanTranscriptEvents(sink: SttEventSink, clean: (text: string) => string): SttEventSink { + return (event) => + sink( + event.type === 'partial' || event.type === 'final' + ? { ...event, text: event.text && clean(event.text) } + : event + ) +} + export class SttService { private readonly state: SttSessionState @@ -34,9 +49,17 @@ export class SttService { sink: SttEventSink, hotwordsFilePath?: string, owner = 'desktop', - language?: string + voice?: Partial ): Promise { - return startSttDictation(this.state, modelId, sink, hotwordsFilePath, owner, language) + const clean = createDictationTextPipeline(getCatalogModel(modelId), voice) + return startSttDictation( + this.state, + modelId, + clean ? cleanTranscriptEvents(sink, clean) : sink, + hotwordsFilePath, + owner, + voice?.dictationLanguage + ) } feedAudio(samples: Float32Array, sampleRate: number, owner = 'desktop'): boolean { diff --git a/src/renderer/src/components/settings/voice/VoicePane.tsx b/src/renderer/src/components/settings/voice/VoicePane.tsx index 5c0aa7ee1cd..b85a62c59cc 100644 --- a/src/renderer/src/components/settings/voice/VoicePane.tsx +++ b/src/renderer/src/components/settings/voice/VoicePane.tsx @@ -14,6 +14,7 @@ import { VoiceDictationLanguageSetting } from './VoiceDictationLanguageSetting' import { VoiceDictationSettingsSection } from './VoiceDictationSettingsSection' import { VoiceModelUnloadSetting } from './VoiceModelUnloadSetting' import { VoiceSpeechModelSection } from './VoiceSpeechModelSection' +import { SettingsSwitchRow } from '../form-controls/SettingsFormControls' import { matchesSettingsSearch } from '../search/settings-search' import { getOpenaiTranscriptionSearchEntry } from './voice-pane-search' import { useTranslate } from '@/i18n/use-translate' @@ -230,6 +231,20 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J + + updateVoiceSettings({ removeFillerWords: !voiceSettings.removeFillerWords }) + } + /> + + + [ ...translateSearchKeyword('auto.components.settings.voice.pane.search.10d45a9fce', 'stt') ] }, + { + title: translate( + 'auto.components.settings.voice.pane.search.fillerWordsTitle', + 'Remove Filler Words' + ), + description: translate( + 'auto.components.settings.voice.pane.search.fillerWordsDescription', + 'Drop “um”, “uh” and similar sounds from dictated text.' + ), + keywords: [ + ...translateSearchKeyword('auto.components.settings.voice.pane.search.7640ed9848', 'voice'), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.089d31a45b', + 'dictation' + ), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.fillerWordsKeyword', + 'filler words' + ) + ] + }, { title: translate( 'auto.components.settings.voice.pane.search.modelUnloadTitle', diff --git a/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx b/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx index c18bbf1a35d..e861cd4ba20 100644 --- a/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx +++ b/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx @@ -86,6 +86,7 @@ export function installBrowserLinkMaterializationProbe() { modelsDir: '', dictationLanguage: 'auto', dictationMode: 'hold', + removeFillerWords: true, terminalConfirmBeforeInsert: true, userModels: [], openAiApiKeyConfigured: false, diff --git a/src/renderer/src/i18n/locales/de.json b/src/renderer/src/i18n/locales/de.json index 18b9e04c6cb..d78eb761b2c 100644 --- a/src/renderer/src/i18n/locales/de.json +++ b/src/renderer/src/i18n/locales/de.json @@ -11300,7 +11300,10 @@ "modelUnloadTitle": "Sprachmodell entladen", "modelUnloadDescription": "Lege fest, wann das lokale Sprachmodell aus dem Speicher entladen wird.", "modelUnloadKeyword": "entladen", - "modelMemoryKeyword": "speicher" + "modelMemoryKeyword": "speicher", + "fillerWordsTitle": "Füllwörter entfernen", + "fillerWordsDescription": "Entfernt „äh“, „ähm“ und ähnliche Laute aus diktiertem Text.", + "fillerWordsKeyword": "füllwörter" } } }, @@ -20664,6 +20667,10 @@ "cloud": "Cloud-Modelle laufen auf den Servern des Anbieters, daher bleibt hier nichts geladen.", "never": "Nie", "immediately": "Sofort" + }, + "fillerWords": { + "title": "Füllwörter entfernen", + "description": "Entfernt Laute wie „äh“ und „ähm“ aus diktiertem Text, passend zur Diktiersprache." } } }, diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index bf0008165e3..b3999e2cb65 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -11616,7 +11616,10 @@ "modelUnloadTitle": "Unload Speech Model", "modelUnloadDescription": "Choose when the on-device speech model is unloaded from memory.", "modelUnloadKeyword": "unload", - "modelMemoryKeyword": "memory" + "modelMemoryKeyword": "memory", + "fillerWordsTitle": "Remove Filler Words", + "fillerWordsDescription": "Drop “um”, “uh” and similar sounds from dictated text.", + "fillerWordsKeyword": "filler words" } } }, @@ -21320,6 +21323,10 @@ "cloud": "Cloud models run on the provider’s servers, so nothing stays loaded here.", "never": "Never", "immediately": "Immediately" + }, + "fillerWords": { + "title": "Remove Filler Words", + "description": "Drops sounds like “um” and “uh” from dictated text, in the dictation language." } } }, diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 99bb03043b0..afec0da848c 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -10902,7 +10902,10 @@ "modelUnloadTitle": "Descargar modelo de voz", "modelUnloadDescription": "Elige cuándo se descarga de la memoria el modelo de voz local.", "modelUnloadKeyword": "descargar", - "modelMemoryKeyword": "memoria" + "modelMemoryKeyword": "memoria", + "fillerWordsTitle": "Quitar muletillas", + "fillerWordsDescription": "Quita «ehm», «em» y sonidos parecidos del texto dictado.", + "fillerWordsKeyword": "muletillas" } } }, @@ -20471,6 +20474,10 @@ "cloud": "Los modelos en la nube se ejecutan en los servidores del proveedor, así que aquí no queda nada cargado.", "never": "Nunca", "immediately": "Inmediatamente" + }, + "fillerWords": { + "title": "Quitar muletillas", + "description": "Quita sonidos como «ehm» y «em» del texto dictado, según el idioma de dictado." } } }, diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 69490a12a5e..20adb6a78f7 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -11169,7 +11169,10 @@ "modelUnloadTitle": "Décharger le modèle vocal", "modelUnloadDescription": "Choisissez quand le modèle vocal local est déchargé de la mémoire.", "modelUnloadKeyword": "décharger", - "modelMemoryKeyword": "mémoire" + "modelMemoryKeyword": "mémoire", + "fillerWordsTitle": "Supprimer les hésitations", + "fillerWordsDescription": "Supprime « euh », « hum » et les sons similaires du texte dicté.", + "fillerWordsKeyword": "hésitations" } } }, @@ -20770,6 +20773,10 @@ "cloud": "Les modèles cloud tournent sur les serveurs du fournisseur, donc rien ne reste chargé ici.", "never": "Jamais", "immediately": "Immédiatement" + }, + "fillerWords": { + "title": "Supprimer les hésitations", + "description": "Supprime les sons comme « euh » et « hum » du texte dicté, selon la langue de dictée." } } }, diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 45d8c0100b1..32c2cdafe37 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -10645,7 +10645,10 @@ "modelUnloadTitle": "音声モデルの解放", "modelUnloadDescription": "ローカル音声モデルをメモリから解放するタイミングを選びます。", "modelUnloadKeyword": "解放", - "modelMemoryKeyword": "メモリ" + "modelMemoryKeyword": "メモリ", + "fillerWordsTitle": "フィラーを削除", + "fillerWordsDescription": "「えーと」「うーん」などのつなぎ言葉を音声入力のテキストから取り除きます。", + "fillerWordsKeyword": "フィラー" } } }, @@ -19963,6 +19966,10 @@ "cloud": "クラウドモデルはプロバイダーのサーバーで動くため、ここには何も読み込まれません。", "never": "しない", "immediately": "すぐに" + }, + "fillerWords": { + "title": "フィラーを削除", + "description": "音声入力の言語に合わせて、「えーと」「うーん」などのつなぎ言葉を入力テキストから取り除きます。" } } }, diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index de2fea3b0a6..76a5d1faa93 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -10608,7 +10608,10 @@ "modelUnloadTitle": "음성 모델 언로드", "modelUnloadDescription": "로컬 음성 모델을 메모리에서 언로드할 시점을 선택합니다.", "modelUnloadKeyword": "언로드", - "modelMemoryKeyword": "메모리" + "modelMemoryKeyword": "메모리", + "fillerWordsTitle": "군말 제거", + "fillerWordsDescription": "받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다.", + "fillerWordsKeyword": "군말" } } }, @@ -19963,6 +19966,10 @@ "cloud": "클라우드 모델은 제공업체 서버에서 실행되므로 여기에는 아무것도 로드되지 않습니다.", "never": "안 함", "immediately": "즉시" + }, + "fillerWords": { + "title": "군말 제거", + "description": "받아쓰기 언어에 맞춰 받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다." } } }, diff --git a/src/renderer/src/i18n/locales/pt.json b/src/renderer/src/i18n/locales/pt.json index ffd6d965851..f8149a363dc 100644 --- a/src/renderer/src/i18n/locales/pt.json +++ b/src/renderer/src/i18n/locales/pt.json @@ -6611,7 +6611,10 @@ "modelUnloadTitle": "Descarregar modelo de voz", "modelUnloadDescription": "Escolha quando o modelo de voz local é descarregado da memória.", "modelUnloadKeyword": "descarregar", - "modelMemoryKeyword": "memória" + "modelMemoryKeyword": "memória", + "fillerWordsTitle": "Remover hesitações", + "fillerWordsDescription": "Remove “hã”, “hum” e sons parecidos do texto ditado.", + "fillerWordsKeyword": "hesitações" } } }, @@ -21653,6 +21656,10 @@ "cloud": "Modelos na nuvem rodam nos servidores do provedor, então nada fica carregado aqui.", "never": "Nunca", "immediately": "Imediatamente" + }, + "fillerWords": { + "title": "Remover hesitações", + "description": "Remove sons como “hã” e “hum” do texto ditado, conforme o idioma do ditado." } } }, diff --git a/src/renderer/src/i18n/locales/ru.json b/src/renderer/src/i18n/locales/ru.json index 8e0c747873c..555016e3141 100644 --- a/src/renderer/src/i18n/locales/ru.json +++ b/src/renderer/src/i18n/locales/ru.json @@ -11654,7 +11654,10 @@ "modelUnloadTitle": "Выгрузка речевой модели", "modelUnloadDescription": "Выберите, когда локальная речевая модель выгружается из памяти.", "modelUnloadKeyword": "выгрузить", - "modelMemoryKeyword": "память" + "modelMemoryKeyword": "память", + "fillerWordsTitle": "Удалять слова-паразиты", + "fillerWordsDescription": "Убирает «э», «эм» и похожие звуки из надиктованного текста.", + "fillerWordsKeyword": "паразиты" } } }, @@ -21353,6 +21356,10 @@ "cloud": "Облачные модели работают на серверах провайдера, поэтому здесь ничего не загружается.", "never": "Никогда", "immediately": "Сразу" + }, + "fillerWords": { + "title": "Удалять слова-паразиты", + "description": "Убирает звуки вроде «э» и «эм» из надиктованного текста с учётом языка диктовки." } } }, diff --git a/src/renderer/src/i18n/locales/tr.json b/src/renderer/src/i18n/locales/tr.json index c2b5176ca38..8cca758d576 100644 --- a/src/renderer/src/i18n/locales/tr.json +++ b/src/renderer/src/i18n/locales/tr.json @@ -11284,7 +11284,10 @@ "modelUnloadTitle": "Konuşma modelini bellekten kaldır", "modelUnloadDescription": "Yerel konuşma modelinin bellekten ne zaman kaldırılacağını seçin.", "modelUnloadKeyword": "kaldır", - "modelMemoryKeyword": "bellek" + "modelMemoryKeyword": "bellek", + "fillerWordsTitle": "Dolgu seslerini kaldır", + "fillerWordsDescription": "Dikte edilen metinden “hmm” gibi duraksama seslerini çıkarır.", + "fillerWordsKeyword": "dolgu" } } }, @@ -20646,6 +20649,10 @@ "cloud": "Bulut modelleri sağlayıcının sunucularında çalışır, bu yüzden burada hiçbir şey yüklü kalmaz.", "never": "Asla", "immediately": "Hemen" + }, + "fillerWords": { + "title": "Dolgu seslerini kaldır", + "description": "Dikte edilen metinden “hmm” gibi duraksama seslerini dikte diline göre çıkarır." } } }, diff --git a/src/renderer/src/i18n/locales/zh-Hant.json b/src/renderer/src/i18n/locales/zh-Hant.json index b1ea1ea3968..510f4384caf 100644 --- a/src/renderer/src/i18n/locales/zh-Hant.json +++ b/src/renderer/src/i18n/locales/zh-Hant.json @@ -11121,7 +11121,10 @@ "modelUnloadTitle": "卸載語音模型", "modelUnloadDescription": "選擇何時從記憶體中卸載本機語音模型。", "modelUnloadKeyword": "卸載", - "modelMemoryKeyword": "記憶體" + "modelMemoryKeyword": "記憶體", + "fillerWordsTitle": "移除語氣詞", + "fillerWordsDescription": "從聽寫文字中移除「嗯」「呃」之類的語氣詞。", + "fillerWordsKeyword": "語氣詞" } } }, @@ -20348,6 +20351,10 @@ "cloud": "雲端模型在服務商的伺服器上執行,因此這裡不會載入任何內容。", "never": "從不", "immediately": "立即" + }, + "fillerWords": { + "title": "移除語氣詞", + "description": "依聽寫語言,從聽寫文字中移除「嗯」「呃」之類的語氣詞。" } } }, diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 16b87331748..1fb865f11e6 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -10608,7 +10608,10 @@ "modelUnloadTitle": "卸载语音模型", "modelUnloadDescription": "选择何时从内存中卸载本地语音模型。", "modelUnloadKeyword": "卸载", - "modelMemoryKeyword": "内存" + "modelMemoryKeyword": "内存", + "fillerWordsTitle": "移除语气词", + "fillerWordsDescription": "从听写文本中去掉“嗯”“呃”之类的语气词。", + "fillerWordsKeyword": "语气词" } } }, @@ -19963,6 +19966,10 @@ "cloud": "云端模型在服务商的服务器上运行,因此这里不会加载任何内容。", "never": "从不", "immediately": "立即" + }, + "fillerWords": { + "title": "移除语气词", + "description": "按听写语言,从听写文本中去掉“嗯”“呃”之类的语气词。" } } }, diff --git a/src/shared/constants.ts b/src/shared/constants.ts index ef4ddf207d9..4d2384c57f7 100644 --- a/src/shared/constants.ts +++ b/src/shared/constants.ts @@ -120,6 +120,7 @@ export function getDefaultVoiceSettings(): VoiceSettings { modelsDir: '', dictationLanguage: DICTATION_LANGUAGE_AUTO, dictationMode: 'toggle' as const, + removeFillerWords: true, terminalConfirmBeforeInsert: false, userModels: [], openAiApiKeyConfigured: false, diff --git a/src/shared/dictation-filler-words.test.ts b/src/shared/dictation-filler-words.test.ts new file mode 100644 index 00000000000..491da992928 --- /dev/null +++ b/src/shared/dictation-filler-words.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest' +import { removeFillerWords } from './dictation-filler-words' +import { createDictationTextPipeline } from './dictation-text-pipeline' + +describe('removeFillerWords', () => { + it.each([ + ['en', 'Um, I think we should, uh, ship it.', 'I think we should, ship it.'], + ['en', 'Let me check umm the logs.', 'Let me check the logs.'], + ['en', 'Yes. Uh. No.', 'Yes. No.'], + ['en', 'We should ship it, hmm', 'We should ship it'], + ['en', 'we should um.', 'we should.'], + ['en', 'We should ship, um.', 'We should ship.'], + ['en', 'um so we ship', 'so we ship'], + ['en', 'Um, iPhone builds first.', 'iPhone builds first.'], + [ + 'de', + 'Äh, das ist um fünf Uhr fertig, ähm, glaube ich.', + 'Das ist um fünf Uhr fertig, glaube ich.' + ], + ['es', 'Ehm, creo que em funciona.', 'Creo que funciona.'], + ['ru', 'Э, ну, давай так.', 'Ну, давай так.'], + ['ja', 'えーと、明日の会議は、うーん、十時です。', '明日の会議は、十時です。'], + ['ko', '음, 내일 회의는 흠, 열 시예요.', '내일 회의는 열 시예요.'], + ['zh', '嗯,我觉得,呃,这个可以。', '我觉得,这个可以。'], + ['zh', '額,我覺得可以。', '我覺得可以。'], + ['en', 'UM I THINK WE SHOULD UH SHIP IT', 'I THINK WE SHOULD SHIP IT'] + ])('%s: %s', (language, text, expected) => { + expect(removeFillerWords(text, [language])).toBe(expected) + }) + + it.each([ + ['en', 'I like it, so, well, we ship.'], + ['en', 'The umbrella is humming, uh-huh.'], + ['en', 'Hmm? What did you say?'], + ['en', 'Ask UM and HMM about it.'], + ['en', 'Open um.txt and set var_um to 3.'], + ['ja', 'ええ、そうです。'], + ['ko', '음이 너무 높아요.'], + ['zh', '金额不对,那个文件也要改。'], + ['zh', '嗯。'] + ])('%s keeps words with meaning: %s', (language, text) => { + expect(removeFillerWords(text, [language])).toBe(text) + }) + + it('uses only fillers that are no word elsewhere when the language is unknown', () => { + expect(removeFillerWords('Um carro, uh, azul.', undefined)).toBe('Um carro, azul.') + expect(removeFillerWords('Er kommt, hmm, morgen.', undefined)).toBe('Er kommt, morgen.') + expect(removeFillerWords('呃,好的。', undefined)).toBe('呃,好的。') + }) +}) + +describe('createDictationTextPipeline', () => { + it('reads the language from a single-language model before the setting', () => { + const clean = createDictationTextPipeline({ language: 'en' }, { dictationLanguage: 'de' }) + expect(clean?.('Äh, um, ship it.')).toBe('Äh, ship it.') + }) + + it('reads the language from the setting for a multilingual model that takes it', () => { + const clean = createDictationTextPipeline( + { language: 'multilingual', dictationLanguages: ['de', 'en'] }, + { dictationLanguage: 'de' } + ) + expect(clean?.('Äh, um fünf Uhr.')).toBe('Um fünf Uhr.') + }) + + it('ignores a language setting the model ignores', () => { + const clean = createDictationTextPipeline( + { language: 'multilingual' }, + { dictationLanguage: 'en' } + ) + expect(clean?.('Er hat um zehn Uhr angerufen.')).toBe('Er hat um zehn Uhr angerufen.') + }) +}) diff --git a/src/shared/dictation-filler-words.ts b/src/shared/dictation-filler-words.ts new file mode 100644 index 00000000000..13ce8e5acc5 --- /dev/null +++ b/src/shared/dictation-filler-words.ts @@ -0,0 +1,104 @@ +// Sounds people make while thinking, per language. Never words with meaning ("like", "so", "well", "那个", "그"). +const FILLERS_BY_LANGUAGE: Record = { + en: 'u+m+|u+h+m*|er|e+r+m+|h+m+|m{3,}', + de: 'ä+h+m*|ö+h+m*|e+h+m+|h+m+|m{3,}', + es: 'e+h{2,}|e+h+m+|e+m+|h+m+|m{3,}', + fr: 'e+u+h+|h+e+u+|h+u+m+|h+m+|m{3,}', + it: 'e+h{2,}|e+h+m+|u+h+m+|h+m+|m{3,}', + pt: 'h+ã+|ã+h+|h+u+m+|h+m+|m{3,}', + nl: 'e+h+m*|u+h+m*|h+m+|m{3,}', + ru: 'э+м*|х+м+|м{3,}', + ja: 'え[ーえっ]*と|えー+|あのー+|うー+ん|んー+', + ko: '으*음+|흠+', + // 嗯 alone is also "yes", so only the one a pause follows goes. + zh: '嗯+(?=\\s*[,,、])|呃+|[额額]+' +} +FILLERS_BY_LANGUAGE.yue = FILLERS_BY_LANGUAGE.zh + +// For an unknown language: English fillers that are no word elsewhere ("um" is German and Portuguese, "er" German). +const LANGUAGE_NEUTRAL_FILLERS = 'u+h+m*|u+m{2,}|e+r+m+|h+m+|m{3,}' + +const WORD_CHAR = String.raw`[\p{L}\p{N}\p{M}_/'\u2019\-]` +// A filler stands alone: nothing joins it to a word (a dot only does inside a name like "um.txt"), +// and "hm?" or "eh!" is a question or a call, so it stays. +const standalone = (fillers: string): string => + String.raw`(? FILLERS_BY_LANGUAGE[language] ?? []) + if (lists.length === 0) { + if (NON_LATIN_LETTER_RE.test(text)) { + return null + } + lists.push(LANGUAGE_NEUTRAL_FILLERS) + } + return new RegExp(standalone(lists.join('|')) + TRAILING_PAUSE, 'giu') +} + +// In mixed-case text an all-caps "UM" or "HMM" is more likely a name or acronym than a sound. +function isAllCaps(word: string): boolean { + return word.length > 1 && word === word.toUpperCase() && word !== word.toLowerCase() +} + +function capitalizeLowercaseWord(text: string): string { + const word = /^\p{L}+/u.exec(text)?.[0] + if (!word || word !== word.toLowerCase()) { + return text + } + return text.charAt(0).toUpperCase() + text.slice(1) +} + +/** Drops standalone filler sounds and the pause punctuation after them; text with none comes back unchanged. */ +export function removeFillerWords(text: string, languages: readonly string[] | undefined): string { + const pattern = fillerPattern(languages, text) + if (!pattern) { + return text + } + let out = '' + let cursor = 0 + let capitalizeNext = false + // Some models write everything in capitals, where "UM" is still a sound. + const caseless = text === text.toUpperCase() + const keep = (segment: string): void => { + if (segment) { + out += capitalizeNext ? capitalizeLowercaseWord(segment) : segment + capitalizeNext = false + } + } + for (const match of text.matchAll(pattern)) { + const filler = match[1] + if (!caseless && isAllCaps(filler)) { + continue + } + keep(text.slice(cursor, match.index)) + cursor = match.index + match[0].length + const before = out.trimEnd() + const atSentenceStart = before === '' || SENTENCE_END_CHARS.includes(before.slice(-1)) + const sentenceEnd = SENTENCE_END_RE.exec(text.slice(cursor)) + // "we should, um." keeps its full stop; "Um." on its own goes entirely. + if (sentenceEnd && atSentenceStart) { + cursor += sentenceEnd[0].length + } else if (sentenceEnd) { + out = out.replace(TRAILING_SOFT_PUNCTUATION_RE, '') + } + // Only a capitalised "Um" began a sentence the next word now begins. + capitalizeNext ||= atSentenceStart && filler.charAt(0) !== filler.charAt(0).toLowerCase() + } + if (cursor === 0) { + return text + } + if (cursor === text.length) { + // A filler at the very end leaves the comma before it dangling. + out = out.replace(TRAILING_SOFT_PUNCTUATION_RE, '') + } else { + keep(text.slice(cursor)) + } + return out.trim() +} diff --git a/src/shared/dictation-text-pipeline.ts b/src/shared/dictation-text-pipeline.ts new file mode 100644 index 00000000000..a1fbb39d319 --- /dev/null +++ b/src/shared/dictation-text-pipeline.ts @@ -0,0 +1,36 @@ +import { engineDictationLanguage } from './dictation-language' +import { removeFillerWords } from './dictation-filler-words' +import type { SpeechModelManifest, VoiceSettings } from './speech-types' + +export type DictationTextSettings = Pick + +type DictationTextModel = Pick + +/** The languages the dictated text is in: a single-language model's own, else the one the engine is told; undefined when unknown. */ +function dictatedTextLanguages( + model: DictationTextModel | undefined, + setting: string | undefined +): string[] | undefined { + if (!model) { + return undefined + } + if (model.language !== 'multilingual') { + // Bilingual models name both, as in 'zh-en'. + return model.language.split('-') + } + // The Voice pane disables a setting the model ignores, so a stale one says nothing about what was spoken. + const language = engineDictationLanguage(model, setting) + return language ? [language] : undefined +} + +/** The passes dictated text goes through before anyone sees it, or null when every pass is off. */ +export function createDictationTextPipeline( + model: DictationTextModel | undefined, + settings: Partial | undefined +): ((text: string) => string) | null { + if (settings?.removeFillerWords === false) { + return null + } + const languages = dictatedTextLanguages(model, settings?.dictationLanguage) + return (text) => removeFillerWords(text, languages) +} diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index 94394937bc4..bd6d1a352de 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -82,6 +82,8 @@ export type VoiceSettings = { /** 'auto' or an ISO-639-1 code. Not `language`: its stored 'en' default never came from the user. */ dictationLanguage: string dictationMode: DictationMode + /** Drops "um", "uh" and their equivalents in the dictation language before text is inserted. */ + removeFillerWords: boolean terminalConfirmBeforeInsert: boolean userModels: UserModelConfig[] openAiApiKeyConfigured: boolean