diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 0268e64691b..feded50d21b 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -221,7 +221,7 @@ export function registerSpeechHandlers(store: Store): void { }, resolvedHotwordsPath, owner, - store.getSettings().voice?.dictationLanguage + store.getSettings().voice ) // Coalesced starts install no sink; only an accepted start supersedes the close hook. if (sessionReady) { diff --git a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts index a088268891b..8ac411c2da1 100644 --- a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts @@ -87,7 +87,7 @@ export class RuntimeMobileDictationController { (event) => acceptEvent(session, event), undefined, owner, - voice.dictationLanguage + voice ) if (this.sessions.get(clientId) !== session) { throw new Error('dictation_canceled') diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index 0b9076a1eda..b0bcdcb323a 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -159,6 +159,7 @@ vi.mock('./model-catalog', () => ({ id, type: 'openai', provider: 'openai', + language: 'multilingual', streaming: false, sampleRate: 16000 } @@ -167,6 +168,7 @@ vi.mock('./model-catalog', () => ({ id, type: 'whisper', provider: 'local', + language: 'multilingual', streaming: false, sampleRate: 16000, files: ['encoder.onnx', 'decoder.onnx', 'tokens.txt'], @@ -176,6 +178,7 @@ vi.mock('./model-catalog', () => ({ id: 'model-a', type: 'transducer', provider: 'local', + language: 'en', streaming: true, sampleRate: 16000, files: ['encoder.onnx', 'decoder.onnx', 'joiner.onnx', 'tokens.txt'] @@ -233,6 +236,38 @@ describe('SttService', () => { expect(worker.messages.at(-1)).toMatchObject({ type: 'feed', streamId: desktopStream }) }) + it('removes filler words from the text every host route reads, unless they are turned off', async () => { + const service = new SttService(readyModels(), spawnMockWorker) + const cleaned = vi.fn() + const raw = vi.fn() + + await service.startDictation('model-a', cleaned, undefined, 'desktop', { + removeFillerWords: true + }) + await service.startDictation('model-a', raw, undefined, 'mobile:phone-a:1', { + removeFillerWords: false + }) + service.feedAudio(new Float32Array([1]), 16000, 'desktop') + service.feedAudio(new Float32Array([2]), 16000, 'mobile:phone-a:1') + const worker = getLastWorker()! + const [cleanedStream, rawStream] = worker.messages + .filter((message) => message.type === 'feed') + .map((message) => message.streamId) + for (const streamId of [cleanedStream, rawStream]) { + worker.emit('message', { type: 'partial', streamId, text: 'Um, ship' }) + worker.emit('message', { type: 'final', streamId, text: 'Um, ship it, uh, today.' }) + } + + expect(cleaned.mock.calls.slice(1)).toEqual([ + [{ type: 'partial', text: 'Ship' }], + [{ type: 'final', text: 'Ship it, today.' }] + ]) + expect(raw.mock.calls.slice(1)).toEqual([ + [{ type: 'partial', text: 'Um, ship' }], + [{ type: 'final', text: 'Um, ship it, uh, today.' }] + ]) + }) + it("hands a renderer's audio port only to its own on-device stream", async () => { const service = new SttService(readyModels(), spawnMockWorker) await service.startDictation('model-a', vi.fn(), undefined, 'mobile:phone-a:1') @@ -294,9 +329,13 @@ describe('SttService', () => { spawnMockWorker ) - await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', 'ko') + await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', { + dictationLanguage: 'ko' + }) await service.stopDictation('desktop') - await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', 'ja') + await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', { + dictationLanguage: 'ja' + }) expect(getCreatedWorkerCount()).toBe(2) expect(getLastWorker()!.messages.find((message) => message.type === 'init')).toMatchObject({ diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 9db198abdb5..c7228ff6bfd 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -1,4 +1,9 @@ import type { MessagePortMain } from 'electron' +import { + createDictationTextPipeline, + type DictationTextSettings +} from '../../shared/dictation-text-pipeline' +import { getCatalogModel } from './model-catalog' import type { ModelManager } from './model-manager' import type { SpawnSttWorkerProcess } from './stt-worker-process' import { prewarmSttModel } from './stt-session-prewarm' @@ -22,6 +27,16 @@ export type SttEvent = export type SttEventSink = (event: SttEvent) => void +// Cleaned once here, so every route the host transcribes for gets the same text. +function cleanTranscriptEvents(sink: SttEventSink, clean: (text: string) => string): SttEventSink { + return (event) => + sink( + event.type === 'partial' || event.type === 'final' + ? { ...event, text: event.text && clean(event.text) } + : event + ) +} + export class SttService { private readonly state: SttSessionState @@ -34,9 +49,17 @@ export class SttService { sink: SttEventSink, hotwordsFilePath?: string, owner = 'desktop', - language?: string + voice?: Partial ): Promise { - return startSttDictation(this.state, modelId, sink, hotwordsFilePath, owner, language) + const clean = createDictationTextPipeline(getCatalogModel(modelId), voice) + return startSttDictation( + this.state, + modelId, + clean ? cleanTranscriptEvents(sink, clean) : sink, + hotwordsFilePath, + owner, + voice?.dictationLanguage + ) } feedAudio(samples: Float32Array, sampleRate: number, owner = 'desktop'): boolean { diff --git a/src/renderer/src/components/settings/voice/VoicePane.tsx b/src/renderer/src/components/settings/voice/VoicePane.tsx index 5c0aa7ee1cd..b85a62c59cc 100644 --- a/src/renderer/src/components/settings/voice/VoicePane.tsx +++ b/src/renderer/src/components/settings/voice/VoicePane.tsx @@ -14,6 +14,7 @@ import { VoiceDictationLanguageSetting } from './VoiceDictationLanguageSetting' import { VoiceDictationSettingsSection } from './VoiceDictationSettingsSection' import { VoiceModelUnloadSetting } from './VoiceModelUnloadSetting' import { VoiceSpeechModelSection } from './VoiceSpeechModelSection' +import { SettingsSwitchRow } from '../form-controls/SettingsFormControls' import { matchesSettingsSearch } from '../search/settings-search' import { getOpenaiTranscriptionSearchEntry } from './voice-pane-search' import { useTranslate } from '@/i18n/use-translate' @@ -230,6 +231,20 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J + + updateVoiceSettings({ removeFillerWords: !voiceSettings.removeFillerWords }) + } + /> + + + [ ...translateSearchKeyword('auto.components.settings.voice.pane.search.10d45a9fce', 'stt') ] }, + { + title: translate( + 'auto.components.settings.voice.pane.search.fillerWordsTitle', + 'Remove Filler Words' + ), + description: translate( + 'auto.components.settings.voice.pane.search.fillerWordsDescription', + 'Drop “um”, “uh” and similar sounds from dictated text.' + ), + keywords: [ + ...translateSearchKeyword('auto.components.settings.voice.pane.search.7640ed9848', 'voice'), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.089d31a45b', + 'dictation' + ), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.fillerWordsKeyword', + 'filler words' + ) + ] + }, { title: translate( 'auto.components.settings.voice.pane.search.modelUnloadTitle', diff --git a/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx b/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx index c18bbf1a35d..e861cd4ba20 100644 --- a/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx +++ b/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx @@ -86,6 +86,7 @@ export function installBrowserLinkMaterializationProbe() { modelsDir: '', dictationLanguage: 'auto', dictationMode: 'hold', + removeFillerWords: true, terminalConfirmBeforeInsert: true, userModels: [], openAiApiKeyConfigured: false, diff --git a/src/renderer/src/i18n/locales/de.json b/src/renderer/src/i18n/locales/de.json index 18b9e04c6cb..d78eb761b2c 100644 --- a/src/renderer/src/i18n/locales/de.json +++ b/src/renderer/src/i18n/locales/de.json @@ -11300,7 +11300,10 @@ "modelUnloadTitle": "Sprachmodell entladen", "modelUnloadDescription": "Lege fest, wann das lokale Sprachmodell aus dem Speicher entladen wird.", "modelUnloadKeyword": "entladen", - "modelMemoryKeyword": "speicher" + "modelMemoryKeyword": "speicher", + "fillerWordsTitle": "Füllwörter entfernen", + "fillerWordsDescription": "Entfernt „äh“, „ähm“ und ähnliche Laute aus diktiertem Text.", + "fillerWordsKeyword": "füllwörter" } } }, @@ -20664,6 +20667,10 @@ "cloud": "Cloud-Modelle laufen auf den Servern des Anbieters, daher bleibt hier nichts geladen.", "never": "Nie", "immediately": "Sofort" + }, + "fillerWords": { + "title": "Füllwörter entfernen", + "description": "Entfernt Laute wie „äh“ und „ähm“ aus diktiertem Text, passend zur Diktiersprache." } } }, diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index bf0008165e3..b3999e2cb65 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -11616,7 +11616,10 @@ "modelUnloadTitle": "Unload Speech Model", "modelUnloadDescription": "Choose when the on-device speech model is unloaded from memory.", "modelUnloadKeyword": "unload", - "modelMemoryKeyword": "memory" + "modelMemoryKeyword": "memory", + "fillerWordsTitle": "Remove Filler Words", + "fillerWordsDescription": "Drop “um”, “uh” and similar sounds from dictated text.", + "fillerWordsKeyword": "filler words" } } }, @@ -21320,6 +21323,10 @@ "cloud": "Cloud models run on the provider’s servers, so nothing stays loaded here.", "never": "Never", "immediately": "Immediately" + }, + "fillerWords": { + "title": "Remove Filler Words", + "description": "Drops sounds like “um” and “uh” from dictated text, in the dictation language." } } }, diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 99bb03043b0..afec0da848c 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -10902,7 +10902,10 @@ "modelUnloadTitle": "Descargar modelo de voz", "modelUnloadDescription": "Elige cuándo se descarga de la memoria el modelo de voz local.", "modelUnloadKeyword": "descargar", - "modelMemoryKeyword": "memoria" + "modelMemoryKeyword": "memoria", + "fillerWordsTitle": "Quitar muletillas", + "fillerWordsDescription": "Quita «ehm», «em» y sonidos parecidos del texto dictado.", + "fillerWordsKeyword": "muletillas" } } }, @@ -20471,6 +20474,10 @@ "cloud": "Los modelos en la nube se ejecutan en los servidores del proveedor, así que aquí no queda nada cargado.", "never": "Nunca", "immediately": "Inmediatamente" + }, + "fillerWords": { + "title": "Quitar muletillas", + "description": "Quita sonidos como «ehm» y «em» del texto dictado, según el idioma de dictado." } } }, diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 69490a12a5e..20adb6a78f7 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -11169,7 +11169,10 @@ "modelUnloadTitle": "Décharger le modèle vocal", "modelUnloadDescription": "Choisissez quand le modèle vocal local est déchargé de la mémoire.", "modelUnloadKeyword": "décharger", - "modelMemoryKeyword": "mémoire" + "modelMemoryKeyword": "mémoire", + "fillerWordsTitle": "Supprimer les hésitations", + "fillerWordsDescription": "Supprime « euh », « hum » et les sons similaires du texte dicté.", + "fillerWordsKeyword": "hésitations" } } }, @@ -20770,6 +20773,10 @@ "cloud": "Les modèles cloud tournent sur les serveurs du fournisseur, donc rien ne reste chargé ici.", "never": "Jamais", "immediately": "Immédiatement" + }, + "fillerWords": { + "title": "Supprimer les hésitations", + "description": "Supprime les sons comme « euh » et « hum » du texte dicté, selon la langue de dictée." } } }, diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 45d8c0100b1..32c2cdafe37 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -10645,7 +10645,10 @@ "modelUnloadTitle": "音声モデルの解放", "modelUnloadDescription": "ローカル音声モデルをメモリから解放するタイミングを選びます。", "modelUnloadKeyword": "解放", - "modelMemoryKeyword": "メモリ" + "modelMemoryKeyword": "メモリ", + "fillerWordsTitle": "フィラーを削除", + "fillerWordsDescription": "「えーと」「うーん」などのつなぎ言葉を音声入力のテキストから取り除きます。", + "fillerWordsKeyword": "フィラー" } } }, @@ -19963,6 +19966,10 @@ "cloud": "クラウドモデルはプロバイダーのサーバーで動くため、ここには何も読み込まれません。", "never": "しない", "immediately": "すぐに" + }, + "fillerWords": { + "title": "フィラーを削除", + "description": "音声入力の言語に合わせて、「えーと」「うーん」などのつなぎ言葉を入力テキストから取り除きます。" } } }, diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index de2fea3b0a6..76a5d1faa93 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -10608,7 +10608,10 @@ "modelUnloadTitle": "음성 모델 언로드", "modelUnloadDescription": "로컬 음성 모델을 메모리에서 언로드할 시점을 선택합니다.", "modelUnloadKeyword": "언로드", - "modelMemoryKeyword": "메모리" + "modelMemoryKeyword": "메모리", + "fillerWordsTitle": "군말 제거", + "fillerWordsDescription": "받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다.", + "fillerWordsKeyword": "군말" } } }, @@ -19963,6 +19966,10 @@ "cloud": "클라우드 모델은 제공업체 서버에서 실행되므로 여기에는 아무것도 로드되지 않습니다.", "never": "안 함", "immediately": "즉시" + }, + "fillerWords": { + "title": "군말 제거", + "description": "받아쓰기 언어에 맞춰 받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다." } } }, diff --git a/src/renderer/src/i18n/locales/pt.json b/src/renderer/src/i18n/locales/pt.json index ffd6d965851..f8149a363dc 100644 --- a/src/renderer/src/i18n/locales/pt.json +++ b/src/renderer/src/i18n/locales/pt.json @@ -6611,7 +6611,10 @@ "modelUnloadTitle": "Descarregar modelo de voz", "modelUnloadDescription": "Escolha quando o modelo de voz local é descarregado da memória.", "modelUnloadKeyword": "descarregar", - "modelMemoryKeyword": "memória" + "modelMemoryKeyword": "memória", + "fillerWordsTitle": "Remover hesitações", + "fillerWordsDescription": "Remove “hã”, “hum” e sons parecidos do texto ditado.", + "fillerWordsKeyword": "hesitações" } } }, @@ -21653,6 +21656,10 @@ "cloud": "Modelos na nuvem rodam nos servidores do provedor, então nada fica carregado aqui.", "never": "Nunca", "immediately": "Imediatamente" + }, + "fillerWords": { + "title": "Remover hesitações", + "description": "Remove sons como “hã” e “hum” do texto ditado, conforme o idioma do ditado." } } }, diff --git a/src/renderer/src/i18n/locales/ru.json b/src/renderer/src/i18n/locales/ru.json index 8e0c747873c..555016e3141 100644 --- a/src/renderer/src/i18n/locales/ru.json +++ b/src/renderer/src/i18n/locales/ru.json @@ -11654,7 +11654,10 @@ "modelUnloadTitle": "Выгрузка речевой модели", "modelUnloadDescription": "Выберите, когда локальная речевая модель выгружается из памяти.", "modelUnloadKeyword": "выгрузить", - "modelMemoryKeyword": "память" + "modelMemoryKeyword": "память", + "fillerWordsTitle": "Удалять слова-паразиты", + "fillerWordsDescription": "Убирает «э», «эм» и похожие звуки из надиктованного текста.", + "fillerWordsKeyword": "паразиты" } } }, @@ -21353,6 +21356,10 @@ "cloud": "Облачные модели работают на серверах провайдера, поэтому здесь ничего не загружается.", "never": "Никогда", "immediately": "Сразу" + }, + "fillerWords": { + "title": "Удалять слова-паразиты", + "description": "Убирает звуки вроде «э» и «эм» из надиктованного текста с учётом языка диктовки." } } }, diff --git a/src/renderer/src/i18n/locales/tr.json b/src/renderer/src/i18n/locales/tr.json index c2b5176ca38..8cca758d576 100644 --- a/src/renderer/src/i18n/locales/tr.json +++ b/src/renderer/src/i18n/locales/tr.json @@ -11284,7 +11284,10 @@ "modelUnloadTitle": "Konuşma modelini bellekten kaldır", "modelUnloadDescription": "Yerel konuşma modelinin bellekten ne zaman kaldırılacağını seçin.", "modelUnloadKeyword": "kaldır", - "modelMemoryKeyword": "bellek" + "modelMemoryKeyword": "bellek", + "fillerWordsTitle": "Dolgu seslerini kaldır", + "fillerWordsDescription": "Dikte edilen metinden “hmm” gibi duraksama seslerini çıkarır.", + "fillerWordsKeyword": "dolgu" } } }, @@ -20646,6 +20649,10 @@ "cloud": "Bulut modelleri sağlayıcının sunucularında çalışır, bu yüzden burada hiçbir şey yüklü kalmaz.", "never": "Asla", "immediately": "Hemen" + }, + "fillerWords": { + "title": "Dolgu seslerini kaldır", + "description": "Dikte edilen metinden “hmm” gibi duraksama seslerini dikte diline göre çıkarır." } } }, diff --git a/src/renderer/src/i18n/locales/zh-Hant.json b/src/renderer/src/i18n/locales/zh-Hant.json index b1ea1ea3968..510f4384caf 100644 --- a/src/renderer/src/i18n/locales/zh-Hant.json +++ b/src/renderer/src/i18n/locales/zh-Hant.json @@ -11121,7 +11121,10 @@ "modelUnloadTitle": "卸載語音模型", "modelUnloadDescription": "選擇何時從記憶體中卸載本機語音模型。", "modelUnloadKeyword": "卸載", - "modelMemoryKeyword": "記憶體" + "modelMemoryKeyword": "記憶體", + "fillerWordsTitle": "移除語氣詞", + "fillerWordsDescription": "從聽寫文字中移除「嗯」「呃」之類的語氣詞。", + "fillerWordsKeyword": "語氣詞" } } }, @@ -20348,6 +20351,10 @@ "cloud": "雲端模型在服務商的伺服器上執行,因此這裡不會載入任何內容。", "never": "從不", "immediately": "立即" + }, + "fillerWords": { + "title": "移除語氣詞", + "description": "依聽寫語言,從聽寫文字中移除「嗯」「呃」之類的語氣詞。" } } }, diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 16b87331748..1fb865f11e6 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -10608,7 +10608,10 @@ "modelUnloadTitle": "卸载语音模型", "modelUnloadDescription": "选择何时从内存中卸载本地语音模型。", "modelUnloadKeyword": "卸载", - "modelMemoryKeyword": "内存" + "modelMemoryKeyword": "内存", + "fillerWordsTitle": "移除语气词", + "fillerWordsDescription": "从听写文本中去掉“嗯”“呃”之类的语气词。", + "fillerWordsKeyword": "语气词" } } }, @@ -19963,6 +19966,10 @@ "cloud": "云端模型在服务商的服务器上运行,因此这里不会加载任何内容。", "never": "从不", "immediately": "立即" + }, + "fillerWords": { + "title": "移除语气词", + "description": "按听写语言,从听写文本中去掉“嗯”“呃”之类的语气词。" } } }, diff --git a/src/shared/constants.ts b/src/shared/constants.ts index ef4ddf207d9..4d2384c57f7 100644 --- a/src/shared/constants.ts +++ b/src/shared/constants.ts @@ -120,6 +120,7 @@ export function getDefaultVoiceSettings(): VoiceSettings { modelsDir: '', dictationLanguage: DICTATION_LANGUAGE_AUTO, dictationMode: 'toggle' as const, + removeFillerWords: true, terminalConfirmBeforeInsert: false, userModels: [], openAiApiKeyConfigured: false, diff --git a/src/shared/dictation-filler-words.test.ts b/src/shared/dictation-filler-words.test.ts new file mode 100644 index 00000000000..491da992928 --- /dev/null +++ b/src/shared/dictation-filler-words.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest' +import { removeFillerWords } from './dictation-filler-words' +import { createDictationTextPipeline } from './dictation-text-pipeline' + +describe('removeFillerWords', () => { + it.each([ + ['en', 'Um, I think we should, uh, ship it.', 'I think we should, ship it.'], + ['en', 'Let me check umm the logs.', 'Let me check the logs.'], + ['en', 'Yes. Uh. No.', 'Yes. No.'], + ['en', 'We should ship it, hmm', 'We should ship it'], + ['en', 'we should um.', 'we should.'], + ['en', 'We should ship, um.', 'We should ship.'], + ['en', 'um so we ship', 'so we ship'], + ['en', 'Um, iPhone builds first.', 'iPhone builds first.'], + [ + 'de', + 'Äh, das ist um fünf Uhr fertig, ähm, glaube ich.', + 'Das ist um fünf Uhr fertig, glaube ich.' + ], + ['es', 'Ehm, creo que em funciona.', 'Creo que funciona.'], + ['ru', 'Э, ну, давай так.', 'Ну, давай так.'], + ['ja', 'えーと、明日の会議は、うーん、十時です。', '明日の会議は、十時です。'], + ['ko', '음, 내일 회의는 흠, 열 시예요.', '내일 회의는 열 시예요.'], + ['zh', '嗯,我觉得,呃,这个可以。', '我觉得,这个可以。'], + ['zh', '額,我覺得可以。', '我覺得可以。'], + ['en', 'UM I THINK WE SHOULD UH SHIP IT', 'I THINK WE SHOULD SHIP IT'] + ])('%s: %s', (language, text, expected) => { + expect(removeFillerWords(text, [language])).toBe(expected) + }) + + it.each([ + ['en', 'I like it, so, well, we ship.'], + ['en', 'The umbrella is humming, uh-huh.'], + ['en', 'Hmm? What did you say?'], + ['en', 'Ask UM and HMM about it.'], + ['en', 'Open um.txt and set var_um to 3.'], + ['ja', 'ええ、そうです。'], + ['ko', '음이 너무 높아요.'], + ['zh', '金额不对,那个文件也要改。'], + ['zh', '嗯。'] + ])('%s keeps words with meaning: %s', (language, text) => { + expect(removeFillerWords(text, [language])).toBe(text) + }) + + it('uses only fillers that are no word elsewhere when the language is unknown', () => { + expect(removeFillerWords('Um carro, uh, azul.', undefined)).toBe('Um carro, azul.') + expect(removeFillerWords('Er kommt, hmm, morgen.', undefined)).toBe('Er kommt, morgen.') + expect(removeFillerWords('呃,好的。', undefined)).toBe('呃,好的。') + }) +}) + +describe('createDictationTextPipeline', () => { + it('reads the language from a single-language model before the setting', () => { + const clean = createDictationTextPipeline({ language: 'en' }, { dictationLanguage: 'de' }) + expect(clean?.('Äh, um, ship it.')).toBe('Äh, ship it.') + }) + + it('reads the language from the setting for a multilingual model that takes it', () => { + const clean = createDictationTextPipeline( + { language: 'multilingual', dictationLanguages: ['de', 'en'] }, + { dictationLanguage: 'de' } + ) + expect(clean?.('Äh, um fünf Uhr.')).toBe('Um fünf Uhr.') + }) + + it('ignores a language setting the model ignores', () => { + const clean = createDictationTextPipeline( + { language: 'multilingual' }, + { dictationLanguage: 'en' } + ) + expect(clean?.('Er hat um zehn Uhr angerufen.')).toBe('Er hat um zehn Uhr angerufen.') + }) +}) diff --git a/src/shared/dictation-filler-words.ts b/src/shared/dictation-filler-words.ts new file mode 100644 index 00000000000..13ce8e5acc5 --- /dev/null +++ b/src/shared/dictation-filler-words.ts @@ -0,0 +1,104 @@ +// Sounds people make while thinking, per language. Never words with meaning ("like", "so", "well", "那个", "그"). +const FILLERS_BY_LANGUAGE: Record = { + en: 'u+m+|u+h+m*|er|e+r+m+|h+m+|m{3,}', + de: 'ä+h+m*|ö+h+m*|e+h+m+|h+m+|m{3,}', + es: 'e+h{2,}|e+h+m+|e+m+|h+m+|m{3,}', + fr: 'e+u+h+|h+e+u+|h+u+m+|h+m+|m{3,}', + it: 'e+h{2,}|e+h+m+|u+h+m+|h+m+|m{3,}', + pt: 'h+ã+|ã+h+|h+u+m+|h+m+|m{3,}', + nl: 'e+h+m*|u+h+m*|h+m+|m{3,}', + ru: 'э+м*|х+м+|м{3,}', + ja: 'え[ーえっ]*と|えー+|あのー+|うー+ん|んー+', + ko: '으*음+|흠+', + // 嗯 alone is also "yes", so only the one a pause follows goes. + zh: '嗯+(?=\\s*[,,、])|呃+|[额額]+' +} +FILLERS_BY_LANGUAGE.yue = FILLERS_BY_LANGUAGE.zh + +// For an unknown language: English fillers that are no word elsewhere ("um" is German and Portuguese, "er" German). +const LANGUAGE_NEUTRAL_FILLERS = 'u+h+m*|u+m{2,}|e+r+m+|h+m+|m{3,}' + +const WORD_CHAR = String.raw`[\p{L}\p{N}\p{M}_/'\u2019\-]` +// A filler stands alone: nothing joins it to a word (a dot only does inside a name like "um.txt"), +// and "hm?" or "eh!" is a question or a call, so it stays. +const standalone = (fillers: string): string => + String.raw`(? FILLERS_BY_LANGUAGE[language] ?? []) + if (lists.length === 0) { + if (NON_LATIN_LETTER_RE.test(text)) { + return null + } + lists.push(LANGUAGE_NEUTRAL_FILLERS) + } + return new RegExp(standalone(lists.join('|')) + TRAILING_PAUSE, 'giu') +} + +// In mixed-case text an all-caps "UM" or "HMM" is more likely a name or acronym than a sound. +function isAllCaps(word: string): boolean { + return word.length > 1 && word === word.toUpperCase() && word !== word.toLowerCase() +} + +function capitalizeLowercaseWord(text: string): string { + const word = /^\p{L}+/u.exec(text)?.[0] + if (!word || word !== word.toLowerCase()) { + return text + } + return text.charAt(0).toUpperCase() + text.slice(1) +} + +/** Drops standalone filler sounds and the pause punctuation after them; text with none comes back unchanged. */ +export function removeFillerWords(text: string, languages: readonly string[] | undefined): string { + const pattern = fillerPattern(languages, text) + if (!pattern) { + return text + } + let out = '' + let cursor = 0 + let capitalizeNext = false + // Some models write everything in capitals, where "UM" is still a sound. + const caseless = text === text.toUpperCase() + const keep = (segment: string): void => { + if (segment) { + out += capitalizeNext ? capitalizeLowercaseWord(segment) : segment + capitalizeNext = false + } + } + for (const match of text.matchAll(pattern)) { + const filler = match[1] + if (!caseless && isAllCaps(filler)) { + continue + } + keep(text.slice(cursor, match.index)) + cursor = match.index + match[0].length + const before = out.trimEnd() + const atSentenceStart = before === '' || SENTENCE_END_CHARS.includes(before.slice(-1)) + const sentenceEnd = SENTENCE_END_RE.exec(text.slice(cursor)) + // "we should, um." keeps its full stop; "Um." on its own goes entirely. + if (sentenceEnd && atSentenceStart) { + cursor += sentenceEnd[0].length + } else if (sentenceEnd) { + out = out.replace(TRAILING_SOFT_PUNCTUATION_RE, '') + } + // Only a capitalised "Um" began a sentence the next word now begins. + capitalizeNext ||= atSentenceStart && filler.charAt(0) !== filler.charAt(0).toLowerCase() + } + if (cursor === 0) { + return text + } + if (cursor === text.length) { + // A filler at the very end leaves the comma before it dangling. + out = out.replace(TRAILING_SOFT_PUNCTUATION_RE, '') + } else { + keep(text.slice(cursor)) + } + return out.trim() +} diff --git a/src/shared/dictation-text-pipeline.ts b/src/shared/dictation-text-pipeline.ts new file mode 100644 index 00000000000..a1fbb39d319 --- /dev/null +++ b/src/shared/dictation-text-pipeline.ts @@ -0,0 +1,36 @@ +import { engineDictationLanguage } from './dictation-language' +import { removeFillerWords } from './dictation-filler-words' +import type { SpeechModelManifest, VoiceSettings } from './speech-types' + +export type DictationTextSettings = Pick + +type DictationTextModel = Pick + +/** The languages the dictated text is in: a single-language model's own, else the one the engine is told; undefined when unknown. */ +function dictatedTextLanguages( + model: DictationTextModel | undefined, + setting: string | undefined +): string[] | undefined { + if (!model) { + return undefined + } + if (model.language !== 'multilingual') { + // Bilingual models name both, as in 'zh-en'. + return model.language.split('-') + } + // The Voice pane disables a setting the model ignores, so a stale one says nothing about what was spoken. + const language = engineDictationLanguage(model, setting) + return language ? [language] : undefined +} + +/** The passes dictated text goes through before anyone sees it, or null when every pass is off. */ +export function createDictationTextPipeline( + model: DictationTextModel | undefined, + settings: Partial | undefined +): ((text: string) => string) | null { + if (settings?.removeFillerWords === false) { + return null + } + const languages = dictatedTextLanguages(model, settings?.dictationLanguage) + return (text) => removeFillerWords(text, languages) +} diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index 94394937bc4..bd6d1a352de 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -82,6 +82,8 @@ export type VoiceSettings = { /** 'auto' or an ISO-639-1 code. Not `language`: its stored 'en' default never came from the user. */ dictationLanguage: string dictationMode: DictationMode + /** Drops "um", "uh" and their equivalents in the dictation language before text is inserted. */ + removeFillerWords: boolean terminalConfirmBeforeInsert: boolean userModels: UserModelConfig[] openAiApiKeyConfigured: boolean