feat(dictation): remove filler words like “um” and “uh” from dictated text (#40137)

* feat(dictation): remove filler words from dictated text, per language

Spoken fillers (um, uh, äh, えーと, 음, 嗯 and their equivalents) are
dropped from dictated text before it is inserted, on desktop, the web
client and phones that dictate through the computer. The pass runs once
in the speech host's event sink, so every route shares it, and lives in
a new shared text pipeline that later passes join.

Only standalone sounds go, never words with meaning, and the comma after
a filler goes with it. The language comes from a single-language model
or the dictation language setting; an unknown language uses only
fillers that are no word in another language. Voice settings gain a
Remove Filler Words switch, on by default.

* fix(dictation): take the filler language from what the engine is told, and drop the comma before a final filler

* fix(dictation): keep 嗯 as a reply, clean all-caps transcripts, and read the sentence start in constant time

* refactor(dictation): one list of sentence-end marks in filler removal, and drop a duplicate off-switch test

* fix(dictation): write non-ASCII filler punctuation as escapes so the orcad bundle stays ASCII
This commit is contained in:
Kelvin Amoaba
2026-11-09 05:04:50 +00:00
committed by GitHub
parent 76b17ad670
commit 09ecfbb90f
23 changed files with 409 additions and 17 deletions
+1 -1
View File
@@ -221,7 +221,7 @@ export function registerSpeechHandlers(store: Store): void {
},
resolvedHotwordsPath,
owner,
store.getSettings().voice?.dictationLanguage
store.getSettings().voice
)
// Coalesced starts install no sink; only an accepted start supersedes the close hook.
if (sessionReady) {
@@ -87,7 +87,7 @@ export class RuntimeMobileDictationController {
(event) => acceptEvent(session, event),
undefined,
owner,
voice.dictationLanguage
voice
)
if (this.sessions.get(clientId) !== session) {
throw new Error('dictation_canceled')
+41 -2
View File
@@ -159,6 +159,7 @@ vi.mock('./model-catalog', () => ({
id,
type: 'openai',
provider: 'openai',
language: 'multilingual',
streaming: false,
sampleRate: 16000
}
@@ -167,6 +168,7 @@ vi.mock('./model-catalog', () => ({
id,
type: 'whisper',
provider: 'local',
language: 'multilingual',
streaming: false,
sampleRate: 16000,
files: ['encoder.onnx', 'decoder.onnx', 'tokens.txt'],
@@ -176,6 +178,7 @@ vi.mock('./model-catalog', () => ({
id: 'model-a',
type: 'transducer',
provider: 'local',
language: 'en',
streaming: true,
sampleRate: 16000,
files: ['encoder.onnx', 'decoder.onnx', 'joiner.onnx', 'tokens.txt']
@@ -233,6 +236,38 @@ describe('SttService', () => {
expect(worker.messages.at(-1)).toMatchObject({ type: 'feed', streamId: desktopStream })
})
it('removes filler words from the text every host route reads, unless they are turned off', async () => {
const service = new SttService(readyModels(), spawnMockWorker)
const cleaned = vi.fn<SttEventSink>()
const raw = vi.fn<SttEventSink>()
await service.startDictation('model-a', cleaned, undefined, 'desktop', {
removeFillerWords: true
})
await service.startDictation('model-a', raw, undefined, 'mobile:phone-a:1', {
removeFillerWords: false
})
service.feedAudio(new Float32Array([1]), 16000, 'desktop')
service.feedAudio(new Float32Array([2]), 16000, 'mobile:phone-a:1')
const worker = getLastWorker()!
const [cleanedStream, rawStream] = worker.messages
.filter((message) => message.type === 'feed')
.map((message) => message.streamId)
for (const streamId of [cleanedStream, rawStream]) {
worker.emit('message', { type: 'partial', streamId, text: 'Um, ship' })
worker.emit('message', { type: 'final', streamId, text: 'Um, ship it, uh, today.' })
}
expect(cleaned.mock.calls.slice(1)).toEqual([
[{ type: 'partial', text: 'Ship' }],
[{ type: 'final', text: 'Ship it, today.' }]
])
expect(raw.mock.calls.slice(1)).toEqual([
[{ type: 'partial', text: 'Um, ship' }],
[{ type: 'final', text: 'Um, ship it, uh, today.' }]
])
})
it("hands a renderer's audio port only to its own on-device stream", async () => {
const service = new SttService(readyModels(), spawnMockWorker)
await service.startDictation('model-a', vi.fn(), undefined, 'mobile:phone-a:1')
@@ -294,9 +329,13 @@ describe('SttService', () => {
spawnMockWorker
)
await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', 'ko')
await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', {
dictationLanguage: 'ko'
})
await service.stopDictation('desktop')
await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', 'ja')
await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop', {
dictationLanguage: 'ja'
})
expect(getCreatedWorkerCount()).toBe(2)
expect(getLastWorker()!.messages.find((message) => message.type === 'init')).toMatchObject({
+25 -2
View File
@@ -1,4 +1,9 @@
import type { MessagePortMain } from 'electron'
import {
createDictationTextPipeline,
type DictationTextSettings
} from '../../shared/dictation-text-pipeline'
import { getCatalogModel } from './model-catalog'
import type { ModelManager } from './model-manager'
import type { SpawnSttWorkerProcess } from './stt-worker-process'
import { prewarmSttModel } from './stt-session-prewarm'
@@ -22,6 +27,16 @@ export type SttEvent =
export type SttEventSink = (event: SttEvent) => void
// Cleaned once here, so every route the host transcribes for gets the same text.
function cleanTranscriptEvents(sink: SttEventSink, clean: (text: string) => string): SttEventSink {
return (event) =>
sink(
event.type === 'partial' || event.type === 'final'
? { ...event, text: event.text && clean(event.text) }
: event
)
}
export class SttService {
private readonly state: SttSessionState
@@ -34,9 +49,17 @@ export class SttService {
sink: SttEventSink,
hotwordsFilePath?: string,
owner = 'desktop',
language?: string
voice?: Partial<DictationTextSettings>
): Promise<void> {
return startSttDictation(this.state, modelId, sink, hotwordsFilePath, owner, language)
const clean = createDictationTextPipeline(getCatalogModel(modelId), voice)
return startSttDictation(
this.state,
modelId,
clean ? cleanTranscriptEvents(sink, clean) : sink,
hotwordsFilePath,
owner,
voice?.dictationLanguage
)
}
feedAudio(samples: Float32Array, sampleRate: number, owner = 'desktop'): boolean {
@@ -14,6 +14,7 @@ import { VoiceDictationLanguageSetting } from './VoiceDictationLanguageSetting'
import { VoiceDictationSettingsSection } from './VoiceDictationSettingsSection'
import { VoiceModelUnloadSetting } from './VoiceModelUnloadSetting'
import { VoiceSpeechModelSection } from './VoiceSpeechModelSection'
import { SettingsSwitchRow } from '../form-controls/SettingsFormControls'
import { matchesSettingsSearch } from '../search/settings-search'
import { getOpenaiTranscriptionSearchEntry } from './voice-pane-search'
import { useTranslate } from '@/i18n/use-translate'
@@ -230,6 +231,20 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J
<Separator />
<SettingsSwitchRow
label={t('components.settings.voice.fillerWords.title', 'Remove Filler Words')}
description={t(
'components.settings.voice.fillerWords.description',
'Drops sounds like “um” and “uh” from dictated text, in the dictation language.'
)}
checked={voiceSettings.removeFillerWords}
onChange={() =>
updateVoiceSettings({ removeFillerWords: !voiceSettings.removeFillerWords })
}
/>
<Separator />
<VoiceModelUnloadSetting
voiceSettings={voiceSettings}
selectedModel={selectedModel}
@@ -120,6 +120,27 @@ export const getVoicePaneSearchEntries = createLocalizedCatalog(() => [
...translateSearchKeyword('auto.components.settings.voice.pane.search.10d45a9fce', 'stt')
]
},
{
title: translate(
'auto.components.settings.voice.pane.search.fillerWordsTitle',
'Remove Filler Words'
),
description: translate(
'auto.components.settings.voice.pane.search.fillerWordsDescription',
'Drop “um”, “uh” and similar sounds from dictated text.'
),
keywords: [
...translateSearchKeyword('auto.components.settings.voice.pane.search.7640ed9848', 'voice'),
...translateSearchKeyword(
'auto.components.settings.voice.pane.search.089d31a45b',
'dictation'
),
...translateSearchKeyword(
'auto.components.settings.voice.pane.search.fillerWordsKeyword',
'filler words'
)
]
},
{
title: translate(
'auto.components.settings.voice.pane.search.modelUnloadTitle',
@@ -86,6 +86,7 @@ export function installBrowserLinkMaterializationProbe() {
modelsDir: '',
dictationLanguage: 'auto',
dictationMode: 'hold',
removeFillerWords: true,
terminalConfirmBeforeInsert: true,
userModels: [],
openAiApiKeyConfigured: false,
+8 -1
View File
@@ -11300,7 +11300,10 @@
"modelUnloadTitle": "Sprachmodell entladen",
"modelUnloadDescription": "Lege fest, wann das lokale Sprachmodell aus dem Speicher entladen wird.",
"modelUnloadKeyword": "entladen",
"modelMemoryKeyword": "speicher"
"modelMemoryKeyword": "speicher",
"fillerWordsTitle": "Füllwörter entfernen",
"fillerWordsDescription": "Entfernt „äh“, „ähm“ und ähnliche Laute aus diktiertem Text.",
"fillerWordsKeyword": "füllwörter"
}
}
},
@@ -20664,6 +20667,10 @@
"cloud": "Cloud-Modelle laufen auf den Servern des Anbieters, daher bleibt hier nichts geladen.",
"never": "Nie",
"immediately": "Sofort"
},
"fillerWords": {
"title": "Füllwörter entfernen",
"description": "Entfernt Laute wie „äh“ und „ähm“ aus diktiertem Text, passend zur Diktiersprache."
}
}
},
+8 -1
View File
@@ -11616,7 +11616,10 @@
"modelUnloadTitle": "Unload Speech Model",
"modelUnloadDescription": "Choose when the on-device speech model is unloaded from memory.",
"modelUnloadKeyword": "unload",
"modelMemoryKeyword": "memory"
"modelMemoryKeyword": "memory",
"fillerWordsTitle": "Remove Filler Words",
"fillerWordsDescription": "Drop “um”, “uh” and similar sounds from dictated text.",
"fillerWordsKeyword": "filler words"
}
}
},
@@ -21320,6 +21323,10 @@
"cloud": "Cloud models run on the provider’s servers, so nothing stays loaded here.",
"never": "Never",
"immediately": "Immediately"
},
"fillerWords": {
"title": "Remove Filler Words",
"description": "Drops sounds like “um” and “uh” from dictated text, in the dictation language."
}
}
},
+8 -1
View File
@@ -10902,7 +10902,10 @@
"modelUnloadTitle": "Descargar modelo de voz",
"modelUnloadDescription": "Elige cuándo se descarga de la memoria el modelo de voz local.",
"modelUnloadKeyword": "descargar",
"modelMemoryKeyword": "memoria"
"modelMemoryKeyword": "memoria",
"fillerWordsTitle": "Quitar muletillas",
"fillerWordsDescription": "Quita «ehm», «em» y sonidos parecidos del texto dictado.",
"fillerWordsKeyword": "muletillas"
}
}
},
@@ -20471,6 +20474,10 @@
"cloud": "Los modelos en la nube se ejecutan en los servidores del proveedor, así que aquí no queda nada cargado.",
"never": "Nunca",
"immediately": "Inmediatamente"
},
"fillerWords": {
"title": "Quitar muletillas",
"description": "Quita sonidos como «ehm» y «em» del texto dictado, según el idioma de dictado."
}
}
},
+8 -1
View File
@@ -11169,7 +11169,10 @@
"modelUnloadTitle": "Décharger le modèle vocal",
"modelUnloadDescription": "Choisissez quand le modèle vocal local est déchargé de la mémoire.",
"modelUnloadKeyword": "décharger",
"modelMemoryKeyword": "mémoire"
"modelMemoryKeyword": "mémoire",
"fillerWordsTitle": "Supprimer les hésitations",
"fillerWordsDescription": "Supprime « euh », « hum » et les sons similaires du texte dicté.",
"fillerWordsKeyword": "hésitations"
}
}
},
@@ -20770,6 +20773,10 @@
"cloud": "Les modèles cloud tournent sur les serveurs du fournisseur, donc rien ne reste chargé ici.",
"never": "Jamais",
"immediately": "Immédiatement"
},
"fillerWords": {
"title": "Supprimer les hésitations",
"description": "Supprime les sons comme « euh » et « hum » du texte dicté, selon la langue de dictée."
}
}
},
+8 -1
View File
@@ -10645,7 +10645,10 @@
"modelUnloadTitle": "音声モデルの解放",
"modelUnloadDescription": "ローカル音声モデルをメモリから解放するタイミングを選びます。",
"modelUnloadKeyword": "解放",
"modelMemoryKeyword": "メモリ"
"modelMemoryKeyword": "メモリ",
"fillerWordsTitle": "フィラーを削除",
"fillerWordsDescription": "「えーと」「うーん」などのつなぎ言葉を音声入力のテキストから取り除きます。",
"fillerWordsKeyword": "フィラー"
}
}
},
@@ -19963,6 +19966,10 @@
"cloud": "クラウドモデルはプロバイダーのサーバーで動くため、ここには何も読み込まれません。",
"never": "しない",
"immediately": "すぐに"
},
"fillerWords": {
"title": "フィラーを削除",
"description": "音声入力の言語に合わせて、「えーと」「うーん」などのつなぎ言葉を入力テキストから取り除きます。"
}
}
},
+8 -1
View File
@@ -10608,7 +10608,10 @@
"modelUnloadTitle": "음성 모델 언로드",
"modelUnloadDescription": "로컬 음성 모델을 메모리에서 언로드할 시점을 선택합니다.",
"modelUnloadKeyword": "언로드",
"modelMemoryKeyword": "메모리"
"modelMemoryKeyword": "메모리",
"fillerWordsTitle": "군말 제거",
"fillerWordsDescription": "받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다.",
"fillerWordsKeyword": "군말"
}
}
},
@@ -19963,6 +19966,10 @@
"cloud": "클라우드 모델은 제공업체 서버에서 실행되므로 여기에는 아무것도 로드되지 않습니다.",
"never": "안 함",
"immediately": "즉시"
},
"fillerWords": {
"title": "군말 제거",
"description": "받아쓰기 언어에 맞춰 받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다."
}
}
},
+8 -1
View File
@@ -6611,7 +6611,10 @@
"modelUnloadTitle": "Descarregar modelo de voz",
"modelUnloadDescription": "Escolha quando o modelo de voz local é descarregado da memória.",
"modelUnloadKeyword": "descarregar",
"modelMemoryKeyword": "memória"
"modelMemoryKeyword": "memória",
"fillerWordsTitle": "Remover hesitações",
"fillerWordsDescription": "Remove “hã”, “hum” e sons parecidos do texto ditado.",
"fillerWordsKeyword": "hesitações"
}
}
},
@@ -21653,6 +21656,10 @@
"cloud": "Modelos na nuvem rodam nos servidores do provedor, então nada fica carregado aqui.",
"never": "Nunca",
"immediately": "Imediatamente"
},
"fillerWords": {
"title": "Remover hesitações",
"description": "Remove sons como “hã” e “hum” do texto ditado, conforme o idioma do ditado."
}
}
},
+8 -1
View File
@@ -11654,7 +11654,10 @@
"modelUnloadTitle": "Выгрузка речевой модели",
"modelUnloadDescription": "Выберите, когда локальная речевая модель выгружается из памяти.",
"modelUnloadKeyword": "выгрузить",
"modelMemoryKeyword": "память"
"modelMemoryKeyword": "память",
"fillerWordsTitle": "Удалять слова-паразиты",
"fillerWordsDescription": "Убирает «э», «эм» и похожие звуки из надиктованного текста.",
"fillerWordsKeyword": "паразиты"
}
}
},
@@ -21353,6 +21356,10 @@
"cloud": "Облачные модели работают на серверах провайдера, поэтому здесь ничего не загружается.",
"never": "Никогда",
"immediately": "Сразу"
},
"fillerWords": {
"title": "Удалять слова-паразиты",
"description": "Убирает звуки вроде «э» и «эм» из надиктованного текста с учётом языка диктовки."
}
}
},
+8 -1
View File
@@ -11284,7 +11284,10 @@
"modelUnloadTitle": "Konuşma modelini bellekten kaldır",
"modelUnloadDescription": "Yerel konuşma modelinin bellekten ne zaman kaldırılacağını seçin.",
"modelUnloadKeyword": "kaldır",
"modelMemoryKeyword": "bellek"
"modelMemoryKeyword": "bellek",
"fillerWordsTitle": "Dolgu seslerini kaldır",
"fillerWordsDescription": "Dikte edilen metinden “hmm” gibi duraksama seslerini çıkarır.",
"fillerWordsKeyword": "dolgu"
}
}
},
@@ -20646,6 +20649,10 @@
"cloud": "Bulut modelleri sağlayıcının sunucularında çalışır, bu yüzden burada hiçbir şey yüklü kalmaz.",
"never": "Asla",
"immediately": "Hemen"
},
"fillerWords": {
"title": "Dolgu seslerini kaldır",
"description": "Dikte edilen metinden “hmm” gibi duraksama seslerini dikte diline göre çıkarır."
}
}
},
+8 -1
View File
@@ -11121,7 +11121,10 @@
"modelUnloadTitle": "卸載語音模型",
"modelUnloadDescription": "選擇何時從記憶體中卸載本機語音模型。",
"modelUnloadKeyword": "卸載",
"modelMemoryKeyword": "記憶體"
"modelMemoryKeyword": "記憶體",
"fillerWordsTitle": "移除語氣詞",
"fillerWordsDescription": "從聽寫文字中移除「嗯」「呃」之類的語氣詞。",
"fillerWordsKeyword": "語氣詞"
}
}
},
@@ -20348,6 +20351,10 @@
"cloud": "雲端模型在服務商的伺服器上執行,因此這裡不會載入任何內容。",
"never": "從不",
"immediately": "立即"
},
"fillerWords": {
"title": "移除語氣詞",
"description": "依聽寫語言,從聽寫文字中移除「嗯」「呃」之類的語氣詞。"
}
}
},
+8 -1
View File
@@ -10608,7 +10608,10 @@
"modelUnloadTitle": "卸载语音模型",
"modelUnloadDescription": "选择何时从内存中卸载本地语音模型。",
"modelUnloadKeyword": "卸载",
"modelMemoryKeyword": "内存"
"modelMemoryKeyword": "内存",
"fillerWordsTitle": "移除语气词",
"fillerWordsDescription": "从听写文本中去掉“嗯”“呃”之类的语气词。",
"fillerWordsKeyword": "语气词"
}
}
},
@@ -19963,6 +19966,10 @@
"cloud": "云端模型在服务商的服务器上运行,因此这里不会加载任何内容。",
"never": "从不",
"immediately": "立即"
},
"fillerWords": {
"title": "移除语气词",
"description": "按听写语言,从听写文本中去掉“嗯”“呃”之类的语气词。"
}
}
},
+1
View File
@@ -120,6 +120,7 @@ export function getDefaultVoiceSettings(): VoiceSettings {
modelsDir: '',
dictationLanguage: DICTATION_LANGUAGE_AUTO,
dictationMode: 'toggle' as const,
removeFillerWords: true,
terminalConfirmBeforeInsert: false,
userModels: [],
openAiApiKeyConfigured: false,
+73
View File
@@ -0,0 +1,73 @@
import { describe, expect, it } from 'vitest'
import { removeFillerWords } from './dictation-filler-words'
import { createDictationTextPipeline } from './dictation-text-pipeline'
describe('removeFillerWords', () => {
it.each([
['en', 'Um, I think we should, uh, ship it.', 'I think we should, ship it.'],
['en', 'Let me check umm the logs.', 'Let me check the logs.'],
['en', 'Yes. Uh. No.', 'Yes. No.'],
['en', 'We should ship it, hmm', 'We should ship it'],
['en', 'we should um.', 'we should.'],
['en', 'We should ship, um.', 'We should ship.'],
['en', 'um so we ship', 'so we ship'],
['en', 'Um, iPhone builds first.', 'iPhone builds first.'],
[
'de',
'Äh, das ist um fünf Uhr fertig, ähm, glaube ich.',
'Das ist um fünf Uhr fertig, glaube ich.'
],
['es', 'Ehm, creo que em funciona.', 'Creo que funciona.'],
['ru', 'Э, ну, давай так.', 'Ну, давай так.'],
['ja', 'えーと、明日の会議は、うーん、十時です。', '明日の会議は、十時です。'],
['ko', '음, 내일 회의는 흠, 열 시예요.', '내일 회의는 열 시예요.'],
['zh', '嗯,我觉得,呃,这个可以。', '我觉得,这个可以。'],
['zh', '額,我覺得可以。', '我覺得可以。'],
['en', 'UM I THINK WE SHOULD UH SHIP IT', 'I THINK WE SHOULD SHIP IT']
])('%s: %s', (language, text, expected) => {
expect(removeFillerWords(text, [language])).toBe(expected)
})
it.each([
['en', 'I like it, so, well, we ship.'],
['en', 'The umbrella is humming, uh-huh.'],
['en', 'Hmm? What did you say?'],
['en', 'Ask UM and HMM about it.'],
['en', 'Open um.txt and set var_um to 3.'],
['ja', 'ええ、そうです。'],
['ko', '음이 너무 높아요.'],
['zh', '金额不对,那个文件也要改。'],
['zh', '嗯。']
])('%s keeps words with meaning: %s', (language, text) => {
expect(removeFillerWords(text, [language])).toBe(text)
})
it('uses only fillers that are no word elsewhere when the language is unknown', () => {
expect(removeFillerWords('Um carro, uh, azul.', undefined)).toBe('Um carro, azul.')
expect(removeFillerWords('Er kommt, hmm, morgen.', undefined)).toBe('Er kommt, morgen.')
expect(removeFillerWords('呃,好的。', undefined)).toBe('呃,好的。')
})
})
describe('createDictationTextPipeline', () => {
it('reads the language from a single-language model before the setting', () => {
const clean = createDictationTextPipeline({ language: 'en' }, { dictationLanguage: 'de' })
expect(clean?.('Äh, um, ship it.')).toBe('Äh, ship it.')
})
it('reads the language from the setting for a multilingual model that takes it', () => {
const clean = createDictationTextPipeline(
{ language: 'multilingual', dictationLanguages: ['de', 'en'] },
{ dictationLanguage: 'de' }
)
expect(clean?.('Äh, um fünf Uhr.')).toBe('Um fünf Uhr.')
})
it('ignores a language setting the model ignores', () => {
const clean = createDictationTextPipeline(
{ language: 'multilingual' },
{ dictationLanguage: 'en' }
)
expect(clean?.('Er hat um zehn Uhr angerufen.')).toBe('Er hat um zehn Uhr angerufen.')
})
})
+104
View File
@@ -0,0 +1,104 @@
// Sounds people make while thinking, per language. Never words with meaning ("like", "so", "well", "那个", "그").
const FILLERS_BY_LANGUAGE: Record<string, string> = {
en: 'u+m+|u+h+m*|er|e+r+m+|h+m+|m{3,}',
de: 'ä+h+m*|ö+h+m*|e+h+m+|h+m+|m{3,}',
es: 'e+h{2,}|e+h+m+|e+m+|h+m+|m{3,}',
fr: 'e+u+h+|h+e+u+|h+u+m+|h+m+|m{3,}',
it: 'e+h{2,}|e+h+m+|u+h+m+|h+m+|m{3,}',
pt: 'h+ã+|ã+h+|h+u+m+|h+m+|m{3,}',
nl: 'e+h+m*|u+h+m*|h+m+|m{3,}',
ru: 'э+м*|х+м+|м{3,}',
ja: 'え[ーえっ]*と|えー+|あのー+|うー+ん|んー+',
ko: '으*음+|흠+',
// 嗯 alone is also "yes", so only the one a pause follows goes.
zh: '嗯+(?=\\s*[,,、])|呃+|[额額]+'
}
FILLERS_BY_LANGUAGE.yue = FILLERS_BY_LANGUAGE.zh
// For an unknown language: English fillers that are no word elsewhere ("um" is German and Portuguese, "er" German).
const LANGUAGE_NEUTRAL_FILLERS = 'u+h+m*|u+m{2,}|e+r+m+|h+m+|m{3,}'
const WORD_CHAR = String.raw`[\p{L}\p{N}\p{M}_/'\u2019\-]`
// A filler stands alone: nothing joins it to a word (a dot only does inside a name like "um.txt"),
// and "hm?" or "eh!" is a question or a call, so it stays.
const standalone = (fillers: string): string =>
String.raw`(?<!${WORD_CHAR}|${WORD_CHAR}\.)(${fillers})(?!${WORD_CHAR}|\.${WORD_CHAR})(?!\s*[?\uFF1F!\uFF01])`
// The comma or ellipsis after a filler goes with it.
const TRAILING_PAUSE = String.raw`(?:\s*(?:[,\uFF0C\u3001\u2026]|\.{3}))?\s*`
const SENTENCE_END_CHARS = '.!?。!?'
const SENTENCE_END_RE = new RegExp(`^[${SENTENCE_END_CHARS}]+\\s*`, 'u')
// A comma left in front of a removed filler that ended the sentence or the text.
const TRAILING_SOFT_PUNCTUATION_RE = /[\s,\uFF0C\u3001]+$/u
const NON_LATIN_LETTER_RE = /(?=\p{L})\P{Script=Latin}/u
function fillerPattern(languages: readonly string[] | undefined, text: string): RegExp | null {
const lists = (languages ?? []).flatMap((language) => FILLERS_BY_LANGUAGE[language] ?? [])
if (lists.length === 0) {
if (NON_LATIN_LETTER_RE.test(text)) {
return null
}
lists.push(LANGUAGE_NEUTRAL_FILLERS)
}
return new RegExp(standalone(lists.join('|')) + TRAILING_PAUSE, 'giu')
}
// In mixed-case text an all-caps "UM" or "HMM" is more likely a name or acronym than a sound.
function isAllCaps(word: string): boolean {
return word.length > 1 && word === word.toUpperCase() && word !== word.toLowerCase()
}
function capitalizeLowercaseWord(text: string): string {
const word = /^\p{L}+/u.exec(text)?.[0]
if (!word || word !== word.toLowerCase()) {
return text
}
return text.charAt(0).toUpperCase() + text.slice(1)
}
/** Drops standalone filler sounds and the pause punctuation after them; text with none comes back unchanged. */
export function removeFillerWords(text: string, languages: readonly string[] | undefined): string {
const pattern = fillerPattern(languages, text)
if (!pattern) {
return text
}
let out = ''
let cursor = 0
let capitalizeNext = false
// Some models write everything in capitals, where "UM" is still a sound.
const caseless = text === text.toUpperCase()
const keep = (segment: string): void => {
if (segment) {
out += capitalizeNext ? capitalizeLowercaseWord(segment) : segment
capitalizeNext = false
}
}
for (const match of text.matchAll(pattern)) {
const filler = match[1]
if (!caseless && isAllCaps(filler)) {
continue
}
keep(text.slice(cursor, match.index))
cursor = match.index + match[0].length
const before = out.trimEnd()
const atSentenceStart = before === '' || SENTENCE_END_CHARS.includes(before.slice(-1))
const sentenceEnd = SENTENCE_END_RE.exec(text.slice(cursor))
// "we should, um." keeps its full stop; "Um." on its own goes entirely.
if (sentenceEnd && atSentenceStart) {
cursor += sentenceEnd[0].length
} else if (sentenceEnd) {
out = out.replace(TRAILING_SOFT_PUNCTUATION_RE, '')
}
// Only a capitalised "Um" began a sentence the next word now begins.
capitalizeNext ||= atSentenceStart && filler.charAt(0) !== filler.charAt(0).toLowerCase()
}
if (cursor === 0) {
return text
}
if (cursor === text.length) {
// A filler at the very end leaves the comma before it dangling.
out = out.replace(TRAILING_SOFT_PUNCTUATION_RE, '')
} else {
keep(text.slice(cursor))
}
return out.trim()
}
+36
View File
@@ -0,0 +1,36 @@
import { engineDictationLanguage } from './dictation-language'
import { removeFillerWords } from './dictation-filler-words'
import type { SpeechModelManifest, VoiceSettings } from './speech-types'
export type DictationTextSettings = Pick<VoiceSettings, 'dictationLanguage' | 'removeFillerWords'>
type DictationTextModel = Pick<SpeechModelManifest, 'language' | 'dictationLanguages'>
/** The languages the dictated text is in: a single-language model's own, else the one the engine is told; undefined when unknown. */
function dictatedTextLanguages(
model: DictationTextModel | undefined,
setting: string | undefined
): string[] | undefined {
if (!model) {
return undefined
}
if (model.language !== 'multilingual') {
// Bilingual models name both, as in 'zh-en'.
return model.language.split('-')
}
// The Voice pane disables a setting the model ignores, so a stale one says nothing about what was spoken.
const language = engineDictationLanguage(model, setting)
return language ? [language] : undefined
}
/** The passes dictated text goes through before anyone sees it, or null when every pass is off. */
export function createDictationTextPipeline(
model: DictationTextModel | undefined,
settings: Partial<DictationTextSettings> | undefined
): ((text: string) => string) | null {
if (settings?.removeFillerWords === false) {
return null
}
const languages = dictatedTextLanguages(model, settings?.dictationLanguage)
return (text) => removeFillerWords(text, languages)
}
+2
View File
@@ -82,6 +82,8 @@ export type VoiceSettings = {
/** 'auto' or an ISO-639-1 code. Not `language`: its stored 'en' default never came from the user. */
dictationLanguage: string
dictationMode: DictationMode
/** Drops "um", "uh" and their equivalents in the dictation language before text is inserted. */
removeFillerWords: boolean
terminalConfirmBeforeInsert: boolean
userModels: UserModelConfig[]
openAiApiKeyConfigured: boolean