From e6c16b4efc49df8ead75ccd64538e4a8f6cfcedb Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:01:00 +0000 Subject: [PATCH] feat(speech): record each model's languages, licence and measured error rates in the catalog Error rates come from Orca's speech bench (transcribe-cpp 0.4.1, Metal, pinned GGUFs): LibriSpeech for English, 5 FLEURS dev clips for de, es, fr, ja, ko, pt, ru, tr and zh. speechModelQuality turns them into the Voice settings word scale. --- src/main/speech/model-catalog.ts | 33 +++++++- src/main/speech/speech-model-error-rates.ts | 79 +++++++++++++++++++ .../speech-model-update-target.test.ts | 1 + .../VoiceDictationLanguageSetting.fixture.tsx | 1 + .../VoiceDictationLanguageSetting.test.tsx | 2 + .../voice/VoiceSpeechModelSection.fixture.tsx | 1 + .../voice/VoiceSpeechModelSection.test.tsx | 1 + src/shared/dictation-language.ts | 10 +++ src/shared/speech-model-quality.test.ts | 40 ++++++++++ src/shared/speech-model-quality.ts | 46 +++++++++++ src/shared/speech-types.ts | 18 +++++ 11 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 src/main/speech/speech-model-error-rates.ts create mode 100644 src/shared/speech-model-quality.test.ts create mode 100644 src/shared/speech-model-quality.ts diff --git a/src/main/speech/model-catalog.ts b/src/main/speech/model-catalog.ts index a02730cd21f..97ab776a901 100644 --- a/src/main/speech/model-catalog.ts +++ b/src/main/speech/model-catalog.ts @@ -2,10 +2,13 @@ import type { SpeechModelManifest } from '../../shared/speech-types' import { DICTATION_LANGUAGES, NEMOTRON_DICTATION_LANGUAGES, - SENSE_VOICE_DICTATION_LANGUAGES + PARAKEET_V3_LANGUAGES, + SENSE_VOICE_DICTATION_LANGUAGES, + WHISPER_LANGUAGES } from '../../shared/dictation-language' import { getSpeechModelDownloadMetadata } from './model-download-catalog' import { getGgufSpeechModelDownload } from './speech-model-gguf-downloads' +import { getSpeechModelErrorRates } from './speech-model-error-rates' const legacy = (id: Parameters[0]) => ({ legacyDownloadFiles: getSpeechModelDownloadMetadata(id).downloadFiles @@ -22,6 +25,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ language: 'multilingual', ...getGgufSpeechModelDownload('parakeet-tdt-0.6b-v3-int8'), ...legacy('parakeet-tdt-0.6b-v3-int8'), + languages: PARAKEET_V3_LANGUAGES, + ...getSpeechModelErrorRates('parakeet-tdt-0.6b-v3-int8'), + licence: 'CC-BY-4.0', sampleRate: 16000, streaming: false, recommended: true @@ -36,6 +42,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ language: 'en', ...getGgufSpeechModelDownload('parakeet-tdt-0.6b-v2-int8'), ...legacy('parakeet-tdt-0.6b-v2-int8'), + languages: ['en'], + ...getSpeechModelErrorRates('parakeet-tdt-0.6b-v2-int8'), + licence: 'CC-BY-4.0', sampleRate: 16000, streaming: false }, @@ -47,6 +56,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'local', language: 'zh-en', ...getSpeechModelDownloadMetadata('zipformer-bilingual-zh-en'), + languages: ['zh', 'en'], sampleRate: 16000, streaming: true, // transcribe-cpp has no zipformer; SenseVoice covers Chinese and English. @@ -61,6 +71,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'local', language: 'zh-en', ...getSpeechModelDownloadMetadata('paraformer-bilingual-zh-en'), + languages: ['zh', 'en'], sampleRate: 16000, streaming: true, // transcribe-cpp has no paraformer; SenseVoice covers Chinese and English. @@ -74,6 +85,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'local', language: 'en', ...getSpeechModelDownloadMetadata('zipformer-streaming-en-20m'), + languages: ['en'], sampleRate: 16000, streaming: true, // The model itself loses the first words of every utterance. @@ -87,6 +99,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'local', language: 'zh', ...getSpeechModelDownloadMetadata('zipformer-streaming-zh-14m'), + languages: ['zh'], sampleRate: 16000, streaming: true, // transcribe-cpp has no zipformer; SenseVoice beat Nemotron 3.5 on Korean and Chinese. @@ -100,6 +113,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'local', language: 'ko', ...getSpeechModelDownloadMetadata('zipformer-streaming-korean'), + languages: ['ko'], sampleRate: 16000, streaming: true, // transcribe-cpp has no zipformer; SenseVoice beat Nemotron 3.5 on Korean and Chinese. @@ -113,6 +127,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'local', language: 'ja', ...getSpeechModelDownloadMetadata('parakeet-tdt-ctc-0.6b-ja-int8'), + languages: ['ja'], sampleRate: 16000, streaming: false, // No GGUF conversion exists; SenseVoice covers Japanese. @@ -128,6 +143,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ ...getGgufSpeechModelDownload('whisper-tiny'), ...legacy('whisper-tiny'), dictationLanguages: DICTATION_LANGUAGES, + languages: WHISPER_LANGUAGES, + ...getSpeechModelErrorRates('whisper-tiny'), + licence: 'Apache-2.0', sampleRate: 16000, streaming: false }, @@ -141,6 +159,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ language: 'multilingual', ...getGgufSpeechModelDownload('whisper-large-v3-turbo'), dictationLanguages: DICTATION_LANGUAGES, + languages: [...WHISPER_LANGUAGES, 'yue'], + ...getSpeechModelErrorRates('whisper-large-v3-turbo'), + licence: 'MIT', sampleRate: 16000, streaming: false }, @@ -155,6 +176,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ ...getGgufSpeechModelDownload('sense-voice-zh-en-ja-ko-yue'), ...legacy('sense-voice-zh-en-ja-ko-yue'), dictationLanguages: SENSE_VOICE_DICTATION_LANGUAGES, + languages: SENSE_VOICE_DICTATION_LANGUAGES, + ...getSpeechModelErrorRates('sense-voice-zh-en-ja-ko-yue'), + licence: 'FunASR Model License', sampleRate: 16000, streaming: false }, @@ -168,6 +192,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ language: 'multilingual', ...getGgufSpeechModelDownload('nemotron-3.5-asr-streaming-0.6b'), dictationLanguages: NEMOTRON_DICTATION_LANGUAGES, + languages: NEMOTRON_DICTATION_LANGUAGES, + ...getSpeechModelErrorRates('nemotron-3.5-asr-streaming-0.6b'), + licence: 'OpenMDW-1.1', sampleRate: 16000, // Decoded per chunk for now; live text arrives with the streaming engine path. streaming: false @@ -180,6 +207,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'openai', language: 'multilingual', dictationLanguages: DICTATION_LANGUAGES, + languages: DICTATION_LANGUAGES, sampleRate: 16000, streaming: false }, @@ -192,6 +220,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'openai-compatible', language: 'multilingual', dictationLanguages: DICTATION_LANGUAGES, + languages: DICTATION_LANGUAGES, sampleRate: 16000, streaming: false }, @@ -204,6 +233,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'openai', language: 'multilingual', dictationLanguages: DICTATION_LANGUAGES, + languages: DICTATION_LANGUAGES, sampleRate: 16000, streaming: false, // OpenAI shuts the gpt-4o transcribe models down on 2027-02-26. @@ -217,6 +247,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ provider: 'openai', language: 'multilingual', dictationLanguages: DICTATION_LANGUAGES, + languages: DICTATION_LANGUAGES, sampleRate: 16000, streaming: false, replacedBy: 'openai-gpt-transcribe' diff --git a/src/main/speech/speech-model-error-rates.ts b/src/main/speech/speech-model-error-rates.ts new file mode 100644 index 00000000000..9418e179081 --- /dev/null +++ b/src/main/speech/speech-model-error-rates.ts @@ -0,0 +1,79 @@ +import type { SpeechModelErrorRate } from '../../shared/speech-types' + +type Measured = readonly [language: string, percent: number, measuredOn: string] + +const LIBRISPEECH_CLIPS = 73 +const FLEURS_CLIPS = 5 +const CHARACTER_SCORED = new Set(['ja', 'ko', 'zh']) + +// Orca's speech bench: transcribe-cpp 0.4.1 with Metal on an M4 Max, the catalog's pinned GGUF +// files, one run per clip. English is LibriSpeech (73 clips); other languages are FLEURS dev +// (5 clips each, so rough). Each clip's language was sent the way Orca sends it; Parakeet ignores it. +const MEASURED = { + 'parakeet-tdt-0.6b-v3-int8': [ + ['en', 3.7, '2026-11-06'], + ['de', 2.2, '2026-11-06'], + ['es', 1.7, '2026-11-06'], + ['fr', 8.3, '2026-11-09'], + ['pt', 11.3, '2026-11-09'], + ['ru', 3.4, '2026-11-06'] + ], + 'parakeet-tdt-0.6b-v2-int8': [['en', 2.8, '2026-11-06']], + 'whisper-tiny': [ + ['en', 13, '2026-11-09'], + ['de', 10, '2026-11-09'], + ['es', 13.6, '2026-11-09'], + ['fr', 52.3, '2026-11-09'], + ['ja', 69.3, '2026-11-09'], + ['ko', 24.9, '2026-11-09'], + ['pt', 31.1, '2026-11-09'], + ['ru', 38.2, '2026-11-09'], + ['tr', 49.5, '2026-11-09'], + ['zh', 47.5, '2026-11-09'] + ], + 'whisper-large-v3-turbo': [ + ['en', 4.5, '2026-11-09'], + ['de', 1.1, '2026-11-09'], + ['es', 1.7, '2026-11-09'], + ['fr', 5.3, '2026-11-09'], + ['ja', 7.4, '2026-11-09'], + ['ko', 2.6, '2026-11-09'], + ['pt', 8.5, '2026-11-09'], + ['ru', 3.4, '2026-11-09'], + ['tr', 7.7, '2026-11-09'], + ['zh', 9.4, '2026-11-09'] + ], + 'sense-voice-zh-en-ja-ko-yue': [ + ['en', 6.9, '2026-11-09'], + ['ja', 10.7, '2026-11-09'], + ['ko', 10, '2026-11-09'], + ['zh', 12.9, '2026-11-09'] + ], + 'nemotron-3.5-asr-streaming-0.6b': [ + ['en', 5.3, '2026-11-09'], + ['de', 5.6, '2026-11-09'], + ['es', 2.5, '2026-11-09'], + ['fr', 9.8, '2026-11-09'], + ['ja', 17.2, '2026-11-09'], + ['ko', 11.8, '2026-11-09'], + ['pt', 13.2, '2026-11-09'], + ['ru', 14.6, '2026-11-09'], + ['tr', 14.3, '2026-11-09'], + ['zh', 17.3, '2026-11-09'] + ] +} satisfies Record + +export function getSpeechModelErrorRates(modelId: keyof typeof MEASURED): { + errorRates: SpeechModelErrorRate[] +} { + return { + errorRates: MEASURED[modelId].map(([language, percent, measuredOn]) => ({ + language, + percent, + metric: CHARACTER_SCORED.has(language) ? 'cer' : 'wer', + test: language === 'en' ? 'librispeech' : 'fleurs', + clips: language === 'en' ? LIBRISPEECH_CLIPS : FLEURS_CLIPS, + measuredOn + })) + } +} diff --git a/src/renderer/src/components/dictation/speech-model-update-target.test.ts b/src/renderer/src/components/dictation/speech-model-update-target.test.ts index aa89f1f7983..c59e4e8bdd6 100644 --- a/src/renderer/src/components/dictation/speech-model-update-target.test.ts +++ b/src/renderer/src/components/dictation/speech-model-update-target.test.ts @@ -10,6 +10,7 @@ function model(id: string, extra: Partial = {}): SpeechMode type: 'transducer', provider: 'local', language: 'en', + languages: ['en'], sampleRate: 16000, streaming: false, ...extra diff --git a/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.fixture.tsx b/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.fixture.tsx index ef3350f173e..7e048404a80 100644 --- a/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.fixture.tsx +++ b/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.fixture.tsx @@ -19,6 +19,7 @@ function model( type, provider: 'local', language: 'multilingual', + languages: dictationLanguages ?? ['en'], sampleRate: 16000, streaming: false, dictationLanguages diff --git a/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.test.tsx b/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.test.tsx index ab7b6cfdfac..94d39441b7f 100644 --- a/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.test.tsx +++ b/src/renderer/src/components/settings/voice/VoiceDictationLanguageSetting.test.tsx @@ -39,6 +39,7 @@ it('keeps a selected language visible when the chosen model does not support it' type: 'whisper', provider: 'local', language: 'multilingual', + languages: ['en', 'es'], sampleRate: 16000, streaming: false, dictationLanguages: ['en', 'es'] @@ -67,6 +68,7 @@ it('keeps the language control disabled for models that ignore language settings type: 'transducer', provider: 'local', language: 'multilingual', + languages: ['en', 'es'], sampleRate: 16000, streaming: false } diff --git a/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.fixture.tsx b/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.fixture.tsx index 4a93339066f..6ebdd0c6c0b 100644 --- a/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.fixture.tsx +++ b/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.fixture.tsx @@ -11,6 +11,7 @@ function model(id: string, label: string, sizeBytes: number, description: string type: 'transducer', provider: 'local', language: 'en', + languages: ['en'], sizeBytes, sampleRate: 16000, streaming: false diff --git a/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.test.tsx b/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.test.tsx index d44309104df..6a5a0433801 100644 --- a/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.test.tsx +++ b/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.test.tsx @@ -64,6 +64,7 @@ const localModel: SpeechModelManifest = { description: 'Runs offline', provider: 'local', language: 'en', + languages: ['en'], type: 'transducer', streaming: true, sampleRate: 16000, diff --git a/src/shared/dictation-language.ts b/src/shared/dictation-language.ts index 369ec666f14..266da8233ef 100644 --- a/src/shared/dictation-language.ts +++ b/src/shared/dictation-language.ts @@ -10,6 +10,16 @@ export const DICTATION_LANGUAGES: readonly string[] = DICTATION_LANGUAGE_CODES.s // SenseVoice refuses to load with any other code. export const SENSE_VOICE_DICTATION_LANGUAGES: readonly string[] = ['en', 'ja', 'ko', 'yue', 'zh'] +// Whisper's language tokens; large-v3 models also have Cantonese. +export const WHISPER_LANGUAGES: readonly string[] = + 'af am ar as az ba be bg bn bo br bs ca cs cy da de el en es et eu fa fi fo fr gl gu haw ha he hi hr ht hu hy id is it ja jw ka kk km kn ko la lb ln lo lt lv mg mi mk ml mn mr ms mt my ne nl nn no oc pa pl ps pt ro ru sa sd si sk sl sn so sq sr su sv sw ta te tg th tk tl tr tt uk ur uz vi yi yo zh'.split( + ' ' + ) + +// The 25 European languages on Parakeet TDT v3's model card; it ignores a language hint. +export const PARAKEET_V3_LANGUAGES: readonly string[] = + 'bg cs da de el en es et fi fr hr hu it lt lv mt nl pl pt ro ru sk sl sv uk'.split(' ') + // Nemotron 3.5's locales (ko-KR, zh-CN, ...) by ISO-639-1 language; the recognizer sends the first matching locale. export const NEMOTRON_DICTATION_LANGUAGES: readonly string[] = 'ar bg cs da de en es et fi fr hi hr hu it ja ko nl pl pt ro ru sk sv tr uk vi zh'.split(' ') diff --git a/src/shared/speech-model-quality.test.ts b/src/shared/speech-model-quality.test.ts new file mode 100644 index 00000000000..280029e0576 --- /dev/null +++ b/src/shared/speech-model-quality.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import { speechModelQuality, speechModelRatingLanguage } from './speech-model-quality' +import type { SpeechModelErrorRate } from './speech-types' + +const rate = (language: string, percent: number): SpeechModelErrorRate => ({ + language, + percent, + metric: 'wer', + test: 'fleurs', + clips: 5, + measuredOn: '2026-11-09' +}) + +describe('speechModelQuality', () => { + it('rates by the approved word scale, inclusive at each ceiling', () => { + const tiers = [3, 3.1, 6, 10, 20, 20.1].map((percent) => { + const quality = speechModelQuality( + { languages: ['de'], errorRates: [rate('de', percent)] }, + 'de' + ) + return quality.kind === 'rated' ? quality.tier : quality.kind + }) + expect(tiers).toEqual([5, 4, 4, 3, 2, 1]) + }) + + it('tells an unsupported language from a supported one nobody measured', () => { + const model = { languages: ['en', 'fr'], errorRates: [rate('en', 4)] } + expect(speechModelQuality(model, 'ko').kind).toBe('unsupported') + expect(speechModelQuality(model, 'fr').kind).toBe('unmeasured') + }) +}) + +describe('speechModelRatingLanguage', () => { + it('uses the dictation language, else the system locale’s language', () => { + expect(speechModelRatingLanguage('de', 'ja-JP')).toBe('de') + expect(speechModelRatingLanguage('auto', 'zh-Hant-TW')).toBe('zh') + expect(speechModelRatingLanguage('auto', 'pt_BR')).toBe('pt') + expect(speechModelRatingLanguage('auto', undefined)).toBeUndefined() + }) +}) diff --git a/src/shared/speech-model-quality.ts b/src/shared/speech-model-quality.ts new file mode 100644 index 00000000000..1b085363fba --- /dev/null +++ b/src/shared/speech-model-quality.ts @@ -0,0 +1,46 @@ +import { DICTATION_LANGUAGE_AUTO } from './dictation-language' +import type { SpeechModelErrorRate, SpeechModelManifest } from './speech-types' + +/** 5 = Excellent, 4 = Very good, 3 = Good, 2 = Fair, 1 = Poor. */ +export type SpeechModelQualityTier = 1 | 2 | 3 | 4 | 5 + +export type SpeechModelQuality = + | { kind: 'unsupported' } + | { kind: 'unmeasured' } + | { kind: 'rated'; tier: SpeechModelQualityTier; errorRate: SpeechModelErrorRate } + +// Highest error % each tier allows, best tier first. +const TIER_CEILINGS: readonly (readonly [percent: number, tier: SpeechModelQualityTier])[] = [ + [3, 5], + [6, 4], + [10, 3], + [20, 2] +] + +function tierFor(percent: number): SpeechModelQualityTier { + return TIER_CEILINGS.find(([ceiling]) => percent <= ceiling)?.[1] ?? 1 +} + +/** The language ratings follow: the dictation language, or the system's when that is Auto-detect. */ +export function speechModelRatingLanguage( + dictationLanguage: string | undefined, + systemLocale: string | undefined +): string | undefined { + if (dictationLanguage && dictationLanguage !== DICTATION_LANGUAGE_AUTO) { + return dictationLanguage + } + return systemLocale?.split(/[-_]/)[0]?.toLowerCase() || undefined +} + +export function speechModelQuality( + model: Pick, + language: string +): SpeechModelQuality { + if (!model.languages.includes(language)) { + return { kind: 'unsupported' } + } + const errorRate = model.errorRates?.find((rate) => rate.language === language) + return errorRate + ? { kind: 'rated', tier: tierFor(errorRate.percent), errorRate } + : { kind: 'unmeasured' } +} diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index 7a453db0841..252e211284b 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -15,13 +15,31 @@ export type SpeechModelDownloadFile = { sha256: string } +/** One measured error rate: word error rate, or character error rate for Chinese, Japanese and Korean. */ +export type SpeechModelErrorRate = { + language: string + percent: number + metric: 'wer' | 'cer' + test: 'librispeech' | 'fleurs' + clips: number + /** YYYY-MM-DD of the benchmark run. */ + measuredOn: string +} + export type SpeechModelManifest = { id: string label: string description: string type: SpeechModelType provider: SpeechModelProvider + /** The text pipeline's hint: one language, a bilingual pair such as 'zh-en', or 'multilingual'. */ language: string + /** Every language the model transcribes (ISO-639-1, plus 'yue' for Cantonese). */ + languages: readonly string[] + /** Measured error rates; a supported language missing here has not been measured. */ + errorRates?: readonly SpeechModelErrorRate[] + /** Licence of the downloaded weights; absent for cloud models. */ + licence?: string sizeBytes?: number downloadFiles?: SpeechModelDownloadFile[] files?: string[]