feat(speech): record each model's languages, licence and measured error rates in the catalog

Error rates come from Orca's speech bench (transcribe-cpp 0.4.1, Metal, pinned GGUFs):
LibriSpeech for English, 5 FLEURS dev clips for de, es, fr, ja, ko, pt, ru, tr and zh.
speechModelQuality turns them into the Voice settings word scale.
This commit is contained in:
Kelvin Amoaba
2026-11-09 19:53:02 +00:00
parent 6a69b2140e
commit e6c16b4efc
11 changed files with 231 additions and 1 deletions
+32 -1
View File
@@ -2,10 +2,13 @@ import type { SpeechModelManifest } from '../../shared/speech-types'
import {
DICTATION_LANGUAGES,
NEMOTRON_DICTATION_LANGUAGES,
SENSE_VOICE_DICTATION_LANGUAGES
PARAKEET_V3_LANGUAGES,
SENSE_VOICE_DICTATION_LANGUAGES,
WHISPER_LANGUAGES
} from '../../shared/dictation-language'
import { getSpeechModelDownloadMetadata } from './model-download-catalog'
import { getGgufSpeechModelDownload } from './speech-model-gguf-downloads'
import { getSpeechModelErrorRates } from './speech-model-error-rates'
const legacy = (id: Parameters<typeof getSpeechModelDownloadMetadata>[0]) => ({
legacyDownloadFiles: getSpeechModelDownloadMetadata(id).downloadFiles
@@ -22,6 +25,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
language: 'multilingual',
...getGgufSpeechModelDownload('parakeet-tdt-0.6b-v3-int8'),
...legacy('parakeet-tdt-0.6b-v3-int8'),
languages: PARAKEET_V3_LANGUAGES,
...getSpeechModelErrorRates('parakeet-tdt-0.6b-v3-int8'),
licence: 'CC-BY-4.0',
sampleRate: 16000,
streaming: false,
recommended: true
@@ -36,6 +42,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
language: 'en',
...getGgufSpeechModelDownload('parakeet-tdt-0.6b-v2-int8'),
...legacy('parakeet-tdt-0.6b-v2-int8'),
languages: ['en'],
...getSpeechModelErrorRates('parakeet-tdt-0.6b-v2-int8'),
licence: 'CC-BY-4.0',
sampleRate: 16000,
streaming: false
},
@@ -47,6 +56,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'local',
language: 'zh-en',
...getSpeechModelDownloadMetadata('zipformer-bilingual-zh-en'),
languages: ['zh', 'en'],
sampleRate: 16000,
streaming: true,
// transcribe-cpp has no zipformer; SenseVoice covers Chinese and English.
@@ -61,6 +71,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'local',
language: 'zh-en',
...getSpeechModelDownloadMetadata('paraformer-bilingual-zh-en'),
languages: ['zh', 'en'],
sampleRate: 16000,
streaming: true,
// transcribe-cpp has no paraformer; SenseVoice covers Chinese and English.
@@ -74,6 +85,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'local',
language: 'en',
...getSpeechModelDownloadMetadata('zipformer-streaming-en-20m'),
languages: ['en'],
sampleRate: 16000,
streaming: true,
// The model itself loses the first words of every utterance.
@@ -87,6 +99,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'local',
language: 'zh',
...getSpeechModelDownloadMetadata('zipformer-streaming-zh-14m'),
languages: ['zh'],
sampleRate: 16000,
streaming: true,
// transcribe-cpp has no zipformer; SenseVoice beat Nemotron 3.5 on Korean and Chinese.
@@ -100,6 +113,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'local',
language: 'ko',
...getSpeechModelDownloadMetadata('zipformer-streaming-korean'),
languages: ['ko'],
sampleRate: 16000,
streaming: true,
// transcribe-cpp has no zipformer; SenseVoice beat Nemotron 3.5 on Korean and Chinese.
@@ -113,6 +127,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'local',
language: 'ja',
...getSpeechModelDownloadMetadata('parakeet-tdt-ctc-0.6b-ja-int8'),
languages: ['ja'],
sampleRate: 16000,
streaming: false,
// No GGUF conversion exists; SenseVoice covers Japanese.
@@ -128,6 +143,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
...getGgufSpeechModelDownload('whisper-tiny'),
...legacy('whisper-tiny'),
dictationLanguages: DICTATION_LANGUAGES,
languages: WHISPER_LANGUAGES,
...getSpeechModelErrorRates('whisper-tiny'),
licence: 'Apache-2.0',
sampleRate: 16000,
streaming: false
},
@@ -141,6 +159,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
language: 'multilingual',
...getGgufSpeechModelDownload('whisper-large-v3-turbo'),
dictationLanguages: DICTATION_LANGUAGES,
languages: [...WHISPER_LANGUAGES, 'yue'],
...getSpeechModelErrorRates('whisper-large-v3-turbo'),
licence: 'MIT',
sampleRate: 16000,
streaming: false
},
@@ -155,6 +176,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
...getGgufSpeechModelDownload('sense-voice-zh-en-ja-ko-yue'),
...legacy('sense-voice-zh-en-ja-ko-yue'),
dictationLanguages: SENSE_VOICE_DICTATION_LANGUAGES,
languages: SENSE_VOICE_DICTATION_LANGUAGES,
...getSpeechModelErrorRates('sense-voice-zh-en-ja-ko-yue'),
licence: 'FunASR Model License',
sampleRate: 16000,
streaming: false
},
@@ -168,6 +192,9 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
language: 'multilingual',
...getGgufSpeechModelDownload('nemotron-3.5-asr-streaming-0.6b'),
dictationLanguages: NEMOTRON_DICTATION_LANGUAGES,
languages: NEMOTRON_DICTATION_LANGUAGES,
...getSpeechModelErrorRates('nemotron-3.5-asr-streaming-0.6b'),
licence: 'OpenMDW-1.1',
sampleRate: 16000,
// Decoded per chunk for now; live text arrives with the streaming engine path.
streaming: false
@@ -180,6 +207,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'openai',
language: 'multilingual',
dictationLanguages: DICTATION_LANGUAGES,
languages: DICTATION_LANGUAGES,
sampleRate: 16000,
streaming: false
},
@@ -192,6 +220,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'openai-compatible',
language: 'multilingual',
dictationLanguages: DICTATION_LANGUAGES,
languages: DICTATION_LANGUAGES,
sampleRate: 16000,
streaming: false
},
@@ -204,6 +233,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'openai',
language: 'multilingual',
dictationLanguages: DICTATION_LANGUAGES,
languages: DICTATION_LANGUAGES,
sampleRate: 16000,
streaming: false,
// OpenAI shuts the gpt-4o transcribe models down on 2027-02-26.
@@ -217,6 +247,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
provider: 'openai',
language: 'multilingual',
dictationLanguages: DICTATION_LANGUAGES,
languages: DICTATION_LANGUAGES,
sampleRate: 16000,
streaming: false,
replacedBy: 'openai-gpt-transcribe'
@@ -0,0 +1,79 @@
import type { SpeechModelErrorRate } from '../../shared/speech-types'
type Measured = readonly [language: string, percent: number, measuredOn: string]
const LIBRISPEECH_CLIPS = 73
const FLEURS_CLIPS = 5
const CHARACTER_SCORED = new Set(['ja', 'ko', 'zh'])
// Orca's speech bench: transcribe-cpp 0.4.1 with Metal on an M4 Max, the catalog's pinned GGUF
// files, one run per clip. English is LibriSpeech (73 clips); other languages are FLEURS dev
// (5 clips each, so rough). Each clip's language was sent the way Orca sends it; Parakeet ignores it.
const MEASURED = {
'parakeet-tdt-0.6b-v3-int8': [
['en', 3.7, '2026-11-06'],
['de', 2.2, '2026-11-06'],
['es', 1.7, '2026-11-06'],
['fr', 8.3, '2026-11-09'],
['pt', 11.3, '2026-11-09'],
['ru', 3.4, '2026-11-06']
],
'parakeet-tdt-0.6b-v2-int8': [['en', 2.8, '2026-11-06']],
'whisper-tiny': [
['en', 13, '2026-11-09'],
['de', 10, '2026-11-09'],
['es', 13.6, '2026-11-09'],
['fr', 52.3, '2026-11-09'],
['ja', 69.3, '2026-11-09'],
['ko', 24.9, '2026-11-09'],
['pt', 31.1, '2026-11-09'],
['ru', 38.2, '2026-11-09'],
['tr', 49.5, '2026-11-09'],
['zh', 47.5, '2026-11-09']
],
'whisper-large-v3-turbo': [
['en', 4.5, '2026-11-09'],
['de', 1.1, '2026-11-09'],
['es', 1.7, '2026-11-09'],
['fr', 5.3, '2026-11-09'],
['ja', 7.4, '2026-11-09'],
['ko', 2.6, '2026-11-09'],
['pt', 8.5, '2026-11-09'],
['ru', 3.4, '2026-11-09'],
['tr', 7.7, '2026-11-09'],
['zh', 9.4, '2026-11-09']
],
'sense-voice-zh-en-ja-ko-yue': [
['en', 6.9, '2026-11-09'],
['ja', 10.7, '2026-11-09'],
['ko', 10, '2026-11-09'],
['zh', 12.9, '2026-11-09']
],
'nemotron-3.5-asr-streaming-0.6b': [
['en', 5.3, '2026-11-09'],
['de', 5.6, '2026-11-09'],
['es', 2.5, '2026-11-09'],
['fr', 9.8, '2026-11-09'],
['ja', 17.2, '2026-11-09'],
['ko', 11.8, '2026-11-09'],
['pt', 13.2, '2026-11-09'],
['ru', 14.6, '2026-11-09'],
['tr', 14.3, '2026-11-09'],
['zh', 17.3, '2026-11-09']
]
} satisfies Record<string, readonly Measured[]>
export function getSpeechModelErrorRates(modelId: keyof typeof MEASURED): {
errorRates: SpeechModelErrorRate[]
} {
return {
errorRates: MEASURED[modelId].map(([language, percent, measuredOn]) => ({
language,
percent,
metric: CHARACTER_SCORED.has(language) ? 'cer' : 'wer',
test: language === 'en' ? 'librispeech' : 'fleurs',
clips: language === 'en' ? LIBRISPEECH_CLIPS : FLEURS_CLIPS,
measuredOn
}))
}
}
@@ -10,6 +10,7 @@ function model(id: string, extra: Partial<SpeechModelManifest> = {}): SpeechMode
type: 'transducer',
provider: 'local',
language: 'en',
languages: ['en'],
sampleRate: 16000,
streaming: false,
...extra
@@ -19,6 +19,7 @@ function model(
type,
provider: 'local',
language: 'multilingual',
languages: dictationLanguages ?? ['en'],
sampleRate: 16000,
streaming: false,
dictationLanguages
@@ -39,6 +39,7 @@ it('keeps a selected language visible when the chosen model does not support it'
type: 'whisper',
provider: 'local',
language: 'multilingual',
languages: ['en', 'es'],
sampleRate: 16000,
streaming: false,
dictationLanguages: ['en', 'es']
@@ -67,6 +68,7 @@ it('keeps the language control disabled for models that ignore language settings
type: 'transducer',
provider: 'local',
language: 'multilingual',
languages: ['en', 'es'],
sampleRate: 16000,
streaming: false
}
@@ -11,6 +11,7 @@ function model(id: string, label: string, sizeBytes: number, description: string
type: 'transducer',
provider: 'local',
language: 'en',
languages: ['en'],
sizeBytes,
sampleRate: 16000,
streaming: false
@@ -64,6 +64,7 @@ const localModel: SpeechModelManifest = {
description: 'Runs offline',
provider: 'local',
language: 'en',
languages: ['en'],
type: 'transducer',
streaming: true,
sampleRate: 16000,
+10
View File
@@ -10,6 +10,16 @@ export const DICTATION_LANGUAGES: readonly string[] = DICTATION_LANGUAGE_CODES.s
// SenseVoice refuses to load with any other code.
export const SENSE_VOICE_DICTATION_LANGUAGES: readonly string[] = ['en', 'ja', 'ko', 'yue', 'zh']
// Whisper's language tokens; large-v3 models also have Cantonese.
export const WHISPER_LANGUAGES: readonly string[] =
'af am ar as az ba be bg bn bo br bs ca cs cy da de el en es et eu fa fi fo fr gl gu haw ha he hi hr ht hu hy id is it ja jw ka kk km kn ko la lb ln lo lt lv mg mi mk ml mn mr ms mt my ne nl nn no oc pa pl ps pt ro ru sa sd si sk sl sn so sq sr su sv sw ta te tg th tk tl tr tt uk ur uz vi yi yo zh'.split(
' '
)
// The 25 European languages on Parakeet TDT v3's model card; it ignores a language hint.
export const PARAKEET_V3_LANGUAGES: readonly string[] =
'bg cs da de el en es et fi fr hr hu it lt lv mt nl pl pt ro ru sk sl sv uk'.split(' ')
// Nemotron 3.5's locales (ko-KR, zh-CN, ...) by ISO-639-1 language; the recognizer sends the first matching locale.
export const NEMOTRON_DICTATION_LANGUAGES: readonly string[] =
'ar bg cs da de en es et fi fr hi hr hu it ja ko nl pl pt ro ru sk sv tr uk vi zh'.split(' ')
+40
View File
@@ -0,0 +1,40 @@
import { describe, expect, it } from 'vitest'
import { speechModelQuality, speechModelRatingLanguage } from './speech-model-quality'
import type { SpeechModelErrorRate } from './speech-types'
const rate = (language: string, percent: number): SpeechModelErrorRate => ({
language,
percent,
metric: 'wer',
test: 'fleurs',
clips: 5,
measuredOn: '2026-11-09'
})
describe('speechModelQuality', () => {
it('rates by the approved word scale, inclusive at each ceiling', () => {
const tiers = [3, 3.1, 6, 10, 20, 20.1].map((percent) => {
const quality = speechModelQuality(
{ languages: ['de'], errorRates: [rate('de', percent)] },
'de'
)
return quality.kind === 'rated' ? quality.tier : quality.kind
})
expect(tiers).toEqual([5, 4, 4, 3, 2, 1])
})
it('tells an unsupported language from a supported one nobody measured', () => {
const model = { languages: ['en', 'fr'], errorRates: [rate('en', 4)] }
expect(speechModelQuality(model, 'ko').kind).toBe('unsupported')
expect(speechModelQuality(model, 'fr').kind).toBe('unmeasured')
})
})
describe('speechModelRatingLanguage', () => {
it('uses the dictation language, else the system locale’s language', () => {
expect(speechModelRatingLanguage('de', 'ja-JP')).toBe('de')
expect(speechModelRatingLanguage('auto', 'zh-Hant-TW')).toBe('zh')
expect(speechModelRatingLanguage('auto', 'pt_BR')).toBe('pt')
expect(speechModelRatingLanguage('auto', undefined)).toBeUndefined()
})
})
+46
View File
@@ -0,0 +1,46 @@
import { DICTATION_LANGUAGE_AUTO } from './dictation-language'
import type { SpeechModelErrorRate, SpeechModelManifest } from './speech-types'
/** 5 = Excellent, 4 = Very good, 3 = Good, 2 = Fair, 1 = Poor. */
export type SpeechModelQualityTier = 1 | 2 | 3 | 4 | 5
export type SpeechModelQuality =
| { kind: 'unsupported' }
| { kind: 'unmeasured' }
| { kind: 'rated'; tier: SpeechModelQualityTier; errorRate: SpeechModelErrorRate }
// Highest error % each tier allows, best tier first.
const TIER_CEILINGS: readonly (readonly [percent: number, tier: SpeechModelQualityTier])[] = [
[3, 5],
[6, 4],
[10, 3],
[20, 2]
]
function tierFor(percent: number): SpeechModelQualityTier {
return TIER_CEILINGS.find(([ceiling]) => percent <= ceiling)?.[1] ?? 1
}
/** The language ratings follow: the dictation language, or the system's when that is Auto-detect. */
export function speechModelRatingLanguage(
dictationLanguage: string | undefined,
systemLocale: string | undefined
): string | undefined {
if (dictationLanguage && dictationLanguage !== DICTATION_LANGUAGE_AUTO) {
return dictationLanguage
}
return systemLocale?.split(/[-_]/)[0]?.toLowerCase() || undefined
}
export function speechModelQuality(
model: Pick<SpeechModelManifest, 'languages' | 'errorRates'>,
language: string
): SpeechModelQuality {
if (!model.languages.includes(language)) {
return { kind: 'unsupported' }
}
const errorRate = model.errorRates?.find((rate) => rate.language === language)
return errorRate
? { kind: 'rated', tier: tierFor(errorRate.percent), errorRate }
: { kind: 'unmeasured' }
}
+18
View File
@@ -15,13 +15,31 @@ export type SpeechModelDownloadFile = {
sha256: string
}
/** One measured error rate: word error rate, or character error rate for Chinese, Japanese and Korean. */
export type SpeechModelErrorRate = {
language: string
percent: number
metric: 'wer' | 'cer'
test: 'librispeech' | 'fleurs'
clips: number
/** YYYY-MM-DD of the benchmark run. */
measuredOn: string
}
export type SpeechModelManifest = {
id: string
label: string
description: string
type: SpeechModelType
provider: SpeechModelProvider
/** The text pipeline's hint: one language, a bilingual pair such as 'zh-en', or 'multilingual'. */
language: string
/** Every language the model transcribes (ISO-639-1, plus 'yue' for Cantonese). */
languages: readonly string[]
/** Measured error rates; a supported language missing here has not been measured. */
errorRates?: readonly SpeechModelErrorRate[]
/** Licence of the downloaded weights; absent for cloud models. */
licence?: string
sizeBytes?: number
downloadFiles?: SpeechModelDownloadFile[]
files?: string[]