refactor(speech): key transcript words once and type phone terms at the reply schema

This commit is contained in:
Kelvin Amoaba
2026-11-09 19:08:05 +00:00
parent cdaed66253
commit dfaba3b0c4
5 changed files with 56 additions and 33 deletions
@@ -1,6 +1,7 @@
import * as z from 'zod'
import type { RuntimeSpeechModelSummary } from '../../../src/shared/runtime-worktree-contracts'
import { hostUnionArms, salvagedOptional, salvagingArray } from '../../../src/shared/zod-salvage'
import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary'
export const SPEECH_MODEL_PROVIDERS = hostUnionArms<RuntimeSpeechModelSummary['provider']>({
local: true,
@@ -36,8 +37,11 @@ export const dictationSetupSchema = z.looseObject({
dictationHoldToRecord: salvagedOptional('dictationHoldToRecord', z.boolean()),
// Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default.
removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()),
// Absent from older hosts; the phone normalizes it before spelling with it.
dictationTerms: salvagedOptional('dictationTerms', z.unknown()),
// Absent from older hosts, which spell nothing; malformed entries are dropped.
dictationTerms: salvagedOptional(
'dictationTerms',
z.unknown().transform(normalizeDictationTerms)
),
models: salvagingArray(speechModelSchema)
})
@@ -2,6 +2,7 @@
import { Buffer } from 'buffer'
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
import type { MobileDictationMeter } from './mobile-dictation-meter'
export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error'
@@ -20,8 +21,8 @@ export type UseMobileDictationOptions = {
onError?: (error: Error, outcome: { kept: boolean }) => void
// The computer's filler switch, for text the phone recognizes itself; unknown means on.
removeFillerWords?: boolean
// The computer's dictionary for this worktree, unvalidated, for text the phone recognizes itself.
dictationTerms?: unknown
// The computer's dictionary for this worktree, for text the phone recognizes itself.
dictationTerms?: readonly DictationTerm[]
// Lets the computer pick this worktree's repo terms for text it recognizes.
worktreeId?: string
// The host's cloud upload is full; the dictation stops and inserts what was kept.
+2 -1
View File
@@ -1,9 +1,10 @@
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
import type { OnDeviceSpeech } from './on-device-speech-contract'
// The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer.
export function useOnDeviceSpeech(
_removeFillerWords?: boolean,
_dictationTerms?: unknown
_dictationTerms?: readonly DictationTerm[]
): OnDeviceSpeech | null {
return null
}
+8 -5
View File
@@ -1,5 +1,5 @@
import { useLayoutEffect, useMemo, useRef } from 'react'
import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary'
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline'
import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
import type { OnDeviceSpeech } from './on-device-speech-contract'
@@ -11,20 +11,23 @@ const keepText = (text: string): string => text
function phoneDictationTextCleaner(
locale: string | null,
removeFillerWords: boolean,
dictationTerms: unknown
dictationTerms: readonly DictationTerm[] | undefined
): (text: string) => string {
const language = locale?.split('-')[0].toLowerCase()
const clean = createDictationTextPipeline(
language ? { language } : undefined,
{ removeFillerWords },
normalizeDictationTerms(dictationTerms)
dictationTerms
)
return clean ?? keepText
}
function createPageOnDeviceSpeech(
verbs: NativeVerbs,
textSettings: () => { removeFillerWords: boolean; dictationTerms: unknown }
textSettings: () => {
removeFillerWords: boolean
dictationTerms: readonly DictationTerm[] | undefined
}
): OnDeviceSpeech {
let clean = keepText
return {
@@ -54,7 +57,7 @@ function createPageOnDeviceSpeech(
// The computer's filler switch and dictionary for this worktree, read at each start.
export function useOnDeviceSpeech(
removeFillerWords = true,
dictationTerms?: unknown
dictationTerms?: readonly DictationTerm[]
): OnDeviceSpeech | null {
const verbs = useNativeVerbs()
const textSettingsRef = useRef({ removeFillerWords, dictationTerms })
+37 -23
View File
@@ -49,25 +49,34 @@ export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown):
]
}
type Spelling = { term: string; lead: string; trail: string }
function spelling(term: string): Spelling {
const [, lead = '', trail = ''] = EDGE_JOINERS_RE.exec(term) ?? []
return { term, lead, trail }
}
/** Rewrites words that key the same as a term (or its "heard as") to the term's spelling; null when there are none. */
export function createDictationDictionary(
terms: readonly DictationTerm[] | undefined
): ((text: string) => string) | null {
const byKey = new Map<string, string>()
const byKey = new Map<string, Spelling>()
const unspacedAliases: [heardAs: string, term: string][] = []
const add = (key: string, term: string): void => {
if (key && !byKey.has(key)) {
byKey.set(key, spelling(term))
}
}
// A "heard as" is what the user asked for, so it beats another term spelled the same.
for (const { term, heardAs } of terms ?? []) {
if (heardAs && UNSPACED_SCRIPT_RE.test(heardAs)) {
unspacedAliases.push([heardAs, term])
} else if (heardAs && wordKey(heardAs) && !byKey.has(wordKey(heardAs))) {
byKey.set(wordKey(heardAs), term)
} else if (heardAs) {
add(wordKey(heardAs), term)
}
}
for (const { term } of terms ?? []) {
const key = wordKey(term)
if (key && !byKey.has(key)) {
byKey.set(key, term)
}
add(wordKey(term), term)
}
if (byKey.size === 0 && unspacedAliases.length === 0) {
return null
@@ -82,43 +91,48 @@ export function createDictationDictionary(
}
// At each word, the longest run of joined words whose key is a term's becomes that term.
function replaceWordRuns(text: string, byKey: Map<string, string>, longestKey: number): string {
const words = [...text.matchAll(WORD_RE)]
function replaceWordRuns(text: string, byKey: Map<string, Spelling>, longestKey: number): string {
const words = [...text.matchAll(WORD_RE)].map((word) => ({
from: word.index,
to: word.index + word[0].length,
key: wordKey(word[0])
}))
// joinsNext[i]: words i and i + 1 may belong to one term.
const joinsNext = words.map(
(word, i) => i + 1 < words.length && JOINING_GAP_RE.test(text.slice(word.to, words[i + 1].from))
)
let out = ''
let cursor = 0
for (let start = 0; start < words.length;) {
let key = ''
let match: { end: number; term: string } | null = null
let match: { end: number; spelling: Spelling } | null = null
for (let end = start; end < words.length && end - start < MAX_WORDS_PER_TERM; end++) {
if (end > start) {
const gapStart = words[end - 1].index + words[end - 1][0].length
if (!JOINING_GAP_RE.test(text.slice(gapStart, words[end].index))) {
break
}
if (end > start && !joinsNext[end - 1]) {
break
}
key += wordKey(words[end][0])
key += words[end].key
if (key.length > longestKey) {
break
}
const term = byKey.get(key)
if (term !== undefined) {
match = { end, term }
const found = byKey.get(key)
if (found) {
match = { end, spelling: found }
}
}
if (!match) {
start++
continue
}
const [, lead, trail] = EDGE_JOINERS_RE.exec(match.term) ?? ['', '', '']
let from = words[start].index
let to = words[match.end].index + words[match.end][0].length
const { term, lead, trail } = match.spelling
let from = words[start].from
let to = words[match.end].to
if (lead && from - lead.length >= cursor && text.slice(from - lead.length, from) === lead) {
from -= lead.length
}
if (trail && text.startsWith(trail, to)) {
to += trail.length
}
out += text.slice(cursor, from) + match.term
out += text.slice(cursor, from) + term
cursor = to
start = match.end + 1
}