diff --git a/mobile/src/dictation/dictation-reply-schema.ts b/mobile/src/dictation/dictation-reply-schema.ts index 192e097c9dd..23a738977ac 100644 --- a/mobile/src/dictation/dictation-reply-schema.ts +++ b/mobile/src/dictation/dictation-reply-schema.ts @@ -1,6 +1,7 @@ import * as z from 'zod' import type { RuntimeSpeechModelSummary } from '../../../src/shared/runtime-worktree-contracts' import { hostUnionArms, salvagedOptional, salvagingArray } from '../../../src/shared/zod-salvage' +import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary' export const SPEECH_MODEL_PROVIDERS = hostUnionArms({ local: true, @@ -36,8 +37,11 @@ export const dictationSetupSchema = z.looseObject({ dictationHoldToRecord: salvagedOptional('dictationHoldToRecord', z.boolean()), // Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default. removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()), - // Absent from older hosts; the phone normalizes it before spelling with it. - dictationTerms: salvagedOptional('dictationTerms', z.unknown()), + // Absent from older hosts, which spell nothing; malformed entries are dropped. + dictationTerms: salvagedOptional( + 'dictationTerms', + z.unknown().transform(normalizeDictationTerms) + ), models: salvagingArray(speechModelSchema) }) diff --git a/mobile/src/hooks/mobile-dictation-session-state.ts b/mobile/src/hooks/mobile-dictation-session-state.ts index eb41797be8d..ec122ea47cb 100644 --- a/mobile/src/hooks/mobile-dictation-session-state.ts +++ b/mobile/src/hooks/mobile-dictation-session-state.ts @@ -2,6 +2,7 @@ import { Buffer } from 'buffer' import type { RpcClient } from '../transport/rpc/rpc-client-contract' +import type { DictationTerm } from '../../../src/shared/dictation-dictionary' import type { MobileDictationMeter } from './mobile-dictation-meter' export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error' @@ -20,8 +21,8 @@ export type UseMobileDictationOptions = { onError?: (error: Error, outcome: { kept: boolean }) => void // The computer's filler switch, for text the phone recognizes itself; unknown means on. removeFillerWords?: boolean - // The computer's dictionary for this worktree, unvalidated, for text the phone recognizes itself. - dictationTerms?: unknown + // The computer's dictionary for this worktree, for text the phone recognizes itself. + dictationTerms?: readonly DictationTerm[] // Lets the computer pick this worktree's repo terms for text it recognizes. worktreeId?: string // The host's cloud upload is full; the dictation stops and inserts what was kept. diff --git a/mobile/src/platform/on-device-speech.ts b/mobile/src/platform/on-device-speech.ts index f4b5c76ac50..b592de40404 100644 --- a/mobile/src/platform/on-device-speech.ts +++ b/mobile/src/platform/on-device-speech.ts @@ -1,9 +1,10 @@ +import type { DictationTerm } from '../../../src/shared/dictation-dictionary' import type { OnDeviceSpeech } from './on-device-speech-contract' // The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer. export function useOnDeviceSpeech( _removeFillerWords?: boolean, - _dictationTerms?: unknown + _dictationTerms?: readonly DictationTerm[] ): OnDeviceSpeech | null { return null } diff --git a/mobile/src/platform/on-device-speech.web.ts b/mobile/src/platform/on-device-speech.web.ts index 5d86d5f9429..fbaee133ffe 100644 --- a/mobile/src/platform/on-device-speech.web.ts +++ b/mobile/src/platform/on-device-speech.web.ts @@ -1,5 +1,5 @@ import { useLayoutEffect, useMemo, useRef } from 'react' -import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary' +import type { DictationTerm } from '../../../src/shared/dictation-dictionary' import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline' import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs' import type { OnDeviceSpeech } from './on-device-speech-contract' @@ -11,20 +11,23 @@ const keepText = (text: string): string => text function phoneDictationTextCleaner( locale: string | null, removeFillerWords: boolean, - dictationTerms: unknown + dictationTerms: readonly DictationTerm[] | undefined ): (text: string) => string { const language = locale?.split('-')[0].toLowerCase() const clean = createDictationTextPipeline( language ? { language } : undefined, { removeFillerWords }, - normalizeDictationTerms(dictationTerms) + dictationTerms ) return clean ?? keepText } function createPageOnDeviceSpeech( verbs: NativeVerbs, - textSettings: () => { removeFillerWords: boolean; dictationTerms: unknown } + textSettings: () => { + removeFillerWords: boolean + dictationTerms: readonly DictationTerm[] | undefined + } ): OnDeviceSpeech { let clean = keepText return { @@ -54,7 +57,7 @@ function createPageOnDeviceSpeech( // The computer's filler switch and dictionary for this worktree, read at each start. export function useOnDeviceSpeech( removeFillerWords = true, - dictationTerms?: unknown + dictationTerms?: readonly DictationTerm[] ): OnDeviceSpeech | null { const verbs = useNativeVerbs() const textSettingsRef = useRef({ removeFillerWords, dictationTerms }) diff --git a/src/shared/dictation-dictionary.ts b/src/shared/dictation-dictionary.ts index fac99c4f751..a66d3f5785b 100644 --- a/src/shared/dictation-dictionary.ts +++ b/src/shared/dictation-dictionary.ts @@ -49,25 +49,34 @@ export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown): ] } +type Spelling = { term: string; lead: string; trail: string } + +function spelling(term: string): Spelling { + const [, lead = '', trail = ''] = EDGE_JOINERS_RE.exec(term) ?? [] + return { term, lead, trail } +} + /** Rewrites words that key the same as a term (or its "heard as") to the term's spelling; null when there are none. */ export function createDictationDictionary( terms: readonly DictationTerm[] | undefined ): ((text: string) => string) | null { - const byKey = new Map() + const byKey = new Map() const unspacedAliases: [heardAs: string, term: string][] = [] + const add = (key: string, term: string): void => { + if (key && !byKey.has(key)) { + byKey.set(key, spelling(term)) + } + } // A "heard as" is what the user asked for, so it beats another term spelled the same. for (const { term, heardAs } of terms ?? []) { if (heardAs && UNSPACED_SCRIPT_RE.test(heardAs)) { unspacedAliases.push([heardAs, term]) - } else if (heardAs && wordKey(heardAs) && !byKey.has(wordKey(heardAs))) { - byKey.set(wordKey(heardAs), term) + } else if (heardAs) { + add(wordKey(heardAs), term) } } for (const { term } of terms ?? []) { - const key = wordKey(term) - if (key && !byKey.has(key)) { - byKey.set(key, term) - } + add(wordKey(term), term) } if (byKey.size === 0 && unspacedAliases.length === 0) { return null @@ -82,43 +91,48 @@ export function createDictationDictionary( } // At each word, the longest run of joined words whose key is a term's becomes that term. -function replaceWordRuns(text: string, byKey: Map, longestKey: number): string { - const words = [...text.matchAll(WORD_RE)] +function replaceWordRuns(text: string, byKey: Map, longestKey: number): string { + const words = [...text.matchAll(WORD_RE)].map((word) => ({ + from: word.index, + to: word.index + word[0].length, + key: wordKey(word[0]) + })) + // joinsNext[i]: words i and i + 1 may belong to one term. + const joinsNext = words.map( + (word, i) => i + 1 < words.length && JOINING_GAP_RE.test(text.slice(word.to, words[i + 1].from)) + ) let out = '' let cursor = 0 for (let start = 0; start < words.length;) { let key = '' - let match: { end: number; term: string } | null = null + let match: { end: number; spelling: Spelling } | null = null for (let end = start; end < words.length && end - start < MAX_WORDS_PER_TERM; end++) { - if (end > start) { - const gapStart = words[end - 1].index + words[end - 1][0].length - if (!JOINING_GAP_RE.test(text.slice(gapStart, words[end].index))) { - break - } + if (end > start && !joinsNext[end - 1]) { + break } - key += wordKey(words[end][0]) + key += words[end].key if (key.length > longestKey) { break } - const term = byKey.get(key) - if (term !== undefined) { - match = { end, term } + const found = byKey.get(key) + if (found) { + match = { end, spelling: found } } } if (!match) { start++ continue } - const [, lead, trail] = EDGE_JOINERS_RE.exec(match.term) ?? ['', '', ''] - let from = words[start].index - let to = words[match.end].index + words[match.end][0].length + const { term, lead, trail } = match.spelling + let from = words[start].from + let to = words[match.end].to if (lead && from - lead.length >= cursor && text.slice(from - lead.length, from) === lead) { from -= lead.length } if (trail && text.startsWith(trail, to)) { to += trail.length } - out += text.slice(cursor, from) + match.term + out += text.slice(cursor, from) + term cursor = to start = match.end + 1 }