mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
refactor(speech): key transcript words once and type phone terms at the reply schema
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import * as z from 'zod'
|
||||
import type { RuntimeSpeechModelSummary } from '../../../src/shared/runtime-worktree-contracts'
|
||||
import { hostUnionArms, salvagedOptional, salvagingArray } from '../../../src/shared/zod-salvage'
|
||||
import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary'
|
||||
|
||||
export const SPEECH_MODEL_PROVIDERS = hostUnionArms<RuntimeSpeechModelSummary['provider']>({
|
||||
local: true,
|
||||
@@ -36,8 +37,11 @@ export const dictationSetupSchema = z.looseObject({
|
||||
dictationHoldToRecord: salvagedOptional('dictationHoldToRecord', z.boolean()),
|
||||
// Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default.
|
||||
removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()),
|
||||
// Absent from older hosts; the phone normalizes it before spelling with it.
|
||||
dictationTerms: salvagedOptional('dictationTerms', z.unknown()),
|
||||
// Absent from older hosts, which spell nothing; malformed entries are dropped.
|
||||
dictationTerms: salvagedOptional(
|
||||
'dictationTerms',
|
||||
z.unknown().transform(normalizeDictationTerms)
|
||||
),
|
||||
models: salvagingArray(speechModelSchema)
|
||||
})
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { Buffer } from 'buffer'
|
||||
|
||||
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
|
||||
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
|
||||
import type { MobileDictationMeter } from './mobile-dictation-meter'
|
||||
|
||||
export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error'
|
||||
@@ -20,8 +21,8 @@ export type UseMobileDictationOptions = {
|
||||
onError?: (error: Error, outcome: { kept: boolean }) => void
|
||||
// The computer's filler switch, for text the phone recognizes itself; unknown means on.
|
||||
removeFillerWords?: boolean
|
||||
// The computer's dictionary for this worktree, unvalidated, for text the phone recognizes itself.
|
||||
dictationTerms?: unknown
|
||||
// The computer's dictionary for this worktree, for text the phone recognizes itself.
|
||||
dictationTerms?: readonly DictationTerm[]
|
||||
// Lets the computer pick this worktree's repo terms for text it recognizes.
|
||||
worktreeId?: string
|
||||
// The host's cloud upload is full; the dictation stops and inserts what was kept.
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
|
||||
import type { OnDeviceSpeech } from './on-device-speech-contract'
|
||||
|
||||
// The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer.
|
||||
export function useOnDeviceSpeech(
|
||||
_removeFillerWords?: boolean,
|
||||
_dictationTerms?: unknown
|
||||
_dictationTerms?: readonly DictationTerm[]
|
||||
): OnDeviceSpeech | null {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useLayoutEffect, useMemo, useRef } from 'react'
|
||||
import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary'
|
||||
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
|
||||
import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline'
|
||||
import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
|
||||
import type { OnDeviceSpeech } from './on-device-speech-contract'
|
||||
@@ -11,20 +11,23 @@ const keepText = (text: string): string => text
|
||||
function phoneDictationTextCleaner(
|
||||
locale: string | null,
|
||||
removeFillerWords: boolean,
|
||||
dictationTerms: unknown
|
||||
dictationTerms: readonly DictationTerm[] | undefined
|
||||
): (text: string) => string {
|
||||
const language = locale?.split('-')[0].toLowerCase()
|
||||
const clean = createDictationTextPipeline(
|
||||
language ? { language } : undefined,
|
||||
{ removeFillerWords },
|
||||
normalizeDictationTerms(dictationTerms)
|
||||
dictationTerms
|
||||
)
|
||||
return clean ?? keepText
|
||||
}
|
||||
|
||||
function createPageOnDeviceSpeech(
|
||||
verbs: NativeVerbs,
|
||||
textSettings: () => { removeFillerWords: boolean; dictationTerms: unknown }
|
||||
textSettings: () => {
|
||||
removeFillerWords: boolean
|
||||
dictationTerms: readonly DictationTerm[] | undefined
|
||||
}
|
||||
): OnDeviceSpeech {
|
||||
let clean = keepText
|
||||
return {
|
||||
@@ -54,7 +57,7 @@ function createPageOnDeviceSpeech(
|
||||
// The computer's filler switch and dictionary for this worktree, read at each start.
|
||||
export function useOnDeviceSpeech(
|
||||
removeFillerWords = true,
|
||||
dictationTerms?: unknown
|
||||
dictationTerms?: readonly DictationTerm[]
|
||||
): OnDeviceSpeech | null {
|
||||
const verbs = useNativeVerbs()
|
||||
const textSettingsRef = useRef({ removeFillerWords, dictationTerms })
|
||||
|
||||
@@ -49,25 +49,34 @@ export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown):
|
||||
]
|
||||
}
|
||||
|
||||
type Spelling = { term: string; lead: string; trail: string }
|
||||
|
||||
function spelling(term: string): Spelling {
|
||||
const [, lead = '', trail = ''] = EDGE_JOINERS_RE.exec(term) ?? []
|
||||
return { term, lead, trail }
|
||||
}
|
||||
|
||||
/** Rewrites words that key the same as a term (or its "heard as") to the term's spelling; null when there are none. */
|
||||
export function createDictationDictionary(
|
||||
terms: readonly DictationTerm[] | undefined
|
||||
): ((text: string) => string) | null {
|
||||
const byKey = new Map<string, string>()
|
||||
const byKey = new Map<string, Spelling>()
|
||||
const unspacedAliases: [heardAs: string, term: string][] = []
|
||||
const add = (key: string, term: string): void => {
|
||||
if (key && !byKey.has(key)) {
|
||||
byKey.set(key, spelling(term))
|
||||
}
|
||||
}
|
||||
// A "heard as" is what the user asked for, so it beats another term spelled the same.
|
||||
for (const { term, heardAs } of terms ?? []) {
|
||||
if (heardAs && UNSPACED_SCRIPT_RE.test(heardAs)) {
|
||||
unspacedAliases.push([heardAs, term])
|
||||
} else if (heardAs && wordKey(heardAs) && !byKey.has(wordKey(heardAs))) {
|
||||
byKey.set(wordKey(heardAs), term)
|
||||
} else if (heardAs) {
|
||||
add(wordKey(heardAs), term)
|
||||
}
|
||||
}
|
||||
for (const { term } of terms ?? []) {
|
||||
const key = wordKey(term)
|
||||
if (key && !byKey.has(key)) {
|
||||
byKey.set(key, term)
|
||||
}
|
||||
add(wordKey(term), term)
|
||||
}
|
||||
if (byKey.size === 0 && unspacedAliases.length === 0) {
|
||||
return null
|
||||
@@ -82,43 +91,48 @@ export function createDictationDictionary(
|
||||
}
|
||||
|
||||
// At each word, the longest run of joined words whose key is a term's becomes that term.
|
||||
function replaceWordRuns(text: string, byKey: Map<string, string>, longestKey: number): string {
|
||||
const words = [...text.matchAll(WORD_RE)]
|
||||
function replaceWordRuns(text: string, byKey: Map<string, Spelling>, longestKey: number): string {
|
||||
const words = [...text.matchAll(WORD_RE)].map((word) => ({
|
||||
from: word.index,
|
||||
to: word.index + word[0].length,
|
||||
key: wordKey(word[0])
|
||||
}))
|
||||
// joinsNext[i]: words i and i + 1 may belong to one term.
|
||||
const joinsNext = words.map(
|
||||
(word, i) => i + 1 < words.length && JOINING_GAP_RE.test(text.slice(word.to, words[i + 1].from))
|
||||
)
|
||||
let out = ''
|
||||
let cursor = 0
|
||||
for (let start = 0; start < words.length;) {
|
||||
let key = ''
|
||||
let match: { end: number; term: string } | null = null
|
||||
let match: { end: number; spelling: Spelling } | null = null
|
||||
for (let end = start; end < words.length && end - start < MAX_WORDS_PER_TERM; end++) {
|
||||
if (end > start) {
|
||||
const gapStart = words[end - 1].index + words[end - 1][0].length
|
||||
if (!JOINING_GAP_RE.test(text.slice(gapStart, words[end].index))) {
|
||||
break
|
||||
}
|
||||
if (end > start && !joinsNext[end - 1]) {
|
||||
break
|
||||
}
|
||||
key += wordKey(words[end][0])
|
||||
key += words[end].key
|
||||
if (key.length > longestKey) {
|
||||
break
|
||||
}
|
||||
const term = byKey.get(key)
|
||||
if (term !== undefined) {
|
||||
match = { end, term }
|
||||
const found = byKey.get(key)
|
||||
if (found) {
|
||||
match = { end, spelling: found }
|
||||
}
|
||||
}
|
||||
if (!match) {
|
||||
start++
|
||||
continue
|
||||
}
|
||||
const [, lead, trail] = EDGE_JOINERS_RE.exec(match.term) ?? ['', '', '']
|
||||
let from = words[start].index
|
||||
let to = words[match.end].index + words[match.end][0].length
|
||||
const { term, lead, trail } = match.spelling
|
||||
let from = words[start].from
|
||||
let to = words[match.end].to
|
||||
if (lead && from - lead.length >= cursor && text.slice(from - lead.length, from) === lead) {
|
||||
from -= lead.length
|
||||
}
|
||||
if (trail && text.startsWith(trail, to)) {
|
||||
to += trail.length
|
||||
}
|
||||
out += text.slice(cursor, from) + match.term
|
||||
out += text.slice(cursor, from) + term
|
||||
cursor = to
|
||||
start = match.end + 1
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user