mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
feat(dictation): remove filler words from text the phone recognizes itself
Phones that dictate with their own recognizer (Automatic or This phone) never send text through the computer, so its filler pass never saw it. The page's on-device speech adapter now runs the same shared text pipeline on the phone's live and final text, in the language of the locale the recognizer started in, and honours the computer's Remove Filler Words switch, which the host now reports in its dictation setup. An older host that doesn't report it leaves the switch on, its default.
This commit is contained in:
@@ -32,6 +32,8 @@ export const dictationSetupSchema = z.looseObject({
|
||||
enabled: salvagedOptional('enabled', z.boolean()),
|
||||
selectedModelId: salvagedOptional('selectedModelId', z.string()),
|
||||
dictationMode: salvagedOptional('dictationMode', z.string()),
|
||||
// Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default.
|
||||
removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()),
|
||||
models: salvagingArray(speechModelSchema)
|
||||
})
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ export type UseMobileDictationOptions = {
|
||||
enabled: boolean
|
||||
onTranscript: (text: string) => void
|
||||
onError?: (error: Error) => void
|
||||
// The computer's filler switch, for text the phone recognizes itself; unknown means on.
|
||||
removeFillerWords?: boolean
|
||||
}
|
||||
|
||||
export type UseMobileDictationResult = {
|
||||
|
||||
@@ -33,10 +33,10 @@ export type { UseMobileDictationResult } from './mobile-dictation-session-state'
|
||||
|
||||
export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult {
|
||||
useMobileTranslation()
|
||||
const { client, enabled, onTranscript, onError } = options
|
||||
const { client, enabled, onTranscript, onError, removeFillerWords } = options
|
||||
// Both capture seams hold the screen while the microphone is open; native uses the device directly and page uses shell audio verbs.
|
||||
const capture = useDictationCapture()
|
||||
const speech = useOnDeviceSpeech()
|
||||
const speech = useOnDeviceSpeech(removeFillerWords)
|
||||
const [status, setStatus] = useState<DictationStatus>('idle')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [transcriptSoFar, setTranscriptSoFar, keptTranscript] = useMobileDictationTranscriptSoFar()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { OnDeviceSpeech } from './on-device-speech-contract'
|
||||
|
||||
// The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer.
|
||||
export function useOnDeviceSpeech(): OnDeviceSpeech | null {
|
||||
export function useOnDeviceSpeech(_removeFillerWords?: boolean): OnDeviceSpeech | null {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
// Text the phone recognizes itself gets the computer's filler removal, in the language the recognizer started in.
|
||||
import { createElement } from 'react'
|
||||
import { act, create } from 'react-test-renderer'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { OnDeviceSpeech } from './on-device-speech-contract'
|
||||
|
||||
vi.mock('../mobile-web-shell/bridge/use-native-verbs', () => {
|
||||
const verbs = {
|
||||
canRecognizeSpeech: true,
|
||||
speechStatus: async () => ({ status: 'ready', locale: 'de-DE', preference: 'automatic' }),
|
||||
speechStart: async () => ({ started: true, locale: 'de-DE', reason: null }),
|
||||
speechRead: async () => ({
|
||||
revision: 1,
|
||||
committed: 'Äh, das ist um fünf Uhr.',
|
||||
tentative: 'ähm',
|
||||
failure: null
|
||||
}),
|
||||
speechFinish: async () => ({
|
||||
text: 'Äh, das ist um fünf Uhr, ähm, glaube ich.',
|
||||
failure: null
|
||||
}),
|
||||
speechCancel: async () => ({})
|
||||
}
|
||||
return { useNativeVerbs: () => verbs }
|
||||
})
|
||||
|
||||
vi.mock('./on-device-speech-locale', () => ({ onDeviceSpeechLocale: () => 'de-DE' }))
|
||||
|
||||
import { useOnDeviceSpeech } from './on-device-speech.web'
|
||||
|
||||
function mountSpeech(removeFillerWords: boolean): OnDeviceSpeech {
|
||||
let speech: OnDeviceSpeech | null = null
|
||||
function Probe(): null {
|
||||
speech = useOnDeviceSpeech(removeFillerWords)
|
||||
return null
|
||||
}
|
||||
act(() => {
|
||||
create(createElement(Probe))
|
||||
})
|
||||
if (speech === null) {
|
||||
throw new Error('no on-device speech')
|
||||
}
|
||||
return speech
|
||||
}
|
||||
|
||||
describe('on-device dictation text', () => {
|
||||
it('drops the fillers of the language the recognizer started in', async () => {
|
||||
const speech = mountSpeech(true)
|
||||
await speech.start()
|
||||
|
||||
expect(await speech.read()).toMatchObject({ committed: 'Das ist um fünf Uhr.', tentative: '' })
|
||||
expect(await speech.finish()).toEqual({
|
||||
text: 'Das ist um fünf Uhr, glaube ich.',
|
||||
failure: null
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps every word when the computer turned filler removal off', async () => {
|
||||
const speech = mountSpeech(false)
|
||||
await speech.start()
|
||||
|
||||
expect(await speech.finish()).toEqual({
|
||||
text: 'Äh, das ist um fünf Uhr, ähm, glaube ich.',
|
||||
failure: null
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,14 +1,43 @@
|
||||
import { useMemo } from 'react'
|
||||
import { useLayoutEffect, useMemo, useRef } from 'react'
|
||||
import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline'
|
||||
import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
|
||||
import type { OnDeviceSpeech } from './on-device-speech-contract'
|
||||
import { onDeviceSpeechLocale } from './on-device-speech-locale'
|
||||
|
||||
function createPageOnDeviceSpeech(verbs: NativeVerbs): OnDeviceSpeech {
|
||||
const keepText = (text: string): string => text
|
||||
|
||||
// The phone's recognizer transcribes only the locale it started in, so that locale's language picks the fillers, as the host does for its model.
|
||||
function phoneDictationTextCleaner(
|
||||
locale: string | null,
|
||||
removeFillerWords: boolean
|
||||
): (text: string) => string {
|
||||
const language = locale?.split('-')[0].toLowerCase()
|
||||
const clean = createDictationTextPipeline(language ? { language } : undefined, {
|
||||
removeFillerWords
|
||||
})
|
||||
return clean ?? keepText
|
||||
}
|
||||
|
||||
function createPageOnDeviceSpeech(
|
||||
verbs: NativeVerbs,
|
||||
removeFillerWords: () => boolean
|
||||
): OnDeviceSpeech {
|
||||
let clean = keepText
|
||||
return {
|
||||
status: () => verbs.speechStatus(onDeviceSpeechLocale()),
|
||||
start: () => verbs.speechStart(onDeviceSpeechLocale()),
|
||||
read: verbs.speechRead,
|
||||
finish: verbs.speechFinish,
|
||||
start: async () => {
|
||||
const started = await verbs.speechStart(onDeviceSpeechLocale())
|
||||
clean = phoneDictationTextCleaner(started.locale, removeFillerWords())
|
||||
return started
|
||||
},
|
||||
read: async () => {
|
||||
const read = await verbs.speechRead()
|
||||
return { ...read, committed: clean(read.committed), tentative: clean(read.tentative) }
|
||||
},
|
||||
finish: async () => {
|
||||
const finished = await verbs.speechFinish()
|
||||
return finished.text === null ? finished : { ...finished, text: clean(finished.text) }
|
||||
},
|
||||
// Cancel has no reply anyone waits on; a shell that lost the session has nothing to stop.
|
||||
cancel: () => {
|
||||
void verbs.speechCancel().catch(() => undefined)
|
||||
@@ -17,7 +46,18 @@ function createPageOnDeviceSpeech(verbs: NativeVerbs): OnDeviceSpeech {
|
||||
}
|
||||
|
||||
// Null on a shell without the verb, so dictation stays on the computer exactly as before.
|
||||
export function useOnDeviceSpeech(): OnDeviceSpeech | null {
|
||||
// removeFillerWords is the computer's switch, read at each start.
|
||||
export function useOnDeviceSpeech(removeFillerWords = true): OnDeviceSpeech | null {
|
||||
const verbs = useNativeVerbs()
|
||||
return useMemo(() => (verbs.canRecognizeSpeech ? createPageOnDeviceSpeech(verbs) : null), [verbs])
|
||||
const removeFillerWordsRef = useRef(removeFillerWords)
|
||||
useLayoutEffect(() => {
|
||||
removeFillerWordsRef.current = removeFillerWords
|
||||
}, [removeFillerWords])
|
||||
return useMemo(
|
||||
() =>
|
||||
verbs.canRecognizeSpeech
|
||||
? createPageOnDeviceSpeech(verbs, () => removeFillerWordsRef.current)
|
||||
: null,
|
||||
[verbs]
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useEffect, useCallback, useRef } from 'react'
|
||||
import { useEffect, useCallback, useRef, useState } from 'react'
|
||||
import { AppState } from 'react-native'
|
||||
import { useFocusEffect } from 'expo-router'
|
||||
import { useMobileDictation } from '../hooks/use-mobile-dictation'
|
||||
@@ -136,9 +136,12 @@ export function useMobileSessionNativeChatDictation(
|
||||
const dictationCanStart = showNativeChat
|
||||
? connState === 'connected' || phoneDictation.ready
|
||||
: canSend
|
||||
// The host's filler switch, kept from its last setup read; on until one says otherwise.
|
||||
const [removeFillerWords, setRemoveFillerWords] = useState(true)
|
||||
const dictation = useMobileDictation({
|
||||
client,
|
||||
enabled: (showNativeChat || canCompose) && dictationEnabled,
|
||||
removeFillerWords,
|
||||
onTranscript: (text) => {
|
||||
// Chat dictation is inserted locally; a terminal's words are typed into it live.
|
||||
if (destination.landsInChat(showNativeChatRef.current)) {
|
||||
@@ -246,6 +249,7 @@ export function useMobileSessionNativeChatDictation(
|
||||
const setup = await fetchDictationSetup(client)
|
||||
if (read === dictationSettingsReadRef.current) {
|
||||
setDictationMode(setup.dictationMode)
|
||||
setRemoveFillerWords(setup.removeFillerWords !== false)
|
||||
setDictationEnabled(!dictationMicHidden(setup))
|
||||
dictationNeedsSetupRef.current = dictationNeedsSetup(setup)
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ export class RuntimeMobileSpeechCatalog {
|
||||
enabled: voice.enabled === true,
|
||||
selectedModelId: voice.sttModel ?? '',
|
||||
dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle',
|
||||
removeFillerWords: voice.removeFillerWords !== false,
|
||||
models
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +171,8 @@ export type RuntimeSpeechSetupState = {
|
||||
enabled: boolean
|
||||
selectedModelId: string
|
||||
dictationMode: 'toggle' | 'hold'
|
||||
/** Phones that recognize speech themselves clean their own text by this switch. */
|
||||
removeFillerWords: boolean
|
||||
models: RuntimeSpeechModelSummary[]
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user