diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 5506ad38690..2451c5acff7 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -204,6 +204,9 @@ export function registerSpeechHandlers(store: Store): void { case 'final': window.webContents.send('speech:final', { text: msg.text ?? '', sessionId }) break + case 'speech-pending': + window.webContents.send('speech:pending', { pending: msg.pending, sessionId }) + break case 'stopped': cleanupSessionListener() window.webContents.send('speech:stopped', { sessionId }) diff --git a/src/main/speech/stt-recognizer.ts b/src/main/speech/stt-recognizer.ts index 95a16492673..2b91dd8d43f 100644 --- a/src/main/speech/stt-recognizer.ts +++ b/src/main/speech/stt-recognizer.ts @@ -12,7 +12,9 @@ export type SttRecognizerInit = { export type SttRecognizerEvent = /** One decoded offline chunk; `samples` counts the stream's input-rate samples decoded so far. */ - { type: 'chunk'; streamId: string; text: string; samples: number } + | { type: 'chunk'; streamId: string; text: string; samples: number } + /** Pause-cut streams only: speech was heard that no chunk has committed yet. */ + | { type: 'speech-pending'; streamId: string; pending: boolean } export type SttRecognizerPost = (event: SttRecognizerEvent) => void diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index e1023185d16..048c4872135 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -23,6 +23,8 @@ export type SttEvent = | { type: 'ready' } | { type: 'partial'; text?: string } | { type: 'final'; text?: string } + /** Speech was heard that no final has committed yet; only from pause-cut on-device models. */ + | { type: 'speech-pending'; pending: boolean } | { type: 'stopped' } | { type: 'error'; error?: string } diff --git a/src/main/speech/stt-transcribe-cpp-recognizer.ts b/src/main/speech/stt-transcribe-cpp-recognizer.ts index 578be1cc57e..e8c62efda14 100644 --- a/src/main/speech/stt-transcribe-cpp-recognizer.ts +++ b/src/main/speech/stt-transcribe-cpp-recognizer.ts @@ -17,6 +17,7 @@ function performanceCoreThreads(): number { // Without the detector, audio is still cut at the decode bound, just not at pauses. type AudioSegmenter = { readonly heldSamples: number + readonly phraseOpen?: boolean push(samples: Float32Array): Float32Array[] | Promise flush(): Float32Array[] | Promise } @@ -44,6 +45,7 @@ type StreamCoverage = { resampledSamples: number reportedSamples: number slowPhraseDecodes: number + speechPending: boolean } // Whisper pads every decode to a 30 s window, so each phrase costs a full decode. Where that takes longer than the @@ -121,7 +123,8 @@ export async function createTranscribeCppRecognizer( inputSamples: 0, resampledSamples: 0, reportedSamples: 0, - slowPhraseDecodes: 0 + slowPhraseDecodes: 0, + speechPending: false } if (vad && vadModel) { stream.segmenter = new VadPauseSegmenter( @@ -201,6 +204,14 @@ export async function createTranscribeCppRecognizer( } } + // Only pause-cut streams report it: without the detector, nothing tells speech from silence. + const notePendingSpeech = (streamId: string, stream: StreamCoverage, pending: boolean) => { + if (stream.vad && stream.speechPending !== pending) { + stream.speechPending = pending + post({ type: 'speech-pending', streamId, pending }) + } + } + const closeStream = (streamId: string): StreamCoverage | undefined => { const stream = streams.get(streamId) streams.delete(streamId) @@ -219,7 +230,11 @@ export async function createTranscribeCppRecognizer( ) stream.inputSamples += samples.length stream.resampledSamples += resampled.length - await decode(streamId, stream, await stream.segmenter.push(resampled)) + const phrases = await stream.segmenter.push(resampled) + // A cut phrase stays pending until its text is posted. + notePendingSpeech(streamId, stream, phrases.length > 0 || !!stream.segmenter.phraseOpen) + await decode(streamId, stream, phrases) + notePendingSpeech(streamId, stream, !!stream.segmenter.phraseOpen) }, async finish(streamId) { const stream = closeStream(streamId) @@ -228,6 +243,7 @@ export async function createTranscribeCppRecognizer( } try { await decode(streamId, stream, await stream.segmenter.flush()) + notePendingSpeech(streamId, stream, false) } finally { stream.vad?.dispose() } diff --git a/src/main/speech/stt-vad-pause-segmenter.test.ts b/src/main/speech/stt-vad-pause-segmenter.test.ts index 0c9015361b2..4ad6cd1efdb 100644 --- a/src/main/speech/stt-vad-pause-segmenter.test.ts +++ b/src/main/speech/stt-vad-pause-segmenter.test.ts @@ -69,13 +69,17 @@ describe('VadPauseSegmenter', () => { ]) }) - it('holds a click shorter than 0.25 s of speech until real speech joins it', async () => { + it('holds a click shorter than 0.25 s of speech until real speech joins it, without opening a phrase', async () => { const segmenter = new VadPauseSegmenter(fakeVad(), RATE, FRAME) expect(await segmenter.push(audio([0.9, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2]))).toEqual([]) - const ready = await segmenter.push(audio([0.8, 0.8, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2])) + expect(segmenter.phraseOpen).toBe(false) + expect(await segmenter.push(audio([0.8, 0.8]))).toEqual([]) + expect(segmenter.phraseOpen).toBe(true) + const ready = await segmenter.push(audio([0.2, 0.2, 0.2, 0.2, 0.2, 0.2])) expect(framesOf(ready)).toEqual([ [0.9, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.8, 0.8, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2] ]) + expect(segmenter.phraseOpen).toBe(false) }) it('returns nothing for a dictation that never had speech', async () => { diff --git a/src/main/speech/stt-vad-pause-segmenter.ts b/src/main/speech/stt-vad-pause-segmenter.ts index 0212724b223..8e7fc3b3d33 100644 --- a/src/main/speech/stt-vad-pause-segmenter.ts +++ b/src/main/speech/stt-vad-pause-segmenter.ts @@ -48,6 +48,11 @@ export class VadPauseSegmenter { return this.unscored.length + this.prerollLength + this.phrase.heldSamples } + /** At least a phrase's worth of speech not cut yet (a lone click isn't); after a detector error, until stop. */ + get phraseOpen(): boolean { + return this.inSpeech && (this.vadFailed || this.speechFrames >= this.minSpeechFrames) + } + /** Return the phrases that ended (or reached the decode bound) within these samples. */ async push(samples: Float32Array): Promise { if (this.vadFailed) { diff --git a/src/main/speech/stt-worker-cpu-fallback.ts b/src/main/speech/stt-worker-cpu-fallback.ts index 3b32588abd7..63bdeba05a4 100644 --- a/src/main/speech/stt-worker-cpu-fallback.ts +++ b/src/main/speech/stt-worker-cpu-fallback.ts @@ -15,6 +15,7 @@ type Feed = Extract type Open = Extract type ChildMessage = | { type: 'chunk'; streamId: string; text: string; samples: number } + | { type: 'speech-pending'; streamId: string; pending: boolean } | { type: 'ready' | 'stopped' | 'error' | 'partial' | 'final' streamId?: string diff --git a/src/preload/api/speech-api.ts b/src/preload/api/speech-api.ts index 9197ed1883b..c7415932ba7 100644 --- a/src/preload/api/speech-api.ts +++ b/src/preload/api/speech-api.ts @@ -7,6 +7,7 @@ import type { SpeechLifecycleEvent, SpeechModelManifest, SpeechModelState, + SpeechPendingEvent, SpeechTranscriptEvent } from '../../shared/speech-types' @@ -39,6 +40,7 @@ export type SpeechApi = { cancelDictation: (sessionId: string) => Promise onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void + onSpeechPending: (callback: (data: SpeechPendingEvent) => void) => () => void onDownloadProgress: ( callback: (data: { modelId: string; progress: number }) => void ) => () => void diff --git a/src/preload/api/speech-bridge.ts b/src/preload/api/speech-bridge.ts index 4ec6ad72ac2..66acdfc0bac 100644 --- a/src/preload/api/speech-bridge.ts +++ b/src/preload/api/speech-bridge.ts @@ -9,6 +9,7 @@ import type { SpeechLifecycleEvent, SpeechModelManifest, SpeechModelState, + SpeechPendingEvent, SpeechTranscriptEvent } from '../../shared/speech-types' import type { PreloadApi } from '../api-types' @@ -111,6 +112,12 @@ export const speechApi = { ipcRenderer.on('speech:final', listener) return () => ipcRenderer.removeListener('speech:final', listener) }, + onSpeechPending: (callback: (data: SpeechPendingEvent) => void): (() => void) => { + const listener = (_event: Electron.IpcRendererEvent, data: SpeechPendingEvent): void => + callback(data) + ipcRenderer.on('speech:pending', listener) + return () => ipcRenderer.removeListener('speech:pending', listener) + }, onDownloadProgress: ( callback: (data: { modelId: string; progress: number }) => void ): (() => void) => { diff --git a/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx b/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx index 3f919e65478..ca8b05f0971 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx @@ -53,7 +53,8 @@ export const liveText = defineFixture({ dictationState: 'listening', liveText: { committed: 'It should go back to the page you came from.', - tentative: 'Check where returnTo gets dropped' + tentative: 'Check where returnTo gets dropped', + speechPending: false } }, setup: () => publishDictationMeter({ level: 0.6, isSpeaking: true, isClipping: false }), @@ -66,13 +67,38 @@ export const committedText = defineFixture({ dictationState: 'listening', liveText: { committed: 'It should go back to the page you came from. Check where returnTo gets dropped.', - tentative: '' + tentative: '', + speechPending: false } }, setup: () => publishDictationMeter({ level: 0.2, isSpeaking: false, isClipping: false }), render: () => }) +export const pendingSpeech = defineFixture({ + width: 520, + store: { + dictationState: 'listening', + liveText: { + committed: 'It should go back to the page you came from.', + tentative: '', + speechPending: true + } + }, + setup: () => publishDictationMeter({ level: 0.6, isSpeaking: true, isClipping: false }), + render: () => +}) + +export const pendingFirstWords = defineFixture({ + width: 520, + store: { + dictationState: 'listening', + liveText: { committed: '', tentative: '', speechPending: true } + }, + setup: () => publishDictationMeter({ level: 0.6, isSpeaking: true, isClipping: false }), + render: () => +}) + export const tooLoud = defineFixture({ width: 520, store: { dictationState: 'listening' }, diff --git a/src/renderer/src/components/dictation/DictationIndicator.tsx b/src/renderer/src/components/dictation/DictationIndicator.tsx index f8559b598ec..83e1eaada07 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.tsx @@ -126,6 +126,21 @@ function DictationPillMeter({ recording, status }: { recording: boolean; status: ) } +// Words heard but not committed yet: they land at the next pause. +function DictationPendingDots() { + return ( + + {[0, 150, 300].map((delayMs) => ( + + ))} + + ) +} + // Presses on the pill keep focus where the words are going. const keepFocus = (event: React.MouseEvent): void => event.preventDefault() const keepFocusProps = { [KEEPS_FOCUS_ATTRIBUTE]: '', onMouseDown: keepFocus } @@ -146,7 +161,7 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica const showSpinner = useDelayedFlag(finishing, FINISH_SPINNER_DELAY_MS) const nearCloudLimit = cloudModel && seconds >= CLOUD_LIMIT_WARNING_SECONDS const failedKind = dictationState === 'error' ? failure : null - const committed = liveText.committed + const { committed, speechPending } = liveText const tentative = liveText.tentative.trim() const queuedStatus = queued === 'chat' @@ -200,7 +215,7 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica )} {queuedStatus} - ) : committed || tentative ? ( + ) : committed || tentative || speechPending ? ( // Right-aligned so the newest words stay in view; older ones fade off the left edge. {tentative} + {speechPending ? : null} ) : ( diff --git a/src/renderer/src/components/dictation/use-dictation-speech-events.ts b/src/renderer/src/components/dictation/use-dictation-speech-events.ts index e93d69bbe4d..c551e1e3d1a 100644 --- a/src/renderer/src/components/dictation/use-dictation-speech-events.ts +++ b/src/renderer/src/components/dictation/use-dictation-speech-events.ts @@ -13,17 +13,28 @@ type DictationSpeechEventHandlers = { } export function useDictationSpeechEvents(handlers: DictationSpeechEventHandlers): void { - const transcriptRef = useRef<{ sessionId: string; transcript: DictationTranscript } | null>(null) + const liveRef = useRef<{ + sessionId: string + transcript: DictationTranscript + speechPending: boolean + } | null>(null) const isCurrent = useEffectEvent((sessionId: string) => sessionId === handlers.currentSessionId()) - const onText = useEffectEvent( - (sessionId: string, event: { type: 'partial' | 'final'; text: string }) => { - if (transcriptRef.current?.sessionId !== sessionId) { - transcriptRef.current = { sessionId, transcript: new DictationTranscript() } + const onLiveEvent = useEffectEvent( + ( + sessionId: string, + event: { type: 'partial' | 'final'; text: string } | { type: 'pending'; pending: boolean } + ) => { + if (liveRef.current?.sessionId !== sessionId) { + liveRef.current = { sessionId, transcript: new DictationTranscript(), speechPending: false } } - const { transcript } = transcriptRef.current - transcript.accept(event) - const { committed, tentative } = transcript.snapshot() - handlers.onLiveText({ committed, tentative }) + const live = liveRef.current + if (event.type === 'pending') { + live.speechPending = event.pending + } else { + live.transcript.accept(event) + } + const { committed, tentative } = live.transcript.snapshot() + handlers.onLiveText({ committed, tentative, speechPending: live.speechPending }) } ) const onFinal = useEffectEvent(handlers.onFinal) @@ -34,15 +45,20 @@ export function useDictationSpeechEvents(handlers: DictationSpeechEventHandlers) const cleanups = [ window.api.speech.onPartialTranscript((data) => { if (isCurrent(data.sessionId)) { - onText(data.sessionId, { type: 'partial', text: data.text }) + onLiveEvent(data.sessionId, { type: 'partial', text: data.text }) } }), window.api.speech.onFinalTranscript((data) => { if (data.text && isCurrent(data.sessionId)) { - onText(data.sessionId, { type: 'final', text: data.text }) + onLiveEvent(data.sessionId, { type: 'final', text: data.text }) onFinal(data.text) } }), + window.api.speech.onSpeechPending((data) => { + if (isCurrent(data.sessionId)) { + onLiveEvent(data.sessionId, { type: 'pending', pending: data.pending }) + } + }), window.api.speech.onStopped((data) => onStopped(data.sessionId)), window.api.speech.onError((data) => { if (isCurrent(data.sessionId)) { diff --git a/src/renderer/src/store/slices/dictation/dictation.ts b/src/renderer/src/store/slices/dictation/dictation.ts index 151d8628833..99f36afc3f0 100644 --- a/src/renderer/src/store/slices/dictation/dictation.ts +++ b/src/renderer/src/store/slices/dictation/dictation.ts @@ -3,9 +3,16 @@ import type { AppState } from '../../types' import type { DictationState, SpeechModelState } from '../../../../../shared/speech-types' import type { DictationTranscriptSnapshot } from '../../../../../shared/dictation-transcript' -export type DictationLiveText = Pick +export type DictationLiveText = Pick & { + /** The engine heard speech it hasn't committed yet (pause-cut on-device models only). */ + speechPending: boolean +} -export const NO_DICTATION_LIVE_TEXT: DictationLiveText = { committed: '', tentative: '' } +export const NO_DICTATION_LIVE_TEXT: DictationLiveText = { + committed: '', + tentative: '', + speechPending: false +} export type DictationSlice = { dictationState: DictationState diff --git a/src/renderer/src/web/preload-api/web-speech-api.ts b/src/renderer/src/web/preload-api/web-speech-api.ts index 5716ae8e426..d811f54b10f 100644 --- a/src/renderer/src/web/preload-api/web-speech-api.ts +++ b/src/renderer/src/web/preload-api/web-speech-api.ts @@ -89,6 +89,7 @@ export function createWebSpeechApi(): Pick< | 'cancelDictation' | 'onPartialTranscript' | 'onFinalTranscript' + | 'onSpeechPending' | 'onStopped' | 'onError' > { @@ -250,6 +251,8 @@ export function createWebSpeechApi(): Pick< }, onPartialTranscript: (callback) => subscribe(partialListeners, callback), onFinalTranscript: (callback) => subscribe(finalListeners, callback), + // Chunk replies carry no pause-cut signal, so the paired client keeps the plain pill. + onSpeechPending: () => () => undefined, onStopped: (callback) => subscribe(stoppedListeners, callback), onError: (callback) => subscribe(errorListeners, callback) } diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index 252e211284b..bd50b1dc7e4 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -84,6 +84,12 @@ export type SpeechTranscriptEvent = { sessionId: string } +/** Speech the engine has heard but not committed yet; only pause-cut on-device models send it. */ +export type SpeechPendingEvent = { + pending: boolean + sessionId: string +} + /** Desktop audio posted from the preload straight to the speech process; 'end' follows the last frame. */ export type SpeechAudioPortMessage = | { type: 'feed'; samples: Float32Array; sampleRate: number }