feat(dictation): pulse three dots in the pill while spoken words are still on their way (#40443)

* feat(dictation): show pending dots while the engine holds speech it hasn't committed

The VAD pause segmenter already knows when a phrase is open. The recognizer
now posts speech-pending when a phrase opens and clears it once that phrase's
text is posted, and the desktop pill shows three dots after the text while it
is on. Only pause-cut on-device streams send it, so cloud, the paired web
client and phones keep today's pill.

* fix(speech): keep a lone click from turning the pending dots on

src/main/ipc/speech.ts

@@ -204,6 +204,9 @@ export function registerSpeechHandlers(store: Store): void {
               case 'final':
                 window.webContents.send('speech:final', { text: msg.text ?? '', sessionId })
                 break
+              case 'speech-pending':
+                window.webContents.send('speech:pending', { pending: msg.pending, sessionId })
+                break
               case 'stopped':
                 cleanupSessionListener()
                 window.webContents.send('speech:stopped', { sessionId })

src/main/speech/stt-recognizer.ts

@@ -12,7 +12,9 @@ export type SttRecognizerInit = {
 
 export type SttRecognizerEvent =
   /** One decoded offline chunk; `samples` counts the stream's input-rate samples decoded so far. */
-  { type: 'chunk'; streamId: string; text: string; samples: number }
+  | { type: 'chunk'; streamId: string; text: string; samples: number }
+  /** Pause-cut streams only: speech was heard that no chunk has committed yet. */
+  | { type: 'speech-pending'; streamId: string; pending: boolean }
 
 export type SttRecognizerPost = (event: SttRecognizerEvent) => void
 

src/main/speech/stt-service.ts

@@ -23,6 +23,8 @@ export type SttEvent =
   | { type: 'ready' }
   | { type: 'partial'; text?: string }
   | { type: 'final'; text?: string }
+  /** Speech was heard that no final has committed yet; only from pause-cut on-device models. */
+  | { type: 'speech-pending'; pending: boolean }
   | { type: 'stopped' }
   | { type: 'error'; error?: string }
 

src/main/speech/stt-transcribe-cpp-recognizer.ts

@@ -17,6 +17,7 @@ function performanceCoreThreads(): number {
 // Without the detector, audio is still cut at the decode bound, just not at pauses.
 type AudioSegmenter = {
   readonly heldSamples: number
+  readonly phraseOpen?: boolean
   push(samples: Float32Array): Float32Array[] | Promise<Float32Array[]>
   flush(): Float32Array[] | Promise<Float32Array[]>
 }
@@ -44,6 +45,7 @@ type StreamCoverage = {
   resampledSamples: number
   reportedSamples: number
   slowPhraseDecodes: number
+  speechPending: boolean
 }
 
 // Whisper pads every decode to a 30 s window, so each phrase costs a full decode. Where that takes longer than the
@@ -121,7 +123,8 @@ export async function createTranscribeCppRecognizer(
       inputSamples: 0,
       resampledSamples: 0,
       reportedSamples: 0,
-      slowPhraseDecodes: 0
+      slowPhraseDecodes: 0,
+      speechPending: false
     }
     if (vad && vadModel) {
       stream.segmenter = new VadPauseSegmenter(
@@ -201,6 +204,14 @@ export async function createTranscribeCppRecognizer(
     }
   }
 
+  // Only pause-cut streams report it: without the detector, nothing tells speech from silence.
+  const notePendingSpeech = (streamId: string, stream: StreamCoverage, pending: boolean) => {
+    if (stream.vad && stream.speechPending !== pending) {
+      stream.speechPending = pending
+      post({ type: 'speech-pending', streamId, pending })
+    }
+  }
+
   const closeStream = (streamId: string): StreamCoverage | undefined => {
     const stream = streams.get(streamId)
     streams.delete(streamId)
@@ -219,7 +230,11 @@ export async function createTranscribeCppRecognizer(
       )
       stream.inputSamples += samples.length
       stream.resampledSamples += resampled.length
-      await decode(streamId, stream, await stream.segmenter.push(resampled))
+      const phrases = await stream.segmenter.push(resampled)
+      // A cut phrase stays pending until its text is posted.
+      notePendingSpeech(streamId, stream, phrases.length > 0 || !!stream.segmenter.phraseOpen)
+      await decode(streamId, stream, phrases)
+      notePendingSpeech(streamId, stream, !!stream.segmenter.phraseOpen)
     },
     async finish(streamId) {
       const stream = closeStream(streamId)
@@ -228,6 +243,7 @@ export async function createTranscribeCppRecognizer(
       }
       try {
         await decode(streamId, stream, await stream.segmenter.flush())
+        notePendingSpeech(streamId, stream, false)
       } finally {
         stream.vad?.dispose()
       }

src/main/speech/stt-vad-pause-segmenter.test.ts

@@ -69,13 +69,17 @@ describe('VadPauseSegmenter', () => {
     ])
   })
 
-  it('holds a click shorter than 0.25 s of speech until real speech joins it', async () => {
+  it('holds a click shorter than 0.25 s of speech until real speech joins it, without opening a phrase', async () => {
     const segmenter = new VadPauseSegmenter(fakeVad(), RATE, FRAME)
     expect(await segmenter.push(audio([0.9, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2]))).toEqual([])
-    const ready = await segmenter.push(audio([0.8, 0.8, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2]))
+    expect(segmenter.phraseOpen).toBe(false)
+    expect(await segmenter.push(audio([0.8, 0.8]))).toEqual([])
+    expect(segmenter.phraseOpen).toBe(true)
+    const ready = await segmenter.push(audio([0.2, 0.2, 0.2, 0.2, 0.2, 0.2]))
     expect(framesOf(ready)).toEqual([
       [0.9, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.8, 0.8, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2]
     ])
+    expect(segmenter.phraseOpen).toBe(false)
   })
 
   it('returns nothing for a dictation that never had speech', async () => {

src/main/speech/stt-vad-pause-segmenter.ts

@@ -48,6 +48,11 @@ export class VadPauseSegmenter {
     return this.unscored.length + this.prerollLength + this.phrase.heldSamples
   }
 
+  /** At least a phrase's worth of speech not cut yet (a lone click isn't); after a detector error, until stop. */
+  get phraseOpen(): boolean {
+    return this.inSpeech && (this.vadFailed || this.speechFrames >= this.minSpeechFrames)
+  }
+
   /** Return the phrases that ended (or reached the decode bound) within these samples. */
   async push(samples: Float32Array): Promise<Float32Array[]> {
     if (this.vadFailed) {

src/main/speech/stt-worker-cpu-fallback.ts

@@ -15,6 +15,7 @@ type Feed = Extract<SttWorkerRequest, { type: 'feed' }>
 type Open = Extract<SttWorkerRequest, { type: 'open' }>
 type ChildMessage =
   | { type: 'chunk'; streamId: string; text: string; samples: number }
+  | { type: 'speech-pending'; streamId: string; pending: boolean }
   | {
       type: 'ready' | 'stopped' | 'error' | 'partial' | 'final'
       streamId?: string

src/preload/api/speech-api.ts

@@ -7,6 +7,7 @@ import type {
   SpeechLifecycleEvent,
   SpeechModelManifest,
   SpeechModelState,
+  SpeechPendingEvent,
   SpeechTranscriptEvent
 } from '../../shared/speech-types'
 
@@ -39,6 +40,7 @@ export type SpeechApi = {
   cancelDictation: (sessionId: string) => Promise<void>
   onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void
   onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void
+  onSpeechPending: (callback: (data: SpeechPendingEvent) => void) => () => void
   onDownloadProgress: (
     callback: (data: { modelId: string; progress: number }) => void
   ) => () => void

src/preload/api/speech-bridge.ts

@@ -9,6 +9,7 @@ import type {
   SpeechLifecycleEvent,
   SpeechModelManifest,
   SpeechModelState,
+  SpeechPendingEvent,
   SpeechTranscriptEvent
 } from '../../shared/speech-types'
 import type { PreloadApi } from '../api-types'
@@ -111,6 +112,12 @@ export const speechApi = {
     ipcRenderer.on('speech:final', listener)
     return () => ipcRenderer.removeListener('speech:final', listener)
   },
+  onSpeechPending: (callback: (data: SpeechPendingEvent) => void): (() => void) => {
+    const listener = (_event: Electron.IpcRendererEvent, data: SpeechPendingEvent): void =>
+      callback(data)
+    ipcRenderer.on('speech:pending', listener)
+    return () => ipcRenderer.removeListener('speech:pending', listener)
+  },
   onDownloadProgress: (
     callback: (data: { modelId: string; progress: number }) => void
   ): (() => void) => {

src/renderer/src/components/dictation/DictationIndicator.fixture.tsx

@@ -53,7 +53,8 @@ export const liveText = defineFixture({
     dictationState: 'listening',
     liveText: {
       committed: 'It should go back to the page you came from.',
-      tentative: 'Check where returnTo gets dropped'
+      tentative: 'Check where returnTo gets dropped',
+      speechPending: false
     }
   },
   setup: () => publishDictationMeter({ level: 0.6, isSpeaking: true, isClipping: false }),
@@ -66,13 +67,38 @@ export const committedText = defineFixture({
     dictationState: 'listening',
     liveText: {
       committed: 'It should go back to the page you came from. Check where returnTo gets dropped.',
-      tentative: ''
+      tentative: '',
+      speechPending: false
     }
   },
   setup: () => publishDictationMeter({ level: 0.2, isSpeaking: false, isClipping: false }),
   render: () => <PaneWithPill />
 })
 
+export const pendingSpeech = defineFixture({
+  width: 520,
+  store: {
+    dictationState: 'listening',
+    liveText: {
+      committed: 'It should go back to the page you came from.',
+      tentative: '',
+      speechPending: true
+    }
+  },
+  setup: () => publishDictationMeter({ level: 0.6, isSpeaking: true, isClipping: false }),
+  render: () => <PaneWithPill />
+})
+
+export const pendingFirstWords = defineFixture({
+  width: 520,
+  store: {
+    dictationState: 'listening',
+    liveText: { committed: '', tentative: '', speechPending: true }
+  },
+  setup: () => publishDictationMeter({ level: 0.6, isSpeaking: true, isClipping: false }),
+  render: () => <PaneWithPill />
+})
+
 export const tooLoud = defineFixture({
   width: 520,
   store: { dictationState: 'listening' },

src/renderer/src/components/dictation/DictationIndicator.tsx

@@ -126,6 +126,21 @@ function DictationPillMeter({ recording, status }: { recording: boolean; status:
   )
 }
 
+// Words heard but not committed yet: they land at the next pause.
+function DictationPendingDots() {
+  return (
+    <span className="ml-1.5 inline-flex gap-1 align-middle">
+      {[0, 150, 300].map((delayMs) => (
+        <span
+          key={delayMs}
+          className="size-1 animate-pulse rounded-full bg-muted-foreground motion-reduce:animate-none"
+          style={{ animationDelay: `${delayMs}ms` }}
+        />
+      ))}
+    </span>
+  )
+}
+
 // Presses on the pill keep focus where the words are going.
 const keepFocus = (event: React.MouseEvent): void => event.preventDefault()
 const keepFocusProps = { [KEEPS_FOCUS_ATTRIBUTE]: '', onMouseDown: keepFocus }
@@ -146,7 +161,7 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica
   const showSpinner = useDelayedFlag(finishing, FINISH_SPINNER_DELAY_MS)
   const nearCloudLimit = cloudModel && seconds >= CLOUD_LIMIT_WARNING_SECONDS
   const failedKind = dictationState === 'error' ? failure : null
-  const committed = liveText.committed
+  const { committed, speechPending } = liveText
   const tentative = liveText.tentative.trim()
   const queuedStatus =
     queued === 'chat'
@@ -200,7 +215,7 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica
               )}
               <span className="truncate">{queuedStatus}</span>
             </span>
-          ) : committed || tentative ? (
+          ) : committed || tentative || speechPending ? (
             // Right-aligned so the newest words stay in view; older ones fade off the left edge.
             <span
               aria-hidden
@@ -210,6 +225,7 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica
                 {committed}
                 {dictationSeparators(committed, tentative, '').leading}
                 <span className="text-muted-foreground">{tentative}</span>
+                {speechPending ? <DictationPendingDots /> : null}
               </span>
             </span>
           ) : (

src/renderer/src/components/dictation/use-dictation-speech-events.ts

@@ -13,17 +13,28 @@ type DictationSpeechEventHandlers = {
 }
 
 export function useDictationSpeechEvents(handlers: DictationSpeechEventHandlers): void {
-  const transcriptRef = useRef<{ sessionId: string; transcript: DictationTranscript } | null>(null)
+  const liveRef = useRef<{
+    sessionId: string
+    transcript: DictationTranscript
+    speechPending: boolean
+  } | null>(null)
   const isCurrent = useEffectEvent((sessionId: string) => sessionId === handlers.currentSessionId())
-  const onText = useEffectEvent(
-    (sessionId: string, event: { type: 'partial' | 'final'; text: string }) => {
-      if (transcriptRef.current?.sessionId !== sessionId) {
-        transcriptRef.current = { sessionId, transcript: new DictationTranscript() }
+  const onLiveEvent = useEffectEvent(
+    (
+      sessionId: string,
+      event: { type: 'partial' | 'final'; text: string } | { type: 'pending'; pending: boolean }
+    ) => {
+      if (liveRef.current?.sessionId !== sessionId) {
+        liveRef.current = { sessionId, transcript: new DictationTranscript(), speechPending: false }
       }
-      const { transcript } = transcriptRef.current
-      transcript.accept(event)
-      const { committed, tentative } = transcript.snapshot()
-      handlers.onLiveText({ committed, tentative })
+      const live = liveRef.current
+      if (event.type === 'pending') {
+        live.speechPending = event.pending
+      } else {
+        live.transcript.accept(event)
+      }
+      const { committed, tentative } = live.transcript.snapshot()
+      handlers.onLiveText({ committed, tentative, speechPending: live.speechPending })
     }
   )
   const onFinal = useEffectEvent(handlers.onFinal)
@@ -34,15 +45,20 @@ export function useDictationSpeechEvents(handlers: DictationSpeechEventHandlers)
     const cleanups = [
       window.api.speech.onPartialTranscript((data) => {
         if (isCurrent(data.sessionId)) {
-          onText(data.sessionId, { type: 'partial', text: data.text })
+          onLiveEvent(data.sessionId, { type: 'partial', text: data.text })
         }
       }),
       window.api.speech.onFinalTranscript((data) => {
         if (data.text && isCurrent(data.sessionId)) {
-          onText(data.sessionId, { type: 'final', text: data.text })
+          onLiveEvent(data.sessionId, { type: 'final', text: data.text })
           onFinal(data.text)
         }
       }),
+      window.api.speech.onSpeechPending((data) => {
+        if (isCurrent(data.sessionId)) {
+          onLiveEvent(data.sessionId, { type: 'pending', pending: data.pending })
+        }
+      }),
       window.api.speech.onStopped((data) => onStopped(data.sessionId)),
       window.api.speech.onError((data) => {
         if (isCurrent(data.sessionId)) {

src/renderer/src/store/slices/dictation/dictation.ts

@@ -3,9 +3,16 @@ import type { AppState } from '../../types'
 import type { DictationState, SpeechModelState } from '../../../../../shared/speech-types'
 import type { DictationTranscriptSnapshot } from '../../../../../shared/dictation-transcript'
 
-export type DictationLiveText = Pick<DictationTranscriptSnapshot, 'committed' | 'tentative'>
+export type DictationLiveText = Pick<DictationTranscriptSnapshot, 'committed' | 'tentative'> & {
+  /** The engine heard speech it hasn't committed yet (pause-cut on-device models only). */
+  speechPending: boolean
+}
 
-export const NO_DICTATION_LIVE_TEXT: DictationLiveText = { committed: '', tentative: '' }
+export const NO_DICTATION_LIVE_TEXT: DictationLiveText = {
+  committed: '',
+  tentative: '',
+  speechPending: false
+}
 
 export type DictationSlice = {
   dictationState: DictationState

src/renderer/src/web/preload-api/web-speech-api.ts

@@ -89,6 +89,7 @@ export function createWebSpeechApi(): Pick<
   | 'cancelDictation'
   | 'onPartialTranscript'
   | 'onFinalTranscript'
+  | 'onSpeechPending'
   | 'onStopped'
   | 'onError'
 > {
@@ -250,6 +251,8 @@ export function createWebSpeechApi(): Pick<
     },
     onPartialTranscript: (callback) => subscribe(partialListeners, callback),
     onFinalTranscript: (callback) => subscribe(finalListeners, callback),
+    // Chunk replies carry no pause-cut signal, so the paired client keeps the plain pill.
+    onSpeechPending: () => () => undefined,
     onStopped: (callback) => subscribe(stoppedListeners, callback),
     onError: (callback) => subscribe(errorListeners, callback)
   }

src/shared/speech-types.ts

@@ -84,6 +84,12 @@ export type SpeechTranscriptEvent = {
   sessionId: string
 }
 
+/** Speech the engine has heard but not committed yet; only pause-cut on-device models send it. */
+export type SpeechPendingEvent = {
+  pending: boolean
+  sessionId: string
+}
+
 /** Desktop audio posted from the preload straight to the speech process; 'end' follows the last frame. */
 export type SpeechAudioPortMessage =
   | { type: 'feed'; samples: Float32Array; sampleRate: number }