fix(dictation): a paired phone or browser waits as long as the computer for long cloud dictations (#40238)

* fix(dictation): wait as long as the host's cloud deadline when a paired phone or browser stops

* fix(dictation): compute the host's cloud deadline from its exact sample count

mobile/src/hooks/mobile-dictation-audio-chunk.ts

@@ -260,7 +260,7 @@ async function sendAudioChunk(
     dictationId,
     audioBase64: bytesToBase64(bytes),
     sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE,
-    seq: queue.pendingAudioBudget.nextSeq()
+    seq: queue.pendingAudioBudget.nextSeq(bytes.byteLength)
   }
   const hostDedupesResends = queue.pendingAudioBudget.hostDedupesResends
   try {

mobile/src/hooks/mobile-dictation-desktop-start.ts

@@ -1,7 +1,4 @@
-import {
-  DICTATION_FINISH_TIMEOUT_MS,
-  isCurrentMobileDictationStart
-} from './mobile-dictation-session-state'
+import { isCurrentMobileDictationStart } from './mobile-dictation-session-state'
 import { sendAcrossConnectionMove } from './mobile-dictation-audio-chunk'
 import { dictationSessionLostMessage } from '../dictation/mobile-dictation-setup'
 import {
@@ -10,6 +7,7 @@ import {
   dictationSessionStart
 } from '../dictation/mobile-dictation-operations'
 import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-contract/speech-params'
+import { pairedDictationFinishTimeoutMs } from '../../../src/shared/dictation-finish-deadlines'
 import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload'
 import { probeLivenessWhileWaiting } from '../transport/rpc/rpc-liveness-probe-while-waiting'
 import type { RpcClient } from '../transport/rpc/rpc-client-contract'
@@ -57,15 +55,13 @@ export function cancelDesktopDictation(client: RpcClient, dictationId: string):
 // Returns the interpreted reply: the caller reads its text only after its staleness guard.
 export async function finishDesktopDictation(
   client: RpcClient,
-  dictationId: string
+  dictationId: string,
+  sentAudioSeconds: number
 ): Promise<unknown> {
+  const timeoutMs = pairedDictationFinishTimeoutMs(sentAudioSeconds)
   return dictationSessionFinish.interpret(
     await sendAcrossConnectionMove(client, () =>
-      dictationSessionFinish.request(
-        client,
-        { dictationId },
-        { timeoutMs: DICTATION_FINISH_TIMEOUT_MS }
-      )
+      dictationSessionFinish.request(client, { dictationId }, { timeoutMs })
     )
   )
 }

mobile/src/hooks/mobile-dictation-pending-audio-budget.ts

@@ -1,6 +1,7 @@
 import {
   DICTATION_SAMPLE_RATE,
-  MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES
+  MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES,
+  PCM_BYTES_PER_SAMPLE
 } from '../../../src/shared/rpc-contract/speech-params'
 
 export const MOBILE_DICTATION_PCM_SAMPLE_RATE = DICTATION_SAMPLE_RATE
@@ -15,6 +16,7 @@ export class MobileDictationPendingAudioBudget {
   // Backlog not yet taken into a request; it sits at the front of the held audio.
   private untakenBacklogBytes = 0
   private sentChunks = 0
+  private sentBytes = 0
   // Set at each start: a host that feeds each chunk seq once can take audio a drop left unanswered again.
   hostDedupesResends = false
 
@@ -24,10 +26,16 @@ export class MobileDictationPendingAudioBudget {
     return this.pendingBytes
   }
 
-  nextSeq(): number {
+  // Numbers a request and counts its audio, which sets how long finish waits.
+  nextSeq(byteLength: number): number {
+    this.sentBytes += normalizeByteLength(byteLength)
     return this.sentChunks++
   }
 
+  get sentAudioSeconds(): number {
+    return this.sentBytes / (PCM_BYTES_PER_SAMPLE * MOBILE_DICTATION_PCM_SAMPLE_RATE)
+  }
+
   reserveBacklog(byteLength: number): void {
     const normalizedByteLength = normalizeByteLength(byteLength)
     this.pendingBytes += normalizedByteLength
@@ -62,6 +70,7 @@ export class MobileDictationPendingAudioBudget {
     this.backlogBytes = 0
     this.untakenBacklogBytes = 0
     this.sentChunks = 0
+    this.sentBytes = 0
     this.hostDedupesResends = hostDedupesResends
   }
 }

mobile/src/hooks/mobile-dictation-session-state.ts

@@ -36,8 +36,6 @@ export type UseMobileDictationResult = {
   meter: MobileDictationMeter
 }
 
-export const DICTATION_FINISH_TIMEOUT_MS = 75_000
-
 // An interrupted user tap needs a renderable error; only explicit user cancellation may end silently.
 export const MOBILE_DICTATION_INPUT_CLOSED_ERROR_MESSAGE =
   'Voice dictation stopped because this session is no longer accepting input. Try again.'

mobile/src/hooks/use-mobile-dictation.ts

@@ -212,7 +212,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
         if (!isCurrent() || !client) {
           return
         }
-        const finished = await finishDesktopDictation(client, dictationId)
+        const sentSeconds = pendingAudioBudgetRef.current.sentAudioSeconds
+        const finished = await finishDesktopDictation(client, dictationId, sentSeconds)
         if (!isCurrent()) {
           return
         }

src/main/speech/openai-transcription-client.ts

@@ -1,17 +1,18 @@
 import { getMainHttpClient } from '../network/http-client'
 import { resampleToRate } from './stt-audio-resample'
 import { replaceRetiredSpeechModelId } from './model-catalog'
+import {
+  CLOUD_TRANSCRIPTION_SAMPLE_RATE,
+  MAX_CLOUD_TRANSCRIPTION_SECONDS,
+  cloudTranscriptionTimeoutMs
+} from '../../shared/dictation-finish-deadlines'
 
 export const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record<string, string> = {
   'openai-gpt-transcribe': 'gpt-transcribe'
 }
 
 export const OPENAI_TRANSCRIPTION_URL = 'https://api.openai.com/v1/audio/transcriptions'
-const CLOUD_TRANSCRIPTION_SAMPLE_RATE = 16000
-const MAX_CLOUD_AUDIO_SAMPLES = 10 * 60 * CLOUD_TRANSCRIPTION_SAMPLE_RATE
-// Room to answer plus the upload time of the WAV at 1 Mbit/s (125 bytes per ms).
-const TRANSCRIPTION_RESPONSE_ALLOWANCE_MS = 30_000
-const UPLOAD_BYTES_PER_MS = 125
+const MAX_CLOUD_AUDIO_SAMPLES = MAX_CLOUD_TRANSCRIPTION_SECONDS * CLOUD_TRANSCRIPTION_SAMPLE_RATE
 
 export type OpenAiTranscriptionResponse = {
   text?: unknown
@@ -154,9 +155,7 @@ export class OpenAiTranscriptionSession {
     // A named WAV blob satisfies multipart uploads without temporary files.
     form.append('file', new Blob([wav], { type: 'audio/wav' }), 'dictation.wav')
 
-    const timeout = AbortSignal.timeout(
-      TRANSCRIPTION_RESPONSE_ALLOWANCE_MS + Math.ceil(wav.byteLength / UPLOAD_BYTES_PER_MS)
-    )
+    const timeout = AbortSignal.timeout(cloudTranscriptionTimeoutMs(this.audioSamples))
     try {
       return await this.upload(form, AbortSignal.any([this.abortController.signal, timeout]))
     } catch (error) {

src/main/speech/stt-session-stop.ts

@@ -7,8 +7,8 @@ import {
 } from './stt-session-state'
 import { waitForSttWorkerStop, type SttWorkerStopOutcome } from './stt-worker-stop'
 import { STT_WORKER_HUNG_ERROR } from './stt-worker-process'
+import { LOCAL_DICTATION_STOP_TIMEOUT_MS } from '../../shared/dictation-finish-deadlines'
 
-const STOP_DICTATION_TIMEOUT_MS = 60_000
 // A cancel waits only this long behind a busy decode before giving up on it.
 const DISCARD_DICTATION_TIMEOUT_MS = 2_000
 
@@ -74,7 +74,7 @@ function stopLocalSession(
     worker: engine.worker,
     streamId: session.streamId,
     sink: session.sink,
-    timeoutMs: discard ? DISCARD_DICTATION_TIMEOUT_MS : STOP_DICTATION_TIMEOUT_MS,
+    timeoutMs: discard ? DISCARD_DICTATION_TIMEOUT_MS : LOCAL_DICTATION_STOP_TIMEOUT_MS,
     discard,
     finish: (outcome) => finishLocalStop(state, session, engine, outcome, discard)
   })

src/renderer/src/web/preload-api/web-speech-api.ts

@@ -16,10 +16,9 @@ import { createNonSecureContextUuid } from '../../../../shared/non-secure-contex
 import { errorMessage } from '../../../../shared/error-message'
 import { callRuntimeResult } from './web-runtime-calls'
 import { withTimeout } from '../../../../shared/promise-timeout-fallback'
+import { pairedDictationFinishTimeoutMs } from '../../../../shared/dictation-finish-deadlines'
 
 const MAX_CHUNK_SAMPLES = MAX_DICTATION_AUDIO_CHUNK_BYTES / PCM_BYTES_PER_SAMPLE
-// Matches the phone: a long recording can take a while to transcribe on the host.
-const FINISH_TIMEOUT_MS = 75_000
 // A slow link can leave a backlog of chunks behind Stop; past this, keep the text so far instead.
 const STOP_DRAIN_TIMEOUT_MS = 10_000
 
@@ -28,6 +27,7 @@ type HostDictation = {
   dictationId: string
   seq: number
   sampleRate: number
+  sentAudioSeconds: number
   pending: Float32Array[]
   sending: Promise<void> | null
   failed: boolean
@@ -116,12 +116,14 @@ export function createWebSpeechApi(): Pick<
   const sendPendingAudio = async (dictation: HostDictation): Promise<void> => {
     while (dictation.pending.length > 0 && !dictation.failed) {
       try {
+        const audio = takePendingAudio(dictation)
+        dictation.sentAudioSeconds += audio.length / dictation.sampleRate
         const reply = await callRuntimeResult<{
           text?: string
           audioLimitReached?: boolean
         }>('speech.dictation.chunk', {
           dictationId: dictation.dictationId,
-          audioBase64: encodePcm16Base64(takePendingAudio(dictation)),
+          audioBase64: encodePcm16Base64(audio),
           sampleRate: dictation.sampleRate,
           seq: dictation.seq++
         })
@@ -153,6 +155,7 @@ export function createWebSpeechApi(): Pick<
         dictationId: `web-${createNonSecureContextUuid()}`,
         seq: 0,
         sampleRate: 0,
+        sentAudioSeconds: 0,
         pending: [],
         sending: null,
         failed: false,
@@ -216,7 +219,7 @@ export function createWebSpeechApi(): Pick<
         const { text } = await callRuntimeResult<{ text: string }>(
           'speech.dictation.finish',
           { dictationId: dictation.dictationId },
-          FINISH_TIMEOUT_MS
+          pairedDictationFinishTimeoutMs(dictation.sentAudioSeconds)
         )
         if (text) {
           emit(finalListeners, { sessionId, text })

src/shared/dictation-finish-deadlines.ts

@@ -0,0 +1,24 @@
+// Cloud uploads are one 16 kHz 16-bit mono WAV, capped at 10 minutes per dictation.
+export const CLOUD_TRANSCRIPTION_SAMPLE_RATE = 16_000
+export const MAX_CLOUD_TRANSCRIPTION_SECONDS = 10 * 60
+const WAV_HEADER_BYTES = 44
+// Room to answer plus the upload time of the WAV at 1 Mbit/s (125 bytes per ms).
+const CLOUD_RESPONSE_ALLOWANCE_MS = 30_000
+const UPLOAD_BYTES_PER_MS = 125
+
+export const LOCAL_DICTATION_STOP_TIMEOUT_MS = 60_000
+// The host's answer still has to cross the network after its own deadline.
+const PAIRED_CLIENT_REPLY_MARGIN_MS = 15_000
+
+/** How long the host lets a cloud upload of this many 16 kHz samples run before failing it. */
+export function cloudTranscriptionTimeoutMs(audioSamples: number): number {
+  const wavBytes = WAV_HEADER_BYTES + audioSamples * 2
+  return CLOUD_RESPONSE_ALLOWANCE_MS + Math.ceil(wavBytes / UPLOAD_BYTES_PER_MS)
+}
+
+/** A paired phone or browser waits out whichever deadline the host applies to this much audio. */
+export function pairedDictationFinishTimeoutMs(audioSeconds: number): number {
+  const seconds = Math.min(audioSeconds, MAX_CLOUD_TRANSCRIPTION_SECONDS)
+  const cloudMs = cloudTranscriptionTimeoutMs(Math.ceil(seconds * CLOUD_TRANSCRIPTION_SAMPLE_RATE))
+  return Math.max(LOCAL_DICTATION_STOP_TIMEOUT_MS, cloudMs) + PAIRED_CLIENT_REPLY_MARGIN_MS
+}