From e9d762fff47bb76f37e184e56bf1480cecf7664f Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:29:04 +0000 Subject: [PATCH] fix(dictation): a paired phone or browser waits as long as the computer for long cloud dictations (#40238) * fix(dictation): wait as long as the host's cloud deadline when a paired phone or browser stops * fix(dictation): compute the host's cloud deadline from its exact sample count --- .../src/hooks/mobile-dictation-audio-chunk.ts | 2 +- .../hooks/mobile-dictation-desktop-start.ts | 16 +++++-------- .../mobile-dictation-pending-audio-budget.ts | 13 ++++++++-- .../hooks/mobile-dictation-session-state.ts | 2 -- mobile/src/hooks/use-mobile-dictation.ts | 3 ++- .../speech/openai-transcription-client.ts | 15 ++++++------ src/main/speech/stt-session-stop.ts | 4 ++-- .../src/web/preload-api/web-speech-api.ts | 11 +++++---- src/shared/dictation-finish-deadlines.ts | 24 +++++++++++++++++++ 9 files changed, 60 insertions(+), 30 deletions(-) create mode 100644 src/shared/dictation-finish-deadlines.ts diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index 18f382de908..c9a18b0d721 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -260,7 +260,7 @@ async function sendAudioChunk( dictationId, audioBase64: bytesToBase64(bytes), sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE, - seq: queue.pendingAudioBudget.nextSeq() + seq: queue.pendingAudioBudget.nextSeq(bytes.byteLength) } const hostDedupesResends = queue.pendingAudioBudget.hostDedupesResends try { diff --git a/mobile/src/hooks/mobile-dictation-desktop-start.ts b/mobile/src/hooks/mobile-dictation-desktop-start.ts index 29eef603b45..2b6b28cdf0c 100644 --- a/mobile/src/hooks/mobile-dictation-desktop-start.ts +++ b/mobile/src/hooks/mobile-dictation-desktop-start.ts @@ -1,7 +1,4 @@ -import { - DICTATION_FINISH_TIMEOUT_MS, - isCurrentMobileDictationStart -} from './mobile-dictation-session-state' +import { isCurrentMobileDictationStart } from './mobile-dictation-session-state' import { sendAcrossConnectionMove } from './mobile-dictation-audio-chunk' import { dictationSessionLostMessage } from '../dictation/mobile-dictation-setup' import { @@ -10,6 +7,7 @@ import { dictationSessionStart } from '../dictation/mobile-dictation-operations' import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-contract/speech-params' +import { pairedDictationFinishTimeoutMs } from '../../../src/shared/dictation-finish-deadlines' import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload' import { probeLivenessWhileWaiting } from '../transport/rpc/rpc-liveness-probe-while-waiting' import type { RpcClient } from '../transport/rpc/rpc-client-contract' @@ -57,15 +55,13 @@ export function cancelDesktopDictation(client: RpcClient, dictationId: string): // Returns the interpreted reply: the caller reads its text only after its staleness guard. export async function finishDesktopDictation( client: RpcClient, - dictationId: string + dictationId: string, + sentAudioSeconds: number ): Promise { + const timeoutMs = pairedDictationFinishTimeoutMs(sentAudioSeconds) return dictationSessionFinish.interpret( await sendAcrossConnectionMove(client, () => - dictationSessionFinish.request( - client, - { dictationId }, - { timeoutMs: DICTATION_FINISH_TIMEOUT_MS } - ) + dictationSessionFinish.request(client, { dictationId }, { timeoutMs }) ) ) } diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts index 6f59df36b63..d2b83fe9b99 100644 --- a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts @@ -1,6 +1,7 @@ import { DICTATION_SAMPLE_RATE, - MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES + MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES, + PCM_BYTES_PER_SAMPLE } from '../../../src/shared/rpc-contract/speech-params' export const MOBILE_DICTATION_PCM_SAMPLE_RATE = DICTATION_SAMPLE_RATE @@ -15,6 +16,7 @@ export class MobileDictationPendingAudioBudget { // Backlog not yet taken into a request; it sits at the front of the held audio. private untakenBacklogBytes = 0 private sentChunks = 0 + private sentBytes = 0 // Set at each start: a host that feeds each chunk seq once can take audio a drop left unanswered again. hostDedupesResends = false @@ -24,10 +26,16 @@ export class MobileDictationPendingAudioBudget { return this.pendingBytes } - nextSeq(): number { + // Numbers a request and counts its audio, which sets how long finish waits. + nextSeq(byteLength: number): number { + this.sentBytes += normalizeByteLength(byteLength) return this.sentChunks++ } + get sentAudioSeconds(): number { + return this.sentBytes / (PCM_BYTES_PER_SAMPLE * MOBILE_DICTATION_PCM_SAMPLE_RATE) + } + reserveBacklog(byteLength: number): void { const normalizedByteLength = normalizeByteLength(byteLength) this.pendingBytes += normalizedByteLength @@ -62,6 +70,7 @@ export class MobileDictationPendingAudioBudget { this.backlogBytes = 0 this.untakenBacklogBytes = 0 this.sentChunks = 0 + this.sentBytes = 0 this.hostDedupesResends = hostDedupesResends } } diff --git a/mobile/src/hooks/mobile-dictation-session-state.ts b/mobile/src/hooks/mobile-dictation-session-state.ts index 2664ae7b540..a2ab4abb53e 100644 --- a/mobile/src/hooks/mobile-dictation-session-state.ts +++ b/mobile/src/hooks/mobile-dictation-session-state.ts @@ -36,8 +36,6 @@ export type UseMobileDictationResult = { meter: MobileDictationMeter } -export const DICTATION_FINISH_TIMEOUT_MS = 75_000 - // An interrupted user tap needs a renderable error; only explicit user cancellation may end silently. export const MOBILE_DICTATION_INPUT_CLOSED_ERROR_MESSAGE = 'Voice dictation stopped because this session is no longer accepting input. Try again.' diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 46c96310138..bfc0e9cd3c9 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -212,7 +212,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil if (!isCurrent() || !client) { return } - const finished = await finishDesktopDictation(client, dictationId) + const sentSeconds = pendingAudioBudgetRef.current.sentAudioSeconds + const finished = await finishDesktopDictation(client, dictationId, sentSeconds) if (!isCurrent()) { return } diff --git a/src/main/speech/openai-transcription-client.ts b/src/main/speech/openai-transcription-client.ts index b17ba3411cf..452685c9b29 100644 --- a/src/main/speech/openai-transcription-client.ts +++ b/src/main/speech/openai-transcription-client.ts @@ -1,17 +1,18 @@ import { getMainHttpClient } from '../network/http-client' import { resampleToRate } from './stt-audio-resample' import { replaceRetiredSpeechModelId } from './model-catalog' +import { + CLOUD_TRANSCRIPTION_SAMPLE_RATE, + MAX_CLOUD_TRANSCRIPTION_SECONDS, + cloudTranscriptionTimeoutMs +} from '../../shared/dictation-finish-deadlines' export const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record = { 'openai-gpt-transcribe': 'gpt-transcribe' } export const OPENAI_TRANSCRIPTION_URL = 'https://api.openai.com/v1/audio/transcriptions' -const CLOUD_TRANSCRIPTION_SAMPLE_RATE = 16000 -const MAX_CLOUD_AUDIO_SAMPLES = 10 * 60 * CLOUD_TRANSCRIPTION_SAMPLE_RATE -// Room to answer plus the upload time of the WAV at 1 Mbit/s (125 bytes per ms). -const TRANSCRIPTION_RESPONSE_ALLOWANCE_MS = 30_000 -const UPLOAD_BYTES_PER_MS = 125 +const MAX_CLOUD_AUDIO_SAMPLES = MAX_CLOUD_TRANSCRIPTION_SECONDS * CLOUD_TRANSCRIPTION_SAMPLE_RATE export type OpenAiTranscriptionResponse = { text?: unknown @@ -154,9 +155,7 @@ export class OpenAiTranscriptionSession { // A named WAV blob satisfies multipart uploads without temporary files. form.append('file', new Blob([wav], { type: 'audio/wav' }), 'dictation.wav') - const timeout = AbortSignal.timeout( - TRANSCRIPTION_RESPONSE_ALLOWANCE_MS + Math.ceil(wav.byteLength / UPLOAD_BYTES_PER_MS) - ) + const timeout = AbortSignal.timeout(cloudTranscriptionTimeoutMs(this.audioSamples)) try { return await this.upload(form, AbortSignal.any([this.abortController.signal, timeout])) } catch (error) { diff --git a/src/main/speech/stt-session-stop.ts b/src/main/speech/stt-session-stop.ts index 1480be40b54..29a4e55820c 100644 --- a/src/main/speech/stt-session-stop.ts +++ b/src/main/speech/stt-session-stop.ts @@ -7,8 +7,8 @@ import { } from './stt-session-state' import { waitForSttWorkerStop, type SttWorkerStopOutcome } from './stt-worker-stop' import { STT_WORKER_HUNG_ERROR } from './stt-worker-process' +import { LOCAL_DICTATION_STOP_TIMEOUT_MS } from '../../shared/dictation-finish-deadlines' -const STOP_DICTATION_TIMEOUT_MS = 60_000 // A cancel waits only this long behind a busy decode before giving up on it. const DISCARD_DICTATION_TIMEOUT_MS = 2_000 @@ -74,7 +74,7 @@ function stopLocalSession( worker: engine.worker, streamId: session.streamId, sink: session.sink, - timeoutMs: discard ? DISCARD_DICTATION_TIMEOUT_MS : STOP_DICTATION_TIMEOUT_MS, + timeoutMs: discard ? DISCARD_DICTATION_TIMEOUT_MS : LOCAL_DICTATION_STOP_TIMEOUT_MS, discard, finish: (outcome) => finishLocalStop(state, session, engine, outcome, discard) }) diff --git a/src/renderer/src/web/preload-api/web-speech-api.ts b/src/renderer/src/web/preload-api/web-speech-api.ts index 2af2e654279..f90437d21ee 100644 --- a/src/renderer/src/web/preload-api/web-speech-api.ts +++ b/src/renderer/src/web/preload-api/web-speech-api.ts @@ -16,10 +16,9 @@ import { createNonSecureContextUuid } from '../../../../shared/non-secure-contex import { errorMessage } from '../../../../shared/error-message' import { callRuntimeResult } from './web-runtime-calls' import { withTimeout } from '../../../../shared/promise-timeout-fallback' +import { pairedDictationFinishTimeoutMs } from '../../../../shared/dictation-finish-deadlines' const MAX_CHUNK_SAMPLES = MAX_DICTATION_AUDIO_CHUNK_BYTES / PCM_BYTES_PER_SAMPLE -// Matches the phone: a long recording can take a while to transcribe on the host. -const FINISH_TIMEOUT_MS = 75_000 // A slow link can leave a backlog of chunks behind Stop; past this, keep the text so far instead. const STOP_DRAIN_TIMEOUT_MS = 10_000 @@ -28,6 +27,7 @@ type HostDictation = { dictationId: string seq: number sampleRate: number + sentAudioSeconds: number pending: Float32Array[] sending: Promise | null failed: boolean @@ -116,12 +116,14 @@ export function createWebSpeechApi(): Pick< const sendPendingAudio = async (dictation: HostDictation): Promise => { while (dictation.pending.length > 0 && !dictation.failed) { try { + const audio = takePendingAudio(dictation) + dictation.sentAudioSeconds += audio.length / dictation.sampleRate const reply = await callRuntimeResult<{ text?: string audioLimitReached?: boolean }>('speech.dictation.chunk', { dictationId: dictation.dictationId, - audioBase64: encodePcm16Base64(takePendingAudio(dictation)), + audioBase64: encodePcm16Base64(audio), sampleRate: dictation.sampleRate, seq: dictation.seq++ }) @@ -153,6 +155,7 @@ export function createWebSpeechApi(): Pick< dictationId: `web-${createNonSecureContextUuid()}`, seq: 0, sampleRate: 0, + sentAudioSeconds: 0, pending: [], sending: null, failed: false, @@ -216,7 +219,7 @@ export function createWebSpeechApi(): Pick< const { text } = await callRuntimeResult<{ text: string }>( 'speech.dictation.finish', { dictationId: dictation.dictationId }, - FINISH_TIMEOUT_MS + pairedDictationFinishTimeoutMs(dictation.sentAudioSeconds) ) if (text) { emit(finalListeners, { sessionId, text }) diff --git a/src/shared/dictation-finish-deadlines.ts b/src/shared/dictation-finish-deadlines.ts new file mode 100644 index 00000000000..e355e364d1b --- /dev/null +++ b/src/shared/dictation-finish-deadlines.ts @@ -0,0 +1,24 @@ +// Cloud uploads are one 16 kHz 16-bit mono WAV, capped at 10 minutes per dictation. +export const CLOUD_TRANSCRIPTION_SAMPLE_RATE = 16_000 +export const MAX_CLOUD_TRANSCRIPTION_SECONDS = 10 * 60 +const WAV_HEADER_BYTES = 44 +// Room to answer plus the upload time of the WAV at 1 Mbit/s (125 bytes per ms). +const CLOUD_RESPONSE_ALLOWANCE_MS = 30_000 +const UPLOAD_BYTES_PER_MS = 125 + +export const LOCAL_DICTATION_STOP_TIMEOUT_MS = 60_000 +// The host's answer still has to cross the network after its own deadline. +const PAIRED_CLIENT_REPLY_MARGIN_MS = 15_000 + +/** How long the host lets a cloud upload of this many 16 kHz samples run before failing it. */ +export function cloudTranscriptionTimeoutMs(audioSamples: number): number { + const wavBytes = WAV_HEADER_BYTES + audioSamples * 2 + return CLOUD_RESPONSE_ALLOWANCE_MS + Math.ceil(wavBytes / UPLOAD_BYTES_PER_MS) +} + +/** A paired phone or browser waits out whichever deadline the host applies to this much audio. */ +export function pairedDictationFinishTimeoutMs(audioSeconds: number): number { + const seconds = Math.min(audioSeconds, MAX_CLOUD_TRANSCRIPTION_SECONDS) + const cloudMs = cloudTranscriptionTimeoutMs(Math.ceil(seconds * CLOUD_TRANSCRIPTION_SAMPLE_RATE)) + return Math.max(LOCAL_DICTATION_STOP_TIMEOUT_MS, cloudMs) + PAIRED_CLIENT_REPLY_MARGIN_MS +}