diff --git a/src/main/speech/stt-recognizer.ts b/src/main/speech/stt-recognizer.ts index fb440a87295..667e3d6d719 100644 --- a/src/main/speech/stt-recognizer.ts +++ b/src/main/speech/stt-recognizer.ts @@ -6,14 +6,16 @@ export type SttRecognizerInit = { sampleRate: number /** Absent = auto-detect. */ language?: string - /** Silero VAD GGUF; when present, offline models commit text at each pause. */ + /** Silero VAD GGUF; when present, offline models show live text at each pause. */ vadModelPath?: string } export type SttRecognizerEvent = - /** One decoded offline chunk; `samples` counts the stream's input-rate samples decoded so far. */ + /** One committed offline chunk; `samples` counts the stream's input-rate samples committed so far. */ | { type: 'chunk'; streamId: string; text: string; samples: number } - /** Pause-cut streams only: speech was heard that no chunk has committed yet. */ + /** The uncommitted phrases' live text so far; the next chunk replaces it. */ + | { type: 'partial'; streamId: string; text: string } + /** Pause-cut streams only: speech was heard whose text hasn't arrived yet. */ | { type: 'speech-pending'; streamId: string; pending: boolean } /** A load step finished, so a slow load is told apart from a hung GPU. */ | { type: 'loading' } diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index bacc4b753ed..97231babfbc 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -24,7 +24,7 @@ export type SttEvent = | { type: 'ready' } | { type: 'partial'; text?: string } | { type: 'final'; text?: string } - /** Speech was heard that no final has committed yet; only from pause-cut on-device models. */ + /** Speech was heard whose text hasn't arrived yet; only from pause-cut on-device models. */ | { type: 'speech-pending'; pending: boolean } | { type: 'stopped' } | { type: 'error'; error?: string } diff --git a/src/main/speech/stt-transcribe-cpp-recognizer.ts b/src/main/speech/stt-transcribe-cpp-recognizer.ts index f292db59403..26b3a793039 100644 --- a/src/main/speech/stt-transcribe-cpp-recognizer.ts +++ b/src/main/speech/stt-transcribe-cpp-recognizer.ts @@ -2,8 +2,9 @@ import { availableParallelism } from 'node:os' import { join } from 'node:path' import { pathToFileURL } from 'node:url' import type * as TranscribeCpp from 'transcribe-cpp' +import { appendDictationText } from '../../shared/dictation-transcript-spacing' import { resampleToRate } from './stt-audio-resample' -import { OfflineAudioChunker } from './stt-offline-audio-chunker' +import { OFFLINE_DECODE_CHUNK_SECONDS, OfflineAudioChunker } from './stt-offline-audio-chunker' import { VadPauseSegmenter } from './stt-vad-pause-segmenter' import type { SttRecognizer, SttRecognizerInit, SttRecognizerPost } from './stt-recognizer' @@ -44,8 +45,12 @@ type StreamCoverage = { inputSamples: number resampledSamples: number reportedSamples: number + /** Stream position the last phrase decode reached, for its speed check. */ + decodedSamples: number slowPhraseDecodes: number speechPending: boolean + /** Phrases since the last commit and their live text. */ + span: { audio: Float32Array[]; samples: number; texts: string[] } } // Whisper pads every decode to a 30 s window, so each phrase costs a full decode. Where that takes longer than the @@ -56,6 +61,9 @@ const SLOW_PHRASE_DECODES_BEFORE_NO_CUTS = 2 // Dropped silence still counts as covered; report it so the host's replay buffer and stall watch don't grow through long pauses. const SILENCE_COVERAGE_REPORT_SECONDS = 5 +// The CPU fallback keeps uncommitted audio for a replay and takes over 30 s of it for a stalled GPU. +const COMMIT_AFTER_SECONDS = 20 + // Optional: without it dictation still works, with text only at the decode bound and at stop. async function loadSileroVad( transcribe: typeof TranscribeCpp, @@ -128,8 +136,10 @@ export async function createTranscribeCppRecognizer( inputSamples: 0, resampledSamples: 0, reportedSamples: 0, + decodedSamples: 0, slowPhraseDecodes: 0, - speechPending: false + speechPending: false, + span: { audio: [], samples: 0, texts: [] } } if (vad && vadModel) { stream.segmenter = new VadPauseSegmenter( @@ -162,9 +172,13 @@ export async function createTranscribeCppRecognizer( // The first run compiles GPU pipelines (seconds on Metal); pay it while loading, not on the first stop. await transcribeChunk(new Float32Array(Math.round(modelRate / 2)), baseOptions) + // Stream position (model-rate samples) the segmenter has handed on, less chunks still waiting to be decoded. + const position = (stream: StreamCoverage, undecoded: number) => + stream.resampledSamples - stream.segmenter.heldSamples - undecoded + // Coverage is in input-rate samples so the host can trim the audio it keeps for a CPU replay. const report = (streamId: string, stream: StreamCoverage, text: string, undecoded: number) => { - const covered = stream.resampledSamples - stream.segmenter.heldSamples - undecoded + const covered = position(stream, undecoded) stream.reportedSamples = covered post({ type: 'chunk', @@ -177,28 +191,59 @@ export async function createTranscribeCppRecognizer( // Canceled streams, until their queued discard runs; also covers a finish that already closed its stream. const canceled = new Set() - const decode = async (streamId: string, stream: StreamCoverage, chunks: Float32Array[]) => { + const spanLimit = OFFLINE_DECODE_CHUNK_SECONDS * modelRate + + // The span's phrases decode again together, so a pause mid-sentence doesn't capitalize the words after it. + const commit = async (streamId: string, stream: StreamCoverage, undecoded: number) => { + const { audio, texts } = stream.span + if (audio.length === 0 || canceled.has(streamId)) { + return + } + stream.span = { audio: [], samples: 0, texts: [] } + const text = + audio.length === 1 && texts.length === 1 + ? texts[0] + : (await transcribeChunk(joinAudio(audio), stream.options)).trim() + report(streamId, stream, text, undecoded) + } + + /** `live`: decode each phrase as it ends for live text; at stop only the commit decodes. */ + const decode = async ( + streamId: string, + stream: StreamCoverage, + chunks: Float32Array[], + live: boolean + ) => { let undecoded = chunks.reduce((total, chunk) => total + chunk.length, 0) for (const chunk of chunks) { if (canceled.has(streamId)) { return } - const startedAt = performance.now() - const text = await transcribeChunk(chunk, stream.options) - const decodeMs = performance.now() - startedAt - const spokenSince = stream.reportedSamples - undecoded -= chunk.length - report(streamId, stream, text.trim(), undecoded) - if (stream.vad) { - noteDecodeSpeed( - stream, - decodeMs, - ((stream.reportedSamples - spokenSince) / modelRate) * 1000 - ) + if (stream.span.samples + chunk.length > spanLimit) { + await commit(streamId, stream, undecoded) } + undecoded -= chunk.length + stream.span.audio.push(chunk) + stream.span.samples += chunk.length + if (!live) { + continue + } + const startedAt = performance.now() + const text = (await transcribeChunk(chunk, stream.options)).trim() + const decodeMs = performance.now() - startedAt + stream.span.texts.push(text) + post({ type: 'partial', streamId, text: stream.span.texts.reduce(appendDictationText, '') }) + const reached = position(stream, undecoded) + if (stream.vad) { + noteDecodeSpeed(stream, decodeMs, ((reached - stream.decodedSamples) / modelRate) * 1000) + } + stream.decodedSamples = reached } - const dropped = stream.resampledSamples - stream.segmenter.heldSamples - stream.reportedSamples - if (dropped >= SILENCE_COVERAGE_REPORT_SECONDS * modelRate) { + if (stream.resampledSamples - stream.reportedSamples >= COMMIT_AFTER_SECONDS * modelRate) { + await commit(streamId, stream, 0) + } + const dropped = position(stream, 0) - stream.reportedSamples + if (stream.span.samples === 0 && dropped >= SILENCE_COVERAGE_REPORT_SECONDS * modelRate) { report(streamId, stream, '', 0) } } @@ -244,7 +289,7 @@ export async function createTranscribeCppRecognizer( const phrases = await stream.segmenter.push(resampled) // A cut phrase stays pending until its text is posted. notePendingSpeech(streamId, stream, phrases.length > 0 || !!stream.segmenter.phraseOpen) - await decode(streamId, stream, phrases) + await decode(streamId, stream, phrases, true) notePendingSpeech(streamId, stream, !!stream.segmenter.phraseOpen) }, async finish(streamId) { @@ -253,7 +298,8 @@ export async function createTranscribeCppRecognizer( return } try { - await decode(streamId, stream, await stream.segmenter.flush()) + await decode(streamId, stream, await stream.segmenter.flush(), false) + await commit(streamId, stream, 0) notePendingSpeech(streamId, stream, false) } finally { stream.vad?.dispose() @@ -268,3 +314,16 @@ export async function createTranscribeCppRecognizer( } } } + +function joinAudio(parts: Float32Array[]): Float32Array { + if (parts.length === 1) { + return parts[0] + } + const joined = new Float32Array(parts.reduce((total, part) => total + part.length, 0)) + let offset = 0 + for (const part of parts) { + joined.set(part, offset) + offset += part.length + } + return joined +} diff --git a/src/main/speech/stt-vad-pause-segmenter.ts b/src/main/speech/stt-vad-pause-segmenter.ts index 8e7fc3b3d33..04af6067957 100644 --- a/src/main/speech/stt-vad-pause-segmenter.ts +++ b/src/main/speech/stt-vad-pause-segmenter.ts @@ -4,7 +4,7 @@ import { OfflineAudioChunker } from './stt-offline-audio-chunker' // Silero's own defaults: a frame at or above this is speech; inside speech, one below the second is silence. const SPEECH_PROBABILITY = 0.5 const SILENCE_PROBABILITY = 0.35 -// A pause this long commits the phrase before it. Shorter cuts split phrases, and each piece gets its own capital and full stop. +// A pause this long ends the phrase before it, which then decodes for live text; every cut costs a decode. export const VAD_PAUSE_SECONDS = 0.6 // Silence kept ahead of detected speech so a soft first syllable isn't clipped. const SPEECH_PREROLL_SECONDS = 0.4