mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
fix(speech): commit dictation once per span, not at every pause
Each pause cut was decoded and committed on its own, so Parakeet capitalized the word after a mid-sentence pause. Pause cuts now only update the live text.
This commit is contained in:
@@ -6,14 +6,16 @@ export type SttRecognizerInit = {
|
||||
sampleRate: number
|
||||
/** Absent = auto-detect. */
|
||||
language?: string
|
||||
/** Silero VAD GGUF; when present, offline models commit text at each pause. */
|
||||
/** Silero VAD GGUF; when present, offline models show live text at each pause. */
|
||||
vadModelPath?: string
|
||||
}
|
||||
|
||||
export type SttRecognizerEvent =
|
||||
/** One decoded offline chunk; `samples` counts the stream's input-rate samples decoded so far. */
|
||||
/** One committed offline chunk; `samples` counts the stream's input-rate samples committed so far. */
|
||||
| { type: 'chunk'; streamId: string; text: string; samples: number }
|
||||
/** Pause-cut streams only: speech was heard that no chunk has committed yet. */
|
||||
/** The uncommitted phrases' live text so far; the next chunk replaces it. */
|
||||
| { type: 'partial'; streamId: string; text: string }
|
||||
/** Pause-cut streams only: speech was heard whose text hasn't arrived yet. */
|
||||
| { type: 'speech-pending'; streamId: string; pending: boolean }
|
||||
/** A load step finished, so a slow load is told apart from a hung GPU. */
|
||||
| { type: 'loading' }
|
||||
|
||||
@@ -24,7 +24,7 @@ export type SttEvent =
|
||||
| { type: 'ready' }
|
||||
| { type: 'partial'; text?: string }
|
||||
| { type: 'final'; text?: string }
|
||||
/** Speech was heard that no final has committed yet; only from pause-cut on-device models. */
|
||||
/** Speech was heard whose text hasn't arrived yet; only from pause-cut on-device models. */
|
||||
| { type: 'speech-pending'; pending: boolean }
|
||||
| { type: 'stopped' }
|
||||
| { type: 'error'; error?: string }
|
||||
|
||||
@@ -2,8 +2,9 @@ import { availableParallelism } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import type * as TranscribeCpp from 'transcribe-cpp'
|
||||
import { appendDictationText } from '../../shared/dictation-transcript-spacing'
|
||||
import { resampleToRate } from './stt-audio-resample'
|
||||
import { OfflineAudioChunker } from './stt-offline-audio-chunker'
|
||||
import { OFFLINE_DECODE_CHUNK_SECONDS, OfflineAudioChunker } from './stt-offline-audio-chunker'
|
||||
import { VadPauseSegmenter } from './stt-vad-pause-segmenter'
|
||||
import type { SttRecognizer, SttRecognizerInit, SttRecognizerPost } from './stt-recognizer'
|
||||
|
||||
@@ -44,8 +45,12 @@ type StreamCoverage = {
|
||||
inputSamples: number
|
||||
resampledSamples: number
|
||||
reportedSamples: number
|
||||
/** Stream position the last phrase decode reached, for its speed check. */
|
||||
decodedSamples: number
|
||||
slowPhraseDecodes: number
|
||||
speechPending: boolean
|
||||
/** Phrases since the last commit and their live text. */
|
||||
span: { audio: Float32Array[]; samples: number; texts: string[] }
|
||||
}
|
||||
|
||||
// Whisper pads every decode to a 30 s window, so each phrase costs a full decode. Where that takes longer than the
|
||||
@@ -56,6 +61,9 @@ const SLOW_PHRASE_DECODES_BEFORE_NO_CUTS = 2
|
||||
// Dropped silence still counts as covered; report it so the host's replay buffer and stall watch don't grow through long pauses.
|
||||
const SILENCE_COVERAGE_REPORT_SECONDS = 5
|
||||
|
||||
// The CPU fallback keeps uncommitted audio for a replay and takes over 30 s of it for a stalled GPU.
|
||||
const COMMIT_AFTER_SECONDS = 20
|
||||
|
||||
// Optional: without it dictation still works, with text only at the decode bound and at stop.
|
||||
async function loadSileroVad(
|
||||
transcribe: typeof TranscribeCpp,
|
||||
@@ -128,8 +136,10 @@ export async function createTranscribeCppRecognizer(
|
||||
inputSamples: 0,
|
||||
resampledSamples: 0,
|
||||
reportedSamples: 0,
|
||||
decodedSamples: 0,
|
||||
slowPhraseDecodes: 0,
|
||||
speechPending: false
|
||||
speechPending: false,
|
||||
span: { audio: [], samples: 0, texts: [] }
|
||||
}
|
||||
if (vad && vadModel) {
|
||||
stream.segmenter = new VadPauseSegmenter(
|
||||
@@ -162,9 +172,13 @@ export async function createTranscribeCppRecognizer(
|
||||
// The first run compiles GPU pipelines (seconds on Metal); pay it while loading, not on the first stop.
|
||||
await transcribeChunk(new Float32Array(Math.round(modelRate / 2)), baseOptions)
|
||||
|
||||
// Stream position (model-rate samples) the segmenter has handed on, less chunks still waiting to be decoded.
|
||||
const position = (stream: StreamCoverage, undecoded: number) =>
|
||||
stream.resampledSamples - stream.segmenter.heldSamples - undecoded
|
||||
|
||||
// Coverage is in input-rate samples so the host can trim the audio it keeps for a CPU replay.
|
||||
const report = (streamId: string, stream: StreamCoverage, text: string, undecoded: number) => {
|
||||
const covered = stream.resampledSamples - stream.segmenter.heldSamples - undecoded
|
||||
const covered = position(stream, undecoded)
|
||||
stream.reportedSamples = covered
|
||||
post({
|
||||
type: 'chunk',
|
||||
@@ -177,28 +191,59 @@ export async function createTranscribeCppRecognizer(
|
||||
// Canceled streams, until their queued discard runs; also covers a finish that already closed its stream.
|
||||
const canceled = new Set<string>()
|
||||
|
||||
const decode = async (streamId: string, stream: StreamCoverage, chunks: Float32Array[]) => {
|
||||
const spanLimit = OFFLINE_DECODE_CHUNK_SECONDS * modelRate
|
||||
|
||||
// The span's phrases decode again together, so a pause mid-sentence doesn't capitalize the words after it.
|
||||
const commit = async (streamId: string, stream: StreamCoverage, undecoded: number) => {
|
||||
const { audio, texts } = stream.span
|
||||
if (audio.length === 0 || canceled.has(streamId)) {
|
||||
return
|
||||
}
|
||||
stream.span = { audio: [], samples: 0, texts: [] }
|
||||
const text =
|
||||
audio.length === 1 && texts.length === 1
|
||||
? texts[0]
|
||||
: (await transcribeChunk(joinAudio(audio), stream.options)).trim()
|
||||
report(streamId, stream, text, undecoded)
|
||||
}
|
||||
|
||||
/** `live`: decode each phrase as it ends for live text; at stop only the commit decodes. */
|
||||
const decode = async (
|
||||
streamId: string,
|
||||
stream: StreamCoverage,
|
||||
chunks: Float32Array[],
|
||||
live: boolean
|
||||
) => {
|
||||
let undecoded = chunks.reduce((total, chunk) => total + chunk.length, 0)
|
||||
for (const chunk of chunks) {
|
||||
if (canceled.has(streamId)) {
|
||||
return
|
||||
}
|
||||
const startedAt = performance.now()
|
||||
const text = await transcribeChunk(chunk, stream.options)
|
||||
const decodeMs = performance.now() - startedAt
|
||||
const spokenSince = stream.reportedSamples
|
||||
undecoded -= chunk.length
|
||||
report(streamId, stream, text.trim(), undecoded)
|
||||
if (stream.vad) {
|
||||
noteDecodeSpeed(
|
||||
stream,
|
||||
decodeMs,
|
||||
((stream.reportedSamples - spokenSince) / modelRate) * 1000
|
||||
)
|
||||
if (stream.span.samples + chunk.length > spanLimit) {
|
||||
await commit(streamId, stream, undecoded)
|
||||
}
|
||||
undecoded -= chunk.length
|
||||
stream.span.audio.push(chunk)
|
||||
stream.span.samples += chunk.length
|
||||
if (!live) {
|
||||
continue
|
||||
}
|
||||
const startedAt = performance.now()
|
||||
const text = (await transcribeChunk(chunk, stream.options)).trim()
|
||||
const decodeMs = performance.now() - startedAt
|
||||
stream.span.texts.push(text)
|
||||
post({ type: 'partial', streamId, text: stream.span.texts.reduce(appendDictationText, '') })
|
||||
const reached = position(stream, undecoded)
|
||||
if (stream.vad) {
|
||||
noteDecodeSpeed(stream, decodeMs, ((reached - stream.decodedSamples) / modelRate) * 1000)
|
||||
}
|
||||
stream.decodedSamples = reached
|
||||
}
|
||||
const dropped = stream.resampledSamples - stream.segmenter.heldSamples - stream.reportedSamples
|
||||
if (dropped >= SILENCE_COVERAGE_REPORT_SECONDS * modelRate) {
|
||||
if (stream.resampledSamples - stream.reportedSamples >= COMMIT_AFTER_SECONDS * modelRate) {
|
||||
await commit(streamId, stream, 0)
|
||||
}
|
||||
const dropped = position(stream, 0) - stream.reportedSamples
|
||||
if (stream.span.samples === 0 && dropped >= SILENCE_COVERAGE_REPORT_SECONDS * modelRate) {
|
||||
report(streamId, stream, '', 0)
|
||||
}
|
||||
}
|
||||
@@ -244,7 +289,7 @@ export async function createTranscribeCppRecognizer(
|
||||
const phrases = await stream.segmenter.push(resampled)
|
||||
// A cut phrase stays pending until its text is posted.
|
||||
notePendingSpeech(streamId, stream, phrases.length > 0 || !!stream.segmenter.phraseOpen)
|
||||
await decode(streamId, stream, phrases)
|
||||
await decode(streamId, stream, phrases, true)
|
||||
notePendingSpeech(streamId, stream, !!stream.segmenter.phraseOpen)
|
||||
},
|
||||
async finish(streamId) {
|
||||
@@ -253,7 +298,8 @@ export async function createTranscribeCppRecognizer(
|
||||
return
|
||||
}
|
||||
try {
|
||||
await decode(streamId, stream, await stream.segmenter.flush())
|
||||
await decode(streamId, stream, await stream.segmenter.flush(), false)
|
||||
await commit(streamId, stream, 0)
|
||||
notePendingSpeech(streamId, stream, false)
|
||||
} finally {
|
||||
stream.vad?.dispose()
|
||||
@@ -268,3 +314,16 @@ export async function createTranscribeCppRecognizer(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function joinAudio(parts: Float32Array[]): Float32Array {
|
||||
if (parts.length === 1) {
|
||||
return parts[0]
|
||||
}
|
||||
const joined = new Float32Array(parts.reduce((total, part) => total + part.length, 0))
|
||||
let offset = 0
|
||||
for (const part of parts) {
|
||||
joined.set(part, offset)
|
||||
offset += part.length
|
||||
}
|
||||
return joined
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import { OfflineAudioChunker } from './stt-offline-audio-chunker'
|
||||
// Silero's own defaults: a frame at or above this is speech; inside speech, one below the second is silence.
|
||||
const SPEECH_PROBABILITY = 0.5
|
||||
const SILENCE_PROBABILITY = 0.35
|
||||
// A pause this long commits the phrase before it. Shorter cuts split phrases, and each piece gets its own capital and full stop.
|
||||
// A pause this long ends the phrase before it, which then decodes for live text; every cut costs a decode.
|
||||
export const VAD_PAUSE_SECONDS = 0.6
|
||||
// Silence kept ahead of detected speech so a soft first syllable isn't clipped.
|
||||
const SPEECH_PREROLL_SECONDS = 0.4
|
||||
|
||||
Reference in New Issue
Block a user