mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
feat(dictation): resample desktop dictation to 16 kHz in an AudioWorklet with an anti-alias filter (#39290)
* feat(dictation): capture desktop audio in an AudioWorklet at 16 kHz Desktop dictation captured with a ScriptProcessorNode on the renderer's main thread and shipped hardware-rate buffers, which main then decimated to 16 kHz with linear interpolation and no low-pass, folding everything above 8 kHz back into the speech band. An AudioWorklet now resamples to 16 kHz with a windowed-sinc low-pass, posts 100 ms frames, and sends a level every 50 ms for the meter and silence auto-stop. Desktop and the web client share it. * refactor(dictation): share the sinc resampler and meter window, one frame sink Move the resampler to src/shared beside the meter it feeds, share the 50 ms meter window with the phone's, drop the constant per-chunk sample rate from buffered audio, and send frames through one deliverFrame so the speech-host port can replace it later.
This commit is contained in:
@@ -20,6 +20,9 @@ export const DEFAULT_DICTATION_METER: DictationMeterState = {
|
||||
isClipping: false
|
||||
}
|
||||
|
||||
// Desktop and phone both feed the analyzer one window this long; its smoothing is tuned to it.
|
||||
export const DICTATION_METER_WINDOW_MS = 50
|
||||
|
||||
const CLIPPING_THRESHOLD = 0.98
|
||||
const CLIPPING_HOLD_MS = 500
|
||||
|
||||
@@ -39,10 +42,12 @@ export function createDictationMeterAnalyzerState(): DictationMeterAnalyzerState
|
||||
}
|
||||
}
|
||||
|
||||
function measureDictationAudioChunk(samples: Float32Array): {
|
||||
export type DictationAudioLevel = {
|
||||
rms: number
|
||||
peak: number
|
||||
} {
|
||||
}
|
||||
|
||||
export function measureDictationAudioChunk(samples: Float32Array): DictationAudioLevel {
|
||||
if (samples.length === 0) {
|
||||
return { rms: 0, peak: 0 }
|
||||
}
|
||||
@@ -65,7 +70,14 @@ export function analyzeDictationAudioChunk(
|
||||
now: number,
|
||||
previous: DictationMeterAnalyzerState
|
||||
): DictationMeterAnalyzerState {
|
||||
const { rms, peak } = measureDictationAudioChunk(samples)
|
||||
return analyzeDictationAudioLevel(measureDictationAudioChunk(samples), now, previous)
|
||||
}
|
||||
|
||||
export function analyzeDictationAudioLevel(
|
||||
{ rms, peak }: DictationAudioLevel,
|
||||
now: number,
|
||||
previous: DictationMeterAnalyzerState
|
||||
): DictationMeterAnalyzerState {
|
||||
const noiseFloor = Math.max(
|
||||
0.004,
|
||||
previous.noiseFloor * 0.96 + Math.min(rms, previous.noiseFloor * 2) * 0.04
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { StreamingSincResampler } from './streaming-sinc-resampler'
|
||||
|
||||
function tone(frequency: number, sampleRate: number, length: number): Float32Array {
|
||||
return Float32Array.from({ length }, (_, index) =>
|
||||
Math.sin((2 * Math.PI * frequency * index) / sampleRate)
|
||||
)
|
||||
}
|
||||
|
||||
function amplitudeAt(samples: Float32Array, frequency: number, sampleRate: number): number {
|
||||
let real = 0
|
||||
let imaginary = 0
|
||||
for (let index = 0; index < samples.length; index += 1) {
|
||||
const phase = (2 * Math.PI * frequency * index) / sampleRate
|
||||
real += samples[index] * Math.cos(phase)
|
||||
imaginary += samples[index] * Math.sin(phase)
|
||||
}
|
||||
return (2 * Math.hypot(real, imaginary)) / samples.length
|
||||
}
|
||||
|
||||
function resampleInBlocks(input: Float32Array, inputRate: number, blockSize: number): Float32Array {
|
||||
const resampler = new StreamingSincResampler(inputRate, 16000)
|
||||
const output: number[] = []
|
||||
for (let offset = 0; offset < input.length; offset += blockSize) {
|
||||
output.push(...resampler.process(input.subarray(offset, offset + blockSize)))
|
||||
}
|
||||
return Float32Array.from(output)
|
||||
}
|
||||
|
||||
describe('StreamingSincResampler', () => {
|
||||
it.each([48000, 44100])('keeps speech and filters what would alias from %i Hz', (rate) => {
|
||||
// Skip the filter's start-up so only steady-state output is measured.
|
||||
const speech = resampleInBlocks(tone(1000, rate, rate), rate, 128).subarray(1000, 15000)
|
||||
const aboveNyquist = resampleInBlocks(tone(10000, rate, rate), rate, 128).subarray(1000, 15000)
|
||||
|
||||
expect(amplitudeAt(speech, 1000, 16000)).toBeCloseTo(1, 2)
|
||||
// A 10 kHz tone folds onto 6 kHz at 16 kHz; plain decimation leaves it at full amplitude.
|
||||
expect(amplitudeAt(aboveNyquist, 6000, 16000)).toBeLessThan(0.001)
|
||||
})
|
||||
|
||||
it('produces the same stream whatever block size the audio thread delivers', () => {
|
||||
const input = tone(440, 44100, 44100)
|
||||
const small = resampleInBlocks(input, 44100, 128)
|
||||
const large = resampleInBlocks(input, 44100, 4096)
|
||||
|
||||
expect(small.length).toBe(large.length)
|
||||
expect(Math.abs(small.length - 16000)).toBeLessThan(40)
|
||||
for (let index = 0; index < small.length; index += 1) {
|
||||
expect(small[index]).toBeCloseTo(large[index], 6)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,101 @@
|
||||
// Kernel zero crossings per side; 16 with a Blackman window keeps aliases below about -70 dB.
|
||||
const ZERO_CROSSINGS = 16
|
||||
const TABLE_STEPS_PER_CROSSING = 256
|
||||
// Cutoff as a share of the lower Nyquist, leaving room for the filter's transition band.
|
||||
const ROLLOFF = 0.9
|
||||
|
||||
function buildKernelTable(): Float32Array {
|
||||
const length = ZERO_CROSSINGS * TABLE_STEPS_PER_CROSSING
|
||||
// One trailing zero lets the interpolation read past the last tap.
|
||||
const table = new Float32Array(length + 2)
|
||||
table[0] = 1
|
||||
for (let index = 1; index <= length; index += 1) {
|
||||
const x = index / TABLE_STEPS_PER_CROSSING
|
||||
const r = index / length
|
||||
const window = 0.42 + 0.5 * Math.cos(Math.PI * r) + 0.08 * Math.cos(2 * Math.PI * r)
|
||||
table[index] = (Math.sin(Math.PI * x) / (Math.PI * x)) * window
|
||||
}
|
||||
return table
|
||||
}
|
||||
|
||||
const KERNEL_TABLE = buildKernelTable()
|
||||
|
||||
/** Windowed-sinc resampler for a continuous stream fed in arbitrary-sized blocks. */
|
||||
export class StreamingSincResampler {
|
||||
private readonly passthrough: boolean
|
||||
private readonly step: number
|
||||
private readonly scale: number
|
||||
private readonly halfWidth: number
|
||||
private history: Float32Array
|
||||
private historyLength: number
|
||||
// Input index (into history) of the next output sample.
|
||||
private position: number
|
||||
private output = new Float32Array(0)
|
||||
|
||||
constructor(inputRate: number, outputRate: number) {
|
||||
this.passthrough = inputRate === outputRate
|
||||
this.step = inputRate / outputRate
|
||||
this.scale = Math.min(1, outputRate / inputRate) * ROLLOFF
|
||||
this.halfWidth = ZERO_CROSSINGS / this.scale
|
||||
// Leading zeros put the first output on the first input sample.
|
||||
this.historyLength = Math.ceil(this.halfWidth)
|
||||
this.position = this.historyLength
|
||||
this.history = new Float32Array(this.historyLength * 2 + 256)
|
||||
}
|
||||
|
||||
/** Returns a view that the next call reuses: copy what you keep. */
|
||||
process(input: Float32Array): Float32Array {
|
||||
if (this.passthrough) {
|
||||
return input
|
||||
}
|
||||
this.append(input)
|
||||
const maxOutput = Math.ceil(this.historyLength / this.step) + 1
|
||||
if (this.output.length < maxOutput) {
|
||||
this.output = new Float32Array(maxOutput)
|
||||
}
|
||||
const { history, scale, halfWidth } = this
|
||||
const table = KERNEL_TABLE
|
||||
const tableScale = scale * TABLE_STEPS_PER_CROSSING
|
||||
const lastIndex = this.historyLength - 1
|
||||
let count = 0
|
||||
while (this.position + halfWidth <= lastIndex) {
|
||||
const center = this.position
|
||||
const first = Math.ceil(center - halfWidth)
|
||||
const last = Math.floor(center + halfWidth)
|
||||
let sum = 0
|
||||
for (let index = first; index <= last; index += 1) {
|
||||
const tablePosition = Math.abs(center - index) * tableScale
|
||||
const tableIndex = Math.floor(tablePosition)
|
||||
const fraction = tablePosition - tableIndex
|
||||
const tap = table[tableIndex] + fraction * (table[tableIndex + 1] - table[tableIndex])
|
||||
sum += history[index] * tap
|
||||
}
|
||||
this.output[count] = sum * scale
|
||||
count += 1
|
||||
this.position += this.step
|
||||
}
|
||||
this.discardConsumed()
|
||||
return this.output.subarray(0, count)
|
||||
}
|
||||
|
||||
private append(input: Float32Array): void {
|
||||
const needed = this.historyLength + input.length
|
||||
if (needed > this.history.length) {
|
||||
const grown = new Float32Array(needed * 2)
|
||||
grown.set(this.history.subarray(0, this.historyLength))
|
||||
this.history = grown
|
||||
}
|
||||
this.history.set(input, this.historyLength)
|
||||
this.historyLength = needed
|
||||
}
|
||||
|
||||
private discardConsumed(): void {
|
||||
const consumed = Math.floor(this.position - this.halfWidth)
|
||||
if (consumed <= 0) {
|
||||
return
|
||||
}
|
||||
this.history.copyWithin(0, consumed, this.historyLength)
|
||||
this.historyLength -= consumed
|
||||
this.position -= consumed
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user