feat(dictation): resample desktop dictation to 16 kHz in an AudioWorklet with an anti-alias filter (#39290)

* feat(dictation): capture desktop audio in an AudioWorklet at 16 kHz

Desktop dictation captured with a ScriptProcessorNode on the renderer's
main thread and shipped hardware-rate buffers, which main then decimated
to 16 kHz with linear interpolation and no low-pass, folding everything
above 8 kHz back into the speech band.

An AudioWorklet now resamples to 16 kHz with a windowed-sinc low-pass,
posts 100 ms frames, and sends a level every 50 ms for the meter and
silence auto-stop. Desktop and the web client share it.

* refactor(dictation): share the sinc resampler and meter window, one frame sink

Move the resampler to src/shared beside the meter it feeds, share the
50 ms meter window with the phone's, drop the constant per-chunk sample
rate from buffered audio, and send frames through one deliverFrame so
the speech-host port can replace it later.
This commit is contained in:
Kelvin Amoaba
2026-11-07 05:07:31 +00:00
committed by GitHub
parent df38124cce
commit f736cab7d7
8 changed files with 329 additions and 55 deletions
+15 -3
View File
@@ -20,6 +20,9 @@ export const DEFAULT_DICTATION_METER: DictationMeterState = {
isClipping: false
}
// Desktop and phone both feed the analyzer one window this long; its smoothing is tuned to it.
export const DICTATION_METER_WINDOW_MS = 50
const CLIPPING_THRESHOLD = 0.98
const CLIPPING_HOLD_MS = 500
@@ -39,10 +42,12 @@ export function createDictationMeterAnalyzerState(): DictationMeterAnalyzerState
}
}
function measureDictationAudioChunk(samples: Float32Array): {
export type DictationAudioLevel = {
rms: number
peak: number
} {
}
export function measureDictationAudioChunk(samples: Float32Array): DictationAudioLevel {
if (samples.length === 0) {
return { rms: 0, peak: 0 }
}
@@ -65,7 +70,14 @@ export function analyzeDictationAudioChunk(
now: number,
previous: DictationMeterAnalyzerState
): DictationMeterAnalyzerState {
const { rms, peak } = measureDictationAudioChunk(samples)
return analyzeDictationAudioLevel(measureDictationAudioChunk(samples), now, previous)
}
export function analyzeDictationAudioLevel(
{ rms, peak }: DictationAudioLevel,
now: number,
previous: DictationMeterAnalyzerState
): DictationMeterAnalyzerState {
const noiseFloor = Math.max(
0.004,
previous.noiseFloor * 0.96 + Math.min(rms, previous.noiseFloor * 2) * 0.04
@@ -0,0 +1,52 @@
import { describe, expect, it } from 'vitest'
import { StreamingSincResampler } from './streaming-sinc-resampler'
function tone(frequency: number, sampleRate: number, length: number): Float32Array {
return Float32Array.from({ length }, (_, index) =>
Math.sin((2 * Math.PI * frequency * index) / sampleRate)
)
}
function amplitudeAt(samples: Float32Array, frequency: number, sampleRate: number): number {
let real = 0
let imaginary = 0
for (let index = 0; index < samples.length; index += 1) {
const phase = (2 * Math.PI * frequency * index) / sampleRate
real += samples[index] * Math.cos(phase)
imaginary += samples[index] * Math.sin(phase)
}
return (2 * Math.hypot(real, imaginary)) / samples.length
}
function resampleInBlocks(input: Float32Array, inputRate: number, blockSize: number): Float32Array {
const resampler = new StreamingSincResampler(inputRate, 16000)
const output: number[] = []
for (let offset = 0; offset < input.length; offset += blockSize) {
output.push(...resampler.process(input.subarray(offset, offset + blockSize)))
}
return Float32Array.from(output)
}
describe('StreamingSincResampler', () => {
it.each([48000, 44100])('keeps speech and filters what would alias from %i Hz', (rate) => {
// Skip the filter's start-up so only steady-state output is measured.
const speech = resampleInBlocks(tone(1000, rate, rate), rate, 128).subarray(1000, 15000)
const aboveNyquist = resampleInBlocks(tone(10000, rate, rate), rate, 128).subarray(1000, 15000)
expect(amplitudeAt(speech, 1000, 16000)).toBeCloseTo(1, 2)
// A 10 kHz tone folds onto 6 kHz at 16 kHz; plain decimation leaves it at full amplitude.
expect(amplitudeAt(aboveNyquist, 6000, 16000)).toBeLessThan(0.001)
})
it('produces the same stream whatever block size the audio thread delivers', () => {
const input = tone(440, 44100, 44100)
const small = resampleInBlocks(input, 44100, 128)
const large = resampleInBlocks(input, 44100, 4096)
expect(small.length).toBe(large.length)
expect(Math.abs(small.length - 16000)).toBeLessThan(40)
for (let index = 0; index < small.length; index += 1) {
expect(small[index]).toBeCloseTo(large[index], 6)
}
})
})
+101
View File
@@ -0,0 +1,101 @@
// Kernel zero crossings per side; 16 with a Blackman window keeps aliases below about -70 dB.
const ZERO_CROSSINGS = 16
const TABLE_STEPS_PER_CROSSING = 256
// Cutoff as a share of the lower Nyquist, leaving room for the filter's transition band.
const ROLLOFF = 0.9
function buildKernelTable(): Float32Array {
const length = ZERO_CROSSINGS * TABLE_STEPS_PER_CROSSING
// One trailing zero lets the interpolation read past the last tap.
const table = new Float32Array(length + 2)
table[0] = 1
for (let index = 1; index <= length; index += 1) {
const x = index / TABLE_STEPS_PER_CROSSING
const r = index / length
const window = 0.42 + 0.5 * Math.cos(Math.PI * r) + 0.08 * Math.cos(2 * Math.PI * r)
table[index] = (Math.sin(Math.PI * x) / (Math.PI * x)) * window
}
return table
}
const KERNEL_TABLE = buildKernelTable()
/** Windowed-sinc resampler for a continuous stream fed in arbitrary-sized blocks. */
export class StreamingSincResampler {
private readonly passthrough: boolean
private readonly step: number
private readonly scale: number
private readonly halfWidth: number
private history: Float32Array
private historyLength: number
// Input index (into history) of the next output sample.
private position: number
private output = new Float32Array(0)
constructor(inputRate: number, outputRate: number) {
this.passthrough = inputRate === outputRate
this.step = inputRate / outputRate
this.scale = Math.min(1, outputRate / inputRate) * ROLLOFF
this.halfWidth = ZERO_CROSSINGS / this.scale
// Leading zeros put the first output on the first input sample.
this.historyLength = Math.ceil(this.halfWidth)
this.position = this.historyLength
this.history = new Float32Array(this.historyLength * 2 + 256)
}
/** Returns a view that the next call reuses: copy what you keep. */
process(input: Float32Array): Float32Array {
if (this.passthrough) {
return input
}
this.append(input)
const maxOutput = Math.ceil(this.historyLength / this.step) + 1
if (this.output.length < maxOutput) {
this.output = new Float32Array(maxOutput)
}
const { history, scale, halfWidth } = this
const table = KERNEL_TABLE
const tableScale = scale * TABLE_STEPS_PER_CROSSING
const lastIndex = this.historyLength - 1
let count = 0
while (this.position + halfWidth <= lastIndex) {
const center = this.position
const first = Math.ceil(center - halfWidth)
const last = Math.floor(center + halfWidth)
let sum = 0
for (let index = first; index <= last; index += 1) {
const tablePosition = Math.abs(center - index) * tableScale
const tableIndex = Math.floor(tablePosition)
const fraction = tablePosition - tableIndex
const tap = table[tableIndex] + fraction * (table[tableIndex + 1] - table[tableIndex])
sum += history[index] * tap
}
this.output[count] = sum * scale
count += 1
this.position += this.step
}
this.discardConsumed()
return this.output.subarray(0, count)
}
private append(input: Float32Array): void {
const needed = this.historyLength + input.length
if (needed > this.history.length) {
const grown = new Float32Array(needed * 2)
grown.set(this.history.subarray(0, this.historyLength))
this.history = grown
}
this.history.set(input, this.historyLength)
this.historyLength = needed
}
private discardConsumed(): void {
const consumed = Math.floor(this.position - this.halfWidth)
if (consumed <= 0) {
return
}
this.history.copyWithin(0, consumed, this.historyLength)
this.historyLength -= consumed
this.position -= consumed
}
}