feat(dictation): resample desktop dictation to 16 kHz in an AudioWorklet with an anti-alias filter (#39290)

* feat(dictation): capture desktop audio in an AudioWorklet at 16 kHz

Desktop dictation captured with a ScriptProcessorNode on the renderer's
main thread and shipped hardware-rate buffers, which main then decimated
to 16 kHz with linear interpolation and no low-pass, folding everything
above 8 kHz back into the speech band.

An AudioWorklet now resamples to 16 kHz with a windowed-sinc low-pass,
posts 100 ms frames, and sends a level every 50 ms for the meter and
silence auto-stop. Desktop and the web client share it.

* refactor(dictation): share the sinc resampler and meter window, one frame sink

Move the resampler to src/shared beside the meter it feeds, share the
50 ms meter window with the phone's, drop the constant per-chunk sample
rate from buffered audio, and send frames through one deliverFrame so
the speech-host port can replace it later.

src/renderer/src/components/dictation/dictation-capture-worklet-protocol.ts

@@ -0,0 +1,12 @@
+// Kept apart from the worklet module, which calls registerProcessor and only loads in AudioWorkletGlobalScope.
+export const DICTATION_CAPTURE_WORKLET_NAME = 'orca-dictation-capture'
+
+export type DictationCaptureWorkletOptions = {
+  outputSampleRate: number
+  frameSamples: number
+  levelWindowMs: number
+}
+
+export type DictationCaptureWorkletMessage =
+  | { type: 'frame'; samples: Float32Array }
+  | { type: 'level'; rms: number; peak: number }

src/renderer/src/components/dictation/dictation-capture-worklet.ts

@@ -0,0 +1,84 @@
+// Runs in AudioWorkletGlobalScope, off the renderer's main thread.
+import { measureDictationAudioChunk } from '../../../../shared/dictation-audio-meter'
+import { StreamingSincResampler } from '../../../../shared/streaming-sinc-resampler'
+import {
+  DICTATION_CAPTURE_WORKLET_NAME,
+  type DictationCaptureWorkletMessage,
+  type DictationCaptureWorkletOptions
+} from './dictation-capture-worklet-protocol'
+
+declare const sampleRate: number
+declare class AudioWorkletProcessor {
+  readonly port: MessagePort
+}
+declare function registerProcessor(
+  name: string,
+  processor: new (options: { processorOptions: DictationCaptureWorkletOptions }) => unknown
+): void
+
+class DictationCaptureProcessor extends AudioWorkletProcessor {
+  private readonly resampler: StreamingSincResampler
+  // The transfer detaches each sent frame, so its length can't be read back.
+  private readonly frameSamples: number
+  private frame: Float32Array
+  private frameLength = 0
+  // Measured at the device rate so clipping sees the samples before the low-pass.
+  private readonly levelWindowSamples: number
+  private levelSamples = 0
+  private levelSumSquares = 0
+  private levelPeak = 0
+
+  constructor({ processorOptions }: { processorOptions: DictationCaptureWorkletOptions }) {
+    super()
+    this.resampler = new StreamingSincResampler(sampleRate, processorOptions.outputSampleRate)
+    this.frameSamples = processorOptions.frameSamples
+    this.frame = new Float32Array(this.frameSamples)
+    this.levelWindowSamples = Math.round((sampleRate * processorOptions.levelWindowMs) / 1000)
+  }
+
+  process(inputs: Float32Array[][]): boolean {
+    const channel = inputs[0]?.[0]
+    if (channel) {
+      this.measure(channel)
+      this.appendToFrame(this.resampler.process(channel))
+    }
+    return true
+  }
+
+  private measure(block: Float32Array): void {
+    const { rms, peak } = measureDictationAudioChunk(block)
+    this.levelSamples += block.length
+    this.levelSumSquares += rms * rms * block.length
+    this.levelPeak = Math.max(this.levelPeak, peak)
+    if (this.levelSamples < this.levelWindowSamples) {
+      return
+    }
+    const message: DictationCaptureWorkletMessage = {
+      type: 'level',
+      rms: Math.sqrt(this.levelSumSquares / this.levelSamples),
+      peak: this.levelPeak
+    }
+    this.port.postMessage(message)
+    this.levelSamples = 0
+    this.levelSumSquares = 0
+    this.levelPeak = 0
+  }
+
+  private appendToFrame(samples: Float32Array): void {
+    let offset = 0
+    while (offset < samples.length) {
+      const count = Math.min(samples.length - offset, this.frameSamples - this.frameLength)
+      this.frame.set(samples.subarray(offset, offset + count), this.frameLength)
+      this.frameLength += count
+      offset += count
+      if (this.frameLength === this.frameSamples) {
+        const message: DictationCaptureWorkletMessage = { type: 'frame', samples: this.frame }
+        this.port.postMessage(message, [this.frame.buffer])
+        this.frame = new Float32Array(this.frameSamples)
+        this.frameLength = 0
+      }
+    }
+  }
+}
+
+registerProcessor(DICTATION_CAPTURE_WORKLET_NAME, DictationCaptureProcessor)

src/renderer/src/hooks/use-audio-capture-meter.ts

@@ -1,38 +1,38 @@
 import { useCallback, useRef } from 'react'
 import {
   DEFAULT_DICTATION_METER,
-  analyzeDictationAudioChunk,
+  analyzeDictationAudioLevel,
   createDictationMeterAnalyzerState,
   dictationMeterStatesEqual,
   toPublicDictationMeterState,
+  type DictationAudioLevel,
   type DictationMeterAnalyzerState,
   type DictationMeterState
 } from '../../../shared/dictation-audio-meter'
 
-const METER_PUBLISH_INTERVAL_MS = 1000 / 15
-
 export type DictationMeterPublisher = (meter: DictationMeterState) => void
 
 export function useAudioCaptureMeter(publishMeter?: DictationMeterPublisher) {
   const analyzerRef = useRef<DictationMeterAnalyzerState>(createDictationMeterAnalyzerState())
   const publishedMeterRef = useRef(DEFAULT_DICTATION_METER)
-  const lastPublishedAtRef = useRef(Number.NEGATIVE_INFINITY)
 
   const resetMeter = useCallback(() => {
     analyzerRef.current = createDictationMeterAnalyzerState()
     publishedMeterRef.current = DEFAULT_DICTATION_METER
-    lastPublishedAtRef.current = Number.NEGATIVE_INFINITY
     publishMeter?.(DEFAULT_DICTATION_METER)
   }, [publishMeter])
 
-  const measureChunk = useCallback(
-    (samples: Float32Array) => {
-      const now = performance.now()
-      analyzerRef.current = analyzeDictationAudioChunk(samples, now, analyzerRef.current)
-      if (document.hidden || now - lastPublishedAtRef.current < METER_PUBLISH_INTERVAL_MS) {
+  // The worklet sends one level per 50 ms window, which paces publishing.
+  const measureLevel = useCallback(
+    (level: DictationAudioLevel) => {
+      analyzerRef.current = analyzeDictationAudioLevel(
+        level,
+        performance.now(),
+        analyzerRef.current
+      )
+      if (document.hidden) {
         return
       }
-      lastPublishedAtRef.current = now
       const meter = toPublicDictationMeterState(analyzerRef.current)
       if (!dictationMeterStatesEqual(publishedMeterRef.current, meter)) {
         publishedMeterRef.current = meter
@@ -42,8 +42,8 @@ export function useAudioCaptureMeter(publishMeter?: DictationMeterPublisher) {
     [publishMeter]
   )
 
-  // On the performance.now() clock measureChunk feeds the analyzer with.
+  // On the performance.now() clock measureLevel feeds the analyzer with.
   const getLastSpeechAt = useCallback(() => analyzerRef.current.lastSpeechAt, [])
 
-  return { resetMeter, measureChunk, getLastSpeechAt }
+  return { resetMeter, measureLevel, getLastSpeechAt }
 }

src/renderer/src/hooks/use-audio-capture-stop-tail.ts

@@ -4,7 +4,7 @@ export type StopCaptureOptions = {
   preserveBufferedAudio?: boolean
 }
 
-// Stop lands on the last syllable while the processor still holds up to one unread 4096-frame buffer.
+// Stop lands on the last syllable while the capture worklet still holds up to one unsent 100 ms frame.
 const STOP_TAIL_MS = 250
 
 /** Keep capturing briefly after a user stop so the word being finished reaches the recognizer. */

src/renderer/src/hooks/use-audio-capture.ts

@@ -1,11 +1,18 @@
 import { useRef, useCallback } from 'react'
 import { openMicrophoneCaptureStream } from '@/components/dictation/microphone-devices'
+import captureWorkletUrl from '@/components/dictation/dictation-capture-worklet?worker&url'
+import {
+  DICTATION_CAPTURE_WORKLET_NAME,
+  type DictationCaptureWorkletMessage,
+  type DictationCaptureWorkletOptions
+} from '@/components/dictation/dictation-capture-worklet-protocol'
+import { DICTATION_SAMPLE_RATE } from '../../../shared/rpc-contract/speech-params'
+import { DICTATION_METER_WINDOW_MS } from '../../../shared/dictation-audio-meter'
 import { useAudioCaptureMeter, type DictationMeterPublisher } from './use-audio-capture-meter'
 import { useAudioCaptureStopTail, type StopCaptureOptions } from './use-audio-capture-stop-tail'
 
 type BufferedAudioChunk = {
   samples: Float32Array
-  sampleRate: number
   sessionId: string
 }
 
@@ -28,11 +35,16 @@ export type StartAudioCaptureResult = {
 
 const MAX_BUFFERED_AUDIO_SECONDS = 30
 const MAX_BUFFERED_AUDIO_BYTES = 8 * 1024 * 1024
+const CAPTURE_WORKLET_OPTIONS: DictationCaptureWorkletOptions = {
+  outputSampleRate: DICTATION_SAMPLE_RATE,
+  frameSamples: DICTATION_SAMPLE_RATE / 10,
+  levelWindowMs: DICTATION_METER_WINDOW_MS
+}
 
 export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
   const streamRef = useRef<MediaStream | null>(null)
   const contextRef = useRef<AudioContext | null>(null)
-  const processorRef = useRef<ScriptProcessorNode | null>(null)
+  const processorRef = useRef<AudioWorkletNode | null>(null)
   const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null)
   const isCapturingRef = useRef(false)
   const startRequestRef = useRef(0)
@@ -44,7 +56,7 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
   const capturedChunkCountRef = useRef(0)
   const sessionIdRef = useRef('desktop')
   const trackLostCleanupRef = useRef<(() => void) | null>(null)
-  const { resetMeter, measureChunk, getLastSpeechAt } = useAudioCaptureMeter(publishMeter)
+  const { resetMeter, measureLevel, getLastSpeechAt } = useAudioCaptureMeter(publishMeter)
 
   const cleanupCaptureResources = useCallback(() => {
     trackLostCleanupRef.current?.()
@@ -77,14 +89,14 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
       return
     }
     bufferedAudioBytesRef.current -= chunk.samples.byteLength
-    bufferedAudioSecondsRef.current -= chunk.samples.length / chunk.sampleRate
+    bufferedAudioSecondsRef.current -= chunk.samples.length / DICTATION_SAMPLE_RATE
   }, [])
 
   const appendBufferedAudioChunk = useCallback(
     (chunk: BufferedAudioChunk) => {
       bufferedAudioRef.current.push(chunk)
       bufferedAudioBytesRef.current += chunk.samples.byteLength
-      bufferedAudioSecondsRef.current += chunk.samples.length / chunk.sampleRate
+      bufferedAudioSecondsRef.current += chunk.samples.length / DICTATION_SAMPLE_RATE
 
       // Bound recent buffering while worker/model startup hangs to prevent unlimited renderer memory growth.
       while (
@@ -98,6 +110,21 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
     [removeOldestBufferedAudioChunk]
   )
 
+  // The one place a 16 kHz frame leaves capture: the startup buffer or the speech host.
+  const deliverFrame = useCallback(
+    (samples: Float32Array, onAudioLimitReached?: () => void) => {
+      if (bufferAudioRef.current) {
+        appendBufferedAudioChunk({ samples, sessionId: sessionIdRef.current })
+        return
+      }
+      void window.api.speech
+        .feedAudio(samples, DICTATION_SAMPLE_RATE, sessionIdRef.current)
+        .then((limitReached) => limitReached && onAudioLimitReached?.())
+        .catch(() => undefined)
+    },
+    [appendBufferedAudioChunk]
+  )
+
   const start = useCallback(
     async (
       options: StartAudioCaptureOptions = {}
@@ -130,16 +157,17 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
 
       let context: AudioContext | null = null
       let source: MediaStreamAudioSourceNode | null = null
-      let processor: ScriptProcessorNode | null = null
+      let processor: AudioWorkletNode | null = null
       try {
-        // Use hardware sample rate and let sherpa resample; forcing 16kHz can produce macOS microphone silence.
+        // Run at the hardware rate and resample in the worklet; forcing 16kHz can produce macOS microphone silence.
         context = new AudioContext()
         contextRef.current = context
 
         // Explicitly resume AudioContext because Chromium may suspend it pending a user gesture.
-        if (context.state === 'suspended') {
-          await context.resume()
-        }
+        await Promise.all([
+          context.state === 'suspended' ? context.resume() : undefined,
+          context.audioWorklet.addModule(captureWorkletUrl)
+        ])
         if (startRequestRef.current !== startRequest || streamRef.current !== stream) {
           if (contextRef.current === context) {
             contextRef.current = null
@@ -155,13 +183,13 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
         }
 
         source = context.createMediaStreamSource(stream)
-
-        // ScriptProcessorNode avoids a separate AudioWorklet module in the Vite build.
-        processor = context.createScriptProcessor(4096, 1, 1)
-
-        const actualRate = context.sampleRate
-
-        processor.onaudioprocess = (e: AudioProcessingEvent) => {
+        // Mono downmix as before; the worklet sends 16 kHz frames and a level per meter window.
+        processor = new AudioWorkletNode(context, DICTATION_CAPTURE_WORKLET_NAME, {
+          channelCount: 1,
+          channelCountMode: 'explicit',
+          processorOptions: CAPTURE_WORKLET_OPTIONS
+        })
+        processor.port.onmessage = ({ data }: MessageEvent<DictationCaptureWorkletMessage>) => {
           if (
             !isCapturingRef.current ||
             startRequestRef.current !== startRequest ||
@@ -169,21 +197,12 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
           ) {
             return
           }
-          const samples = new Float32Array(e.inputBuffer.getChannelData(0))
-          measureChunk(samples)
-          capturedChunkCountRef.current += 1
-          if (bufferAudioRef.current) {
-            appendBufferedAudioChunk({
-              samples,
-              sampleRate: actualRate,
-              sessionId: sessionIdRef.current
-            })
+          if (data.type === 'level') {
+            measureLevel(data)
             return
           }
-          void window.api.speech
-            .feedAudio(samples, actualRate, sessionIdRef.current)
-            .then((limitReached) => limitReached && options.onAudioLimitReached?.())
-            .catch(() => undefined)
+          capturedChunkCountRef.current += 1
+          deliverFrame(data.samples, options.onAudioLimitReached)
         }
 
         source.connect(processor)
@@ -238,13 +257,7 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
         throw err
       }
     },
-    [
-      appendBufferedAudioChunk,
-      cleanupCaptureResources,
-      measureChunk,
-      resetBufferedAudio,
-      resetMeter
-    ]
+    [cleanupCaptureResources, deliverFrame, measureLevel, resetBufferedAudio, resetMeter]
   )
 
   const flushBufferedAudio = useCallback(async () => {
@@ -260,7 +273,7 @@ export function useAudioCapture(publishMeter?: DictationMeterPublisher) {
           break
         }
         removeOldestBufferedAudioChunk()
-        await window.api.speech.feedAudio(chunk.samples, chunk.sampleRate, chunk.sessionId)
+        await window.api.speech.feedAudio(chunk.samples, DICTATION_SAMPLE_RATE, chunk.sessionId)
       }
     } finally {
       if (bufferedAudioGenerationRef.current === flushGeneration) {

src/shared/dictation-audio-meter.ts

@@ -20,6 +20,9 @@ export const DEFAULT_DICTATION_METER: DictationMeterState = {
   isClipping: false
 }
 
+// Desktop and phone both feed the analyzer one window this long; its smoothing is tuned to it.
+export const DICTATION_METER_WINDOW_MS = 50
+
 const CLIPPING_THRESHOLD = 0.98
 const CLIPPING_HOLD_MS = 500
 
@@ -39,10 +42,12 @@ export function createDictationMeterAnalyzerState(): DictationMeterAnalyzerState
   }
 }
 
-function measureDictationAudioChunk(samples: Float32Array): {
+export type DictationAudioLevel = {
   rms: number
   peak: number
-} {
+}
+
+export function measureDictationAudioChunk(samples: Float32Array): DictationAudioLevel {
   if (samples.length === 0) {
     return { rms: 0, peak: 0 }
   }
@@ -65,7 +70,14 @@ export function analyzeDictationAudioChunk(
   now: number,
   previous: DictationMeterAnalyzerState
 ): DictationMeterAnalyzerState {
-  const { rms, peak } = measureDictationAudioChunk(samples)
+  return analyzeDictationAudioLevel(measureDictationAudioChunk(samples), now, previous)
+}
+
+export function analyzeDictationAudioLevel(
+  { rms, peak }: DictationAudioLevel,
+  now: number,
+  previous: DictationMeterAnalyzerState
+): DictationMeterAnalyzerState {
   const noiseFloor = Math.max(
     0.004,
     previous.noiseFloor * 0.96 + Math.min(rms, previous.noiseFloor * 2) * 0.04

src/shared/streaming-sinc-resampler.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from 'vitest'
+import { StreamingSincResampler } from './streaming-sinc-resampler'
+
+function tone(frequency: number, sampleRate: number, length: number): Float32Array {
+  return Float32Array.from({ length }, (_, index) =>
+    Math.sin((2 * Math.PI * frequency * index) / sampleRate)
+  )
+}
+
+function amplitudeAt(samples: Float32Array, frequency: number, sampleRate: number): number {
+  let real = 0
+  let imaginary = 0
+  for (let index = 0; index < samples.length; index += 1) {
+    const phase = (2 * Math.PI * frequency * index) / sampleRate
+    real += samples[index] * Math.cos(phase)
+    imaginary += samples[index] * Math.sin(phase)
+  }
+  return (2 * Math.hypot(real, imaginary)) / samples.length
+}
+
+function resampleInBlocks(input: Float32Array, inputRate: number, blockSize: number): Float32Array {
+  const resampler = new StreamingSincResampler(inputRate, 16000)
+  const output: number[] = []
+  for (let offset = 0; offset < input.length; offset += blockSize) {
+    output.push(...resampler.process(input.subarray(offset, offset + blockSize)))
+  }
+  return Float32Array.from(output)
+}
+
+describe('StreamingSincResampler', () => {
+  it.each([48000, 44100])('keeps speech and filters what would alias from %i Hz', (rate) => {
+    // Skip the filter's start-up so only steady-state output is measured.
+    const speech = resampleInBlocks(tone(1000, rate, rate), rate, 128).subarray(1000, 15000)
+    const aboveNyquist = resampleInBlocks(tone(10000, rate, rate), rate, 128).subarray(1000, 15000)
+
+    expect(amplitudeAt(speech, 1000, 16000)).toBeCloseTo(1, 2)
+    // A 10 kHz tone folds onto 6 kHz at 16 kHz; plain decimation leaves it at full amplitude.
+    expect(amplitudeAt(aboveNyquist, 6000, 16000)).toBeLessThan(0.001)
+  })
+
+  it('produces the same stream whatever block size the audio thread delivers', () => {
+    const input = tone(440, 44100, 44100)
+    const small = resampleInBlocks(input, 44100, 128)
+    const large = resampleInBlocks(input, 44100, 4096)
+
+    expect(small.length).toBe(large.length)
+    expect(Math.abs(small.length - 16000)).toBeLessThan(40)
+    for (let index = 0; index < small.length; index += 1) {
+      expect(small[index]).toBeCloseTo(large[index], 6)
+    }
+  })
+})

src/shared/streaming-sinc-resampler.ts

@@ -0,0 +1,101 @@
+// Kernel zero crossings per side; 16 with a Blackman window keeps aliases below about -70 dB.
+const ZERO_CROSSINGS = 16
+const TABLE_STEPS_PER_CROSSING = 256
+// Cutoff as a share of the lower Nyquist, leaving room for the filter's transition band.
+const ROLLOFF = 0.9
+
+function buildKernelTable(): Float32Array {
+  const length = ZERO_CROSSINGS * TABLE_STEPS_PER_CROSSING
+  // One trailing zero lets the interpolation read past the last tap.
+  const table = new Float32Array(length + 2)
+  table[0] = 1
+  for (let index = 1; index <= length; index += 1) {
+    const x = index / TABLE_STEPS_PER_CROSSING
+    const r = index / length
+    const window = 0.42 + 0.5 * Math.cos(Math.PI * r) + 0.08 * Math.cos(2 * Math.PI * r)
+    table[index] = (Math.sin(Math.PI * x) / (Math.PI * x)) * window
+  }
+  return table
+}
+
+const KERNEL_TABLE = buildKernelTable()
+
+/** Windowed-sinc resampler for a continuous stream fed in arbitrary-sized blocks. */
+export class StreamingSincResampler {
+  private readonly passthrough: boolean
+  private readonly step: number
+  private readonly scale: number
+  private readonly halfWidth: number
+  private history: Float32Array
+  private historyLength: number
+  // Input index (into history) of the next output sample.
+  private position: number
+  private output = new Float32Array(0)
+
+  constructor(inputRate: number, outputRate: number) {
+    this.passthrough = inputRate === outputRate
+    this.step = inputRate / outputRate
+    this.scale = Math.min(1, outputRate / inputRate) * ROLLOFF
+    this.halfWidth = ZERO_CROSSINGS / this.scale
+    // Leading zeros put the first output on the first input sample.
+    this.historyLength = Math.ceil(this.halfWidth)
+    this.position = this.historyLength
+    this.history = new Float32Array(this.historyLength * 2 + 256)
+  }
+
+  /** Returns a view that the next call reuses: copy what you keep. */
+  process(input: Float32Array): Float32Array {
+    if (this.passthrough) {
+      return input
+    }
+    this.append(input)
+    const maxOutput = Math.ceil(this.historyLength / this.step) + 1
+    if (this.output.length < maxOutput) {
+      this.output = new Float32Array(maxOutput)
+    }
+    const { history, scale, halfWidth } = this
+    const table = KERNEL_TABLE
+    const tableScale = scale * TABLE_STEPS_PER_CROSSING
+    const lastIndex = this.historyLength - 1
+    let count = 0
+    while (this.position + halfWidth <= lastIndex) {
+      const center = this.position
+      const first = Math.ceil(center - halfWidth)
+      const last = Math.floor(center + halfWidth)
+      let sum = 0
+      for (let index = first; index <= last; index += 1) {
+        const tablePosition = Math.abs(center - index) * tableScale
+        const tableIndex = Math.floor(tablePosition)
+        const fraction = tablePosition - tableIndex
+        const tap = table[tableIndex] + fraction * (table[tableIndex + 1] - table[tableIndex])
+        sum += history[index] * tap
+      }
+      this.output[count] = sum * scale
+      count += 1
+      this.position += this.step
+    }
+    this.discardConsumed()
+    return this.output.subarray(0, count)
+  }
+
+  private append(input: Float32Array): void {
+    const needed = this.historyLength + input.length
+    if (needed > this.history.length) {
+      const grown = new Float32Array(needed * 2)
+      grown.set(this.history.subarray(0, this.historyLength))
+      this.history = grown
+    }
+    this.history.set(input, this.historyLength)
+    this.historyLength = needed
+  }
+
+  private discardConsumed(): void {
+    const consumed = Math.floor(this.position - this.halfWidth)
+    if (consumed <= 0) {
+      return
+    }
+    this.history.copyWithin(0, consumed, this.historyLength)
+    this.historyLength -= consumed
+    this.position -= consumed
+  }
+}