diff --git a/src/main/speech/speech-vad-model.ts b/src/main/speech/speech-vad-model.ts index 5aa954a01b0..12403954e06 100644 --- a/src/main/speech/speech-vad-model.ts +++ b/src/main/speech/speech-vad-model.ts @@ -19,10 +19,7 @@ export function downloadSileroVadInBackground(models: Pick -): string | undefined { +function placedSileroVadPath(models: Pick): string | undefined { const [file] = SILERO_VAD_MODEL.downloadFiles const path = join(models.getModelDir(SILERO_VAD_MODEL.id), file.name) if (path === verifiedVadPath) { @@ -36,6 +33,24 @@ export function sileroVadModelPath( } catch { // Not downloaded yet, e.g. the model came before VAD support. } - downloadSileroVadInBackground(models) return undefined } + +/** Whether a model-state change for `modelId` means the Silero VAD file is now in place. */ +export function sileroVadLanded( + models: Pick, + modelId: string +): boolean { + return modelId === SILERO_VAD_MODEL.id && placedSileroVadPath(models) !== undefined +} + +/** The downloaded Silero VAD file; when it's missing, starts fetching it and returns undefined. */ +export function sileroVadModelPath( + models: Pick +): string | undefined { + const path = placedSileroVadPath(models) + if (!path) { + downloadSileroVadInBackground(models) + } + return path +} diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index cdd27198850..f397c55b819 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -1,6 +1,9 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import type { ModelManager } from './model-manager' import type { SpeechModelState } from '../../shared/speech-types' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.hoisted(() => { class HoistedMockWorker extends EventTarget { @@ -104,9 +107,16 @@ const stoppedReply = (worker: InstanceType) => ({ streamId: worker.messages.findLast((message) => message.type === 'stop')?.streamId }) -const testModels = (models: Pick): ModelManager => - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SttService reads getModelState and getModelDir, and fetches the VAD with downloadModel. - ({ ...models, downloadModel: vi.fn().mockResolvedValue(undefined) }) as never +const testModels = ( + models: Pick, + onModelStateChange: ModelManager['setProgressCallback'] = () => () => {} +): ModelManager => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SttService reads getModelState and getModelDir, fetches the VAD with downloadModel and watches it land. + ({ + ...models, + downloadModel: vi.fn().mockResolvedValue(undefined), + setProgressCallback: onModelStateChange + }) as never const readyModels = (): ModelManager => testModels({ @@ -122,6 +132,7 @@ async function flushMicrotasks(): Promise { } vi.mock('./model-catalog', () => ({ + isLocalSpeechModel: (manifest: { provider: string }) => manifest.provider === 'local', getCatalogModel: (id: string) => id === 'openai-model' ? { @@ -174,6 +185,31 @@ describe('SttService', () => { resetWorkers() }) + it('reloads a recognizer loaded before the VAD landed once the dictation using it ends', async () => { + const modelsDir = mkdtempSync(join(tmpdir(), 'stt-vad-')) + onTestFinished(() => rmSync(modelsDir, { recursive: true, force: true })) + const watchModels = vi.fn(() => () => {}) + const models = testModels( + { getModelState: readyModels().getModelState, getModelDir: (id) => join(modelsDir, id) }, + watchModels + ) + const service = new SttService(models, spawnMockWorker) + await service.startDictation('model-a', vi.fn(), undefined, 'desktop') + const initOf = (worker = getLastWorker()) => worker?.messages.find((m) => m.type === 'init') + expect(initOf()).toMatchObject({ vadModelPath: undefined }) + + const vadPath = join(modelsDir, 'silero-vad-v6.2', 'silero-vad-v6.2-F32.gguf') + mkdirSync(join(modelsDir, 'silero-vad-v6.2')) + writeFileSync(vadPath, Buffer.alloc(1_241_056)) + watchModels.mock.calls[0]?.[0]('silero-vad-v6.2', -1) + await flushMicrotasks() + expect(getCreatedWorkerCount()).toBe(1) + + await service.stopDictation('desktop') + await vi.waitFor(() => expect(getCreatedWorkerCount()).toBe(2)) + expect(initOf()).toMatchObject({ vadModelPath: vadPath }) + }) + it('gives each device dictating at once only its own text from one recognizer', async () => { const service = new SttService(readyModels(), spawnMockWorker) const desktop = vi.fn() @@ -739,15 +775,19 @@ describe('SttService', () => { async (event) => { vi.useFakeTimers() try { - const models: Pick = { + const models: Pick< + ModelManager, + 'getModelState' | 'getModelDir' | 'downloadModel' | 'setProgressCallback' + > = { getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), getModelDir: vi.fn().mockReturnValue('/tmp/model-a'), - downloadModel: vi.fn().mockResolvedValue(undefined) + downloadModel: vi.fn().mockResolvedValue(undefined), + setProgressCallback: () => () => {} } - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This lifecycle reads getModelState/getModelDir and downloadModel (VAD) on the model manager. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This lifecycle reads getModelState/getModelDir, downloadModel (VAD) and setProgressCallback on the model manager. const service = new SttService(models as ModelManager, spawnMockWorker) const sink = vi.fn() await service.startDictation('model-a', sink, undefined, 'desktop') diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 048c4872135..2333dfae0ed 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -6,6 +6,7 @@ import { } from '../../shared/dictation-text-pipeline' import { getCatalogModel } from './model-catalog' import type { ModelManager } from './model-manager' +import { sileroVadLanded } from './speech-vad-model' import type { SpawnSttWorkerProcess } from './stt-worker-process' import { prewarmSttModel } from './stt-session-prewarm' import { startSttDictation } from './stt-session-start' @@ -45,6 +46,27 @@ export class SttService { constructor(modelManager: ModelManager, spawnWorker: SpawnSttWorkerProcess) { this.state = createSttSessionState(modelManager, spawnWorker) + modelManager.setProgressCallback((modelId) => { + if (sileroVadLanded(modelManager, modelId)) { + void this.reloadForVad() + } + }) + } + + /** A recognizer loaded before the VAD landed cuts text only at stop; reload it once no dictation uses it. */ + private async reloadForVad(): Promise { + const engine = this.state.engine + if (!engine || engine.retired || engine.vadModelPath) { + return + } + const { modelId, language } = engine + const unloaded = this.unloadModelWhenIdle() + const generation = this.state.prewarmGeneration + await unloaded + // A newer pre-warm or a dictation took over; loading is theirs now. + if (this.state.prewarmGeneration === generation && !this.state.engine) { + await this.prewarm(modelId, language) + } } /** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */ diff --git a/src/main/speech/stt-session-start.ts b/src/main/speech/stt-session-start.ts index cd815fbc920..ee7bd5b1833 100644 --- a/src/main/speech/stt-session-start.ts +++ b/src/main/speech/stt-session-start.ts @@ -187,7 +187,7 @@ export function spawnSttEngine( if (!speechModulesDir) { throw new Error(LOCAL_SPEECH_UNAVAILABLE_ERROR) } - // A VAD fetched now serves the next engine start. + // A VAD fetched now reloads this engine once it is idle (SttService). const vadModelPath = sileroVadModelPath(state.modelManager) clearSttIdleTeardownTimer(state) const worker = state.spawnWorker() @@ -210,6 +210,7 @@ export function spawnSttEngine( worker, detach, retired: false, + vadModelPath, onUnloaded: [], ready: readyPromise.catch((error: unknown) => { failSttEngine(state, engine, STT_WORKER_EXITED_ERROR) diff --git a/src/main/speech/stt-session-state.ts b/src/main/speech/stt-session-state.ts index 21c081dacf9..c54d4dd5eed 100644 --- a/src/main/speech/stt-session-state.ts +++ b/src/main/speech/stt-session-state.ts @@ -19,6 +19,8 @@ export type SttEngine = SttRecognizerConfig & { detach: () => void /** Torn down on purpose, so a load it cut short is not a failure of the model. */ retired: boolean + /** Absent when the VAD wasn't downloaded yet at load, so this process cuts text only at stop. */ + vadModelPath: string | undefined /** Called once this process is gone; set when it must not outlive its last dictation. */ onUnloaded: (() => void)[] }