diff --git a/mobile/src/dictation/dictation-reply-schema.ts b/mobile/src/dictation/dictation-reply-schema.ts index 70c3bd707fa..192e097c9dd 100644 --- a/mobile/src/dictation/dictation-reply-schema.ts +++ b/mobile/src/dictation/dictation-reply-schema.ts @@ -36,6 +36,8 @@ export const dictationSetupSchema = z.looseObject({ dictationHoldToRecord: salvagedOptional('dictationHoldToRecord', z.boolean()), // Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default. removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()), + // Absent from older hosts; the phone normalizes it before spelling with it. + dictationTerms: salvagedOptional('dictationTerms', z.unknown()), models: salvagingArray(speechModelSchema) }) diff --git a/mobile/src/dictation/mobile-dictation-host-context.ts b/mobile/src/dictation/mobile-dictation-host-context.ts index 44fd857d028..240a31f50b2 100644 --- a/mobile/src/dictation/mobile-dictation-host-context.ts +++ b/mobile/src/dictation/mobile-dictation-host-context.ts @@ -2,6 +2,7 @@ import { createContext, useContext, useEffect, useState, useSyncExternalStore } import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state' import type { RpcClient } from '../transport/rpc/rpc-client-contract' import type { MobileDictationAwayTarget } from './mobile-dictation-away-insert' +import type { DictationTerm } from '../../../src/shared/dictation-dictionary' export type MobileDictationHostToast = { message: string; open?: () => void } @@ -20,6 +21,10 @@ export type MobileDictationOwner = { readonly paneKey: string | undefined readonly dictationMode: string | undefined readonly removeFillerWords: boolean + // The computer's terms for this worktree, for words the phone recognizes itself. + readonly dictationTerms: readonly DictationTerm[] | undefined + // The computer spells its own words with this worktree's repo terms. + readonly worktreeId: string } // A mounted session screen, as the dictation host sees it. diff --git a/mobile/src/dictation/mobile-dictation-host.tsx b/mobile/src/dictation/mobile-dictation-host.tsx index d7e7d33632b..c4960704a7a 100644 --- a/mobile/src/dictation/mobile-dictation-host.tsx +++ b/mobile/src/dictation/mobile-dictation-host.tsx @@ -100,6 +100,8 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode client: owner?.client ?? null, enabled: true, removeFillerWords: owner?.removeFillerWords ?? true, + dictationTerms: owner?.dictationTerms, + worktreeId: owner?.worktreeId, onTranscript: (text) => { const current = ownerRef.current if (!current) { diff --git a/mobile/src/dictation/mobile-dictation-setup.ts b/mobile/src/dictation/mobile-dictation-setup.ts index 421cd3c95ac..61e607ca7a4 100644 --- a/mobile/src/dictation/mobile-dictation-setup.ts +++ b/mobile/src/dictation/mobile-dictation-setup.ts @@ -126,8 +126,12 @@ export function isDictationSetupRequiredError(message: string): boolean { return SETUP_REQUIRED_CODES.has(message) || message.startsWith('voice_model_not_ready:') } -export async function fetchDictationSetup(client: RpcOperationSender): Promise { - const reply = await requestDictationSetupReply(client) +/** `worktreeId` asks for that repo's dictionary terms too. */ +export async function fetchDictationSetup( + client: RpcOperationSender, + worktreeId?: string +): Promise { + const reply = await requestDictationSetupReply(client, worktreeId ? { worktreeId } : null) if (isLegacyDesktopSpeechSetupReply(reply)) { throw new Error( translate( @@ -145,15 +149,18 @@ export async function fetchDictationSetup(client: RpcOperationSender): Promise { +async function requestDictationSetupReply( + client: RpcOperationSender, + params: { worktreeId: string } | null +): Promise { try { - return await dictationSetupRead.request(client, null) + return await dictationSetupRead.request(client, params) } catch (error) { if (!(error instanceof LogicalClientCutoverError)) { throw error } // Replay this read on authenticated replacement; mutations must surface cutover to avoid replaying unknown commits. - return dictationSetupRead.request(client, null) + return dictationSetupRead.request(client, params) } } diff --git a/mobile/src/hooks/mobile-dictation-desktop-start.ts b/mobile/src/hooks/mobile-dictation-desktop-start.ts index 2b6b28cdf0c..e355d530b39 100644 --- a/mobile/src/hooks/mobile-dictation-desktop-start.ts +++ b/mobile/src/hooks/mobile-dictation-desktop-start.ts @@ -15,6 +15,7 @@ import type { RpcClient } from '../transport/rpc/rpc-client-contract' type StartMobileDictationDesktopSessionOptions = { client: RpcClient dictationId: string + worktreeId: string | undefined generation: number getCurrentGeneration: () => number getEnabled: () => boolean @@ -102,7 +103,7 @@ export function startMobileDictationDesktopSession( } async function startOnHost(options: StartMobileDictationDesktopSessionOptions): Promise { - const { client, dictationId } = options + const { client, dictationId, worktreeId } = options let hostDedupesResends = false try { @@ -111,7 +112,7 @@ async function startOnHost(options: StartMobileDictationDesktopSessionOptions): client, dictationSessionStart.request( client, - { dictationId }, + { dictationId, ...(worktreeId ? { worktreeId } : {}) }, { timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS } ) ).catch(() => { diff --git a/mobile/src/hooks/mobile-dictation-engine-run.ts b/mobile/src/hooks/mobile-dictation-engine-run.ts index 5d7307b19d8..b61d5390e51 100644 --- a/mobile/src/hooks/mobile-dictation-engine-run.ts +++ b/mobile/src/hooks/mobile-dictation-engine-run.ts @@ -48,6 +48,7 @@ export type MobileDictationRun = { readonly finish: RefObject<(emptyTranscriptMessage: string) => Promise> readonly onTranscript: RefObject<(text: string) => void> readonly onAudioLimitReached: RefObject<(() => void) | undefined> + readonly worktreeId: RefObject readonly keptTranscript: MobileDictationTranscriptSoFar // keepText: leave the words so far on screen (recording, finishing, or a failure kept for Retry). readonly applyStatus: (next: DictationStatus, keepText?: boolean) => void diff --git a/mobile/src/hooks/mobile-dictation-host-start.ts b/mobile/src/hooks/mobile-dictation-host-start.ts index 2ad40864322..fc9898e3fa0 100644 --- a/mobile/src/hooks/mobile-dictation-host-start.ts +++ b/mobile/src/hooks/mobile-dictation-host-start.ts @@ -22,6 +22,7 @@ export async function startMobileDictationOnHost( await startMobileDictationDesktopSession({ client, dictationId, + worktreeId: run.worktreeId.current, generation, getCurrentGeneration: () => run.generation.current, getEnabled: () => run.enabled.current, diff --git a/mobile/src/hooks/mobile-dictation-session-state.ts b/mobile/src/hooks/mobile-dictation-session-state.ts index 4bab0bd3fcd..eb41797be8d 100644 --- a/mobile/src/hooks/mobile-dictation-session-state.ts +++ b/mobile/src/hooks/mobile-dictation-session-state.ts @@ -20,6 +20,10 @@ export type UseMobileDictationOptions = { onError?: (error: Error, outcome: { kept: boolean }) => void // The computer's filler switch, for text the phone recognizes itself; unknown means on. removeFillerWords?: boolean + // The computer's dictionary for this worktree, unvalidated, for text the phone recognizes itself. + dictationTerms?: unknown + // Lets the computer pick this worktree's repo terms for text it recognizes. + worktreeId?: string // The host's cloud upload is full; the dictation stops and inserts what was kept. onAudioLimitReached?: () => void } diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 6fea90e2e7e..027042548be 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -36,9 +36,10 @@ export type { UseMobileDictationResult } from './mobile-dictation-session-state' export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult { useMobileTranslation() const { client, enabled, onTranscript, onError, removeFillerWords, onAudioLimitReached } = options + const { dictationTerms, worktreeId } = options // Both capture seams hold the screen while the microphone is open; native uses the device directly and page uses shell audio verbs. const capture = useDictationCapture() - const speech = useOnDeviceSpeech(removeFillerWords) + const speech = useOnDeviceSpeech(removeFillerWords, dictationTerms) const [status, setStatus] = useState('idle') const [error, setError] = useState(null) // A failure whose recording is kept for Retry; cleared by Retry, discard and the next dictation. @@ -61,6 +62,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const onTranscriptRef = useRef(onTranscript) const onErrorRef = useRef(onError) const onAudioLimitReachedRef = useRef(onAudioLimitReached) + const worktreeIdRef = useRef(worktreeId) const pendingChunksRef = useRef>>(new Set()) const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget()) const audioPhaseRef = useRef('closed') @@ -79,7 +81,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil onTranscriptRef.current = onTranscript onErrorRef.current = onError onAudioLimitReachedRef.current = onAudioLimitReached - }, [client, speech, enabled, onTranscript, onError, onAudioLimitReached]) + worktreeIdRef.current = worktreeId + }, [client, speech, enabled, onTranscript, onError, onAudioLimitReached, worktreeId]) const applyStatus = useCallback( // Kept through processing so the text so far stays in place until the final transcript replaces it in the same render. @@ -146,6 +149,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil finish: finishRef, onTranscript: onTranscriptRef, onAudioLimitReached: onAudioLimitReachedRef, + worktreeId: worktreeIdRef, keptTranscript, applyStatus, showEngine, diff --git a/mobile/src/platform/on-device-speech.ts b/mobile/src/platform/on-device-speech.ts index 143c56c3317..f4b5c76ac50 100644 --- a/mobile/src/platform/on-device-speech.ts +++ b/mobile/src/platform/on-device-speech.ts @@ -1,6 +1,9 @@ import type { OnDeviceSpeech } from './on-device-speech-contract' // The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer. -export function useOnDeviceSpeech(_removeFillerWords?: boolean): OnDeviceSpeech | null { +export function useOnDeviceSpeech( + _removeFillerWords?: boolean, + _dictationTerms?: unknown +): OnDeviceSpeech | null { return null } diff --git a/mobile/src/platform/on-device-speech.web.ts b/mobile/src/platform/on-device-speech.web.ts index 00a8c777803..5d86d5f9429 100644 --- a/mobile/src/platform/on-device-speech.web.ts +++ b/mobile/src/platform/on-device-speech.web.ts @@ -1,4 +1,5 @@ import { useLayoutEffect, useMemo, useRef } from 'react' +import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary' import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline' import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs' import type { OnDeviceSpeech } from './on-device-speech-contract' @@ -9,25 +10,29 @@ const keepText = (text: string): string => text // The phone's recognizer transcribes only the locale it started in, so that locale's language picks the fillers, as the host does for its model. function phoneDictationTextCleaner( locale: string | null, - removeFillerWords: boolean + removeFillerWords: boolean, + dictationTerms: unknown ): (text: string) => string { const language = locale?.split('-')[0].toLowerCase() - const clean = createDictationTextPipeline(language ? { language } : undefined, { - removeFillerWords - }) + const clean = createDictationTextPipeline( + language ? { language } : undefined, + { removeFillerWords }, + normalizeDictationTerms(dictationTerms) + ) return clean ?? keepText } function createPageOnDeviceSpeech( verbs: NativeVerbs, - removeFillerWords: () => boolean + textSettings: () => { removeFillerWords: boolean; dictationTerms: unknown } ): OnDeviceSpeech { let clean = keepText return { status: () => verbs.speechStatus(onDeviceSpeechLocale()), start: async () => { const started = await verbs.speechStart(onDeviceSpeechLocale()) - clean = phoneDictationTextCleaner(started.locale, removeFillerWords()) + const { removeFillerWords, dictationTerms } = textSettings() + clean = phoneDictationTextCleaner(started.locale, removeFillerWords, dictationTerms) return started }, read: async () => { @@ -46,17 +51,20 @@ function createPageOnDeviceSpeech( } // Null on a shell without the verb, so dictation stays on the computer exactly as before. -// removeFillerWords is the computer's switch, read at each start. -export function useOnDeviceSpeech(removeFillerWords = true): OnDeviceSpeech | null { +// The computer's filler switch and dictionary for this worktree, read at each start. +export function useOnDeviceSpeech( + removeFillerWords = true, + dictationTerms?: unknown +): OnDeviceSpeech | null { const verbs = useNativeVerbs() - const removeFillerWordsRef = useRef(removeFillerWords) + const textSettingsRef = useRef({ removeFillerWords, dictationTerms }) useLayoutEffect(() => { - removeFillerWordsRef.current = removeFillerWords - }, [removeFillerWords]) + textSettingsRef.current = { removeFillerWords, dictationTerms } + }, [removeFillerWords, dictationTerms]) return useMemo( () => verbs.canRecognizeSpeech - ? createPageOnDeviceSpeech(verbs, () => removeFillerWordsRef.current) + ? createPageOnDeviceSpeech(verbs, () => textSettingsRef.current) : null, [verbs] ) diff --git a/mobile/src/session/use-mobile-session-dictation-setup.ts b/mobile/src/session/use-mobile-session-dictation-setup.ts index 3bc4aadeeab..a1c46e002b8 100644 --- a/mobile/src/session/use-mobile-session-dictation-setup.ts +++ b/mobile/src/session/use-mobile-session-dictation-setup.ts @@ -8,14 +8,17 @@ import { fetchDictationSetup } from '../dictation/mobile-dictation-setup' import type { MobileSessionFeedbackCapabilitiesModel } from './use-mobile-session-feedback-capabilities' +import type { DictationTerm } from '../../../src/shared/dictation-dictionary' // The computer's dictation settings as this session reads them: mic shown, mode, filler switch, and whether setup is needed first. export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapabilitiesModel) { - const { client, connState, setShowDictationSetup, setDictationMode, setDictationEnabled } = scope + const { client, connState, worktreeId, setShowDictationSetup, setDictationMode } = scope + const { setDictationEnabled } = scope // The phone's own recognizer needs no computer setup, and chat inserts its words on the phone. const phoneDictation = usePhoneDictationReady() - // The host's filler switch, kept from its last setup read; on until one says otherwise. + // The host's filler switch and this worktree's terms, kept from its last setup read; fillers on until one says otherwise. const [removeFillerWords, setRemoveFillerWords] = useState(true) + const [dictationTerms, setDictationTerms] = useState(undefined) // Only a host setup read sets this; while false, the host's start refusal decides (older hosts, first tap before the read). const dictationNeedsSetupRef = useRef(false) @@ -27,17 +30,18 @@ export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapab return } try { - const setup = await fetchDictationSetup(client) + const setup = await fetchDictationSetup(client, worktreeId) if (read === dictationSettingsReadRef.current) { setDictationMode(setup.dictationMode) setRemoveFillerWords(setup.removeFillerWords !== false) + setDictationTerms(setup.dictationTerms) setDictationEnabled(!dictationMicHidden(setup)) dictationNeedsSetupRef.current = dictationNeedsSetup(setup) } } catch { // Keep the last settings when an older host cannot report them. } - }, [client, connState, phoneDictation.refresh, setDictationEnabled, setDictationMode]) + }, [client, connState, phoneDictation.refresh, setDictationEnabled, setDictationMode, worktreeId]) // The sheet may have changed host setup: let the host judge taps until the fresh read lands. const closeDictationSetup = useCallback(() => { @@ -65,6 +69,7 @@ export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapab return { phoneDictation, removeFillerWords, + dictationTerms, dictationNeedsSetupRef, closeDictationSetup, refreshDictationMode diff --git a/mobile/src/session/use-mobile-session-dictation.ts b/mobile/src/session/use-mobile-session-dictation.ts index 5b4c6dffd25..9212ed7d75d 100644 --- a/mobile/src/session/use-mobile-session-dictation.ts +++ b/mobile/src/session/use-mobile-session-dictation.ts @@ -185,6 +185,7 @@ export function useMobileSessionDictation( const { phoneDictation, removeFillerWords, + dictationTerms, dictationNeedsSetupRef, closeDictationSetup, refreshDictationMode @@ -234,7 +235,9 @@ export function useMobileSessionDictation( executionHost ), dictationMode: scope.dictationMode, - removeFillerWords + removeFillerWords, + dictationTerms, + worktreeId }) }, [ activeHandle, @@ -250,6 +253,7 @@ export function useMobileSessionDictation( instance, phoneDictation.readyRef, removeFillerWords, + dictationTerms, scope.dictationMode, sessionKey, setShowDictationSetup, diff --git a/src/main/ipc/repos/repo-update-handler.ts b/src/main/ipc/repos/repo-update-handler.ts index 205243cc719..d4fea3f3571 100644 --- a/src/main/ipc/repos/repo-update-handler.ts +++ b/src/main/ipc/repos/repo-update-handler.ts @@ -17,6 +17,7 @@ import { } from '../../../shared/worktree/visibility-sources' import { prepareLocalWorktreeRootForRepo } from '../../workspaces/creation/worktree-root-preparation' import { commitProvenRepoKindChange } from '../../workspaces/catalog/folder-repo-git-conversion' +import { normalizeDictationTerms } from '../../../shared/dictation-dictionary' import { invalidateAuthorizedRootsCache } from '../worktrees/removal/registered-worktree-roots-cache' import { notifyReposChanged } from './repos-changed-notification' import { @@ -45,6 +46,7 @@ export function registerRepoUpdateHandler(mainWindow: BrowserWindow, store: Stor | 'worktreeBasePath' | 'kind' | 'symlinkPaths' + | 'dictationTerms' | 'issueSourcePreference' | 'forkSyncMode' | 'agentWorkspaceTrust' @@ -110,6 +112,14 @@ export function registerRepoUpdateHandler(mainWindow: BrowserWindow, store: Stor delete updates.symlinkPaths } } + if ('dictationTerms' in updates) { + const terms = normalizeDictationTerms(updates.dictationTerms) + if (terms) { + updates.dictationTerms = terms + } else { + delete updates.dictationTerms + } + } if ('worktreeBasePath' in updates && updates.worktreeBasePath !== undefined) { const v = updates.worktreeBasePath as unknown if (typeof v !== 'string') { diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 423c64e6dab..ea070256d20 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -6,9 +6,8 @@ import { type MessagePortMain } from 'electron' import { abortWhenRendererGone } from './renderer-lifetime-abort' -import { join } from 'node:path' -import { writeFile, unlink } from 'node:fs/promises' -import { createHash } from 'node:crypto' +import { mergeDictationTerms } from '../../shared/dictation-dictionary' +import type { DictationContext } from '../../shared/speech-types' import { SPEECH_MODEL_CATALOG } from '../speech/model-catalog' import { deleteLocalSpeechModel } from '../speech/speech-model-deletion' import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service' @@ -112,29 +111,6 @@ export function registerSpeechHandlers(store: Store): void { }) }) - const getHotwordsFilePath = (content: string): string => { - const digest = createHash('sha256').update(content).digest('hex').slice(0, 12) - // Native speech libraries may open paths with ANSI Windows APIs; keep hotwords in the ASCII-safe model cache. - return join(getSpeechModelManager(store).getModelsDir(), `speech-hotwords-${digest}.txt`) - } - - const hotwordFileUsers = new Map() - const retainHotwordsFile = (path: string): ((written: boolean) => void) => { - const entry = hotwordFileUsers.get(path) ?? { users: 0, written: false } - entry.users += 1 - hotwordFileUsers.set(path, entry) - return (written) => { - entry.written ||= written - entry.users -= 1 - if (entry.users === 0) { - hotwordFileUsers.delete(path) - if (entry.written) { - unlink(path).catch(() => {}) - } - } - } - } - // Renderer run ids restart per document; a retired document's owner must not be reused by its successor. const retiredDocumentCounts = new Map() const getDesktopOwner = (senderId: number, sessionId: string): string => @@ -153,7 +129,7 @@ export function registerSpeechHandlers(store: Store): void { ipcMain.handle( 'speech:startDictation', - async (event, modelId: string, hotwords?: string[], sessionId = 'desktop') => { + async (event, modelId: string, context?: DictationContext, sessionId = 'desktop') => { const senderId = event.sender.id const owner = getDesktopOwner(senderId, sessionId) const audioPort = pendingAudioPorts.get(owner) @@ -164,8 +140,6 @@ export function registerSpeechHandlers(store: Store): void { audioPort?.close() return { directAudio } } - let resolvedHotwordsPath: string | undefined - let releaseHotwords: ((written: boolean) => void) | undefined let rendererRetired = false const rendererLifetime = abortWhenRendererGone(event.sender) let sessionListenerRemoved = false @@ -207,14 +181,6 @@ export function registerSpeechHandlers(store: Store): void { } } - if (hotwords && hotwords.length > 0) { - const content = `${hotwords.join('\n')}\n` - const hotwordsFilePath = getHotwordsFilePath(content) - releaseHotwords = retainHotwordsFile(hotwordsFilePath) - await writeFile(hotwordsFilePath, content, 'utf-8') - resolvedHotwordsPath = hotwordsFilePath - } - if (rendererRetired || window.isDestroyed()) { cleanupSessionListener() return { directAudio } @@ -252,7 +218,7 @@ export function registerSpeechHandlers(store: Store): void { break } }, - resolvedHotwordsPath, + mergeDictationTerms(context?.repoTerms, store.getSettings().voice?.dictationTerms), owner, store.getSettings().voice ) @@ -272,7 +238,6 @@ export function registerSpeechHandlers(store: Store): void { cleanupSessionListener() throw err } finally { - releaseHotwords?.(resolvedHotwordsPath !== undefined) if (!directAudio) { audioPort?.close() } diff --git a/src/main/persistence/loading-store/repo-lifecycle-operations.ts b/src/main/persistence/loading-store/repo-lifecycle-operations.ts index 1b1748da2e4..ea15fa64885 100644 --- a/src/main/persistence/loading-store/repo-lifecycle-operations.ts +++ b/src/main/persistence/loading-store/repo-lifecycle-operations.ts @@ -187,6 +187,7 @@ export class RepoLifecycleOperations { | 'folderUpgradeGitRootPath' | 'executionHostId' | 'symlinkPaths' + | 'dictationTerms' | 'issueSourcePreference' | 'forkSyncMode' | 'agentWorkspaceTrust' diff --git a/src/main/persistence/tracking-repos/repo-update-operations.ts b/src/main/persistence/tracking-repos/repo-update-operations.ts index 8841638c145..ee07378f475 100644 --- a/src/main/persistence/tracking-repos/repo-update-operations.ts +++ b/src/main/persistence/tracking-repos/repo-update-operations.ts @@ -101,6 +101,7 @@ export class RepoUpdatePersistenceOperations { | 'folderUpgradeGitRootPath' | 'executionHostId' | 'symlinkPaths' + | 'dictationTerms' | 'issueSourcePreference' | 'defaultJiraProject' | 'forkSyncMode' diff --git a/src/main/runtime/repositories/runtime-repository-settings-controller.ts b/src/main/runtime/repositories/runtime-repository-settings-controller.ts index ff8af0588e4..e9829d7f8f5 100644 --- a/src/main/runtime/repositories/runtime-repository-settings-controller.ts +++ b/src/main/runtime/repositories/runtime-repository-settings-controller.ts @@ -41,6 +41,7 @@ type RepositoryUpdates = Partial< | 'worktreeBasePath' | 'kind' | 'symlinkPaths' + | 'dictationTerms' | 'issueSourcePreference' | 'defaultJiraProject' | 'externalWorktreeVisibility' diff --git a/src/main/runtime/rpc/methods/speech/speech.ts b/src/main/runtime/rpc/methods/speech/speech.ts index 352dc1357d7..a7120dcb2eb 100644 --- a/src/main/runtime/rpc/methods/speech/speech.ts +++ b/src/main/runtime/rpc/methods/speech/speech.ts @@ -5,15 +5,16 @@ import { DictationSetup, DictationStart, OpenAiSpeechKeySet, - SpeechModelAction + SpeechModelAction, + SpeechSetupRead } from '../../../../../shared/rpc-contract/speech-params' export const SPEECH_METHODS = [ defineMethod({ name: 'speech.models.list', permission: 'workspace', - params: null, - handler: async (_params, { runtime }) => runtime.listMobileSpeechModels() + params: SpeechSetupRead, + handler: async (params, { runtime }) => runtime.listMobileSpeechModels(params.worktreeId) }), defineMethod({ name: 'speech.models.download', diff --git a/src/main/runtime/speech/runtime-dictation-terms.ts b/src/main/runtime/speech/runtime-dictation-terms.ts new file mode 100644 index 00000000000..13785a2cc2d --- /dev/null +++ b/src/main/runtime/speech/runtime-dictation-terms.ts @@ -0,0 +1,12 @@ +import { mergeDictationTerms, type DictationTerm } from '../../../shared/dictation-dictionary' +import { getRepoIdFromWorktreeId } from '../../../shared/worktree/id' +import type { RuntimeStore } from '../construction/runtime-store-contract' + +/** This host's dictionary for a worktree: its repo's terms, then the personal ones. A folder workspace has no repo row, so it gets the personal ones. */ +export function hostDictationTerms( + store: RuntimeStore, + worktreeId: string | undefined +): DictationTerm[] { + const repo = worktreeId ? store.getRepo(getRepoIdFromWorktreeId(worktreeId)) : undefined + return mergeDictationTerms(repo?.dictationTerms, store.getSettings().voice?.dictationTerms) +} diff --git a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts index e7af1ecf9be..2bbf14ed4ed 100644 --- a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts @@ -12,6 +12,7 @@ import { type MobileDictationChunk } from './mobile-dictation-chunk-order' import type { RuntimeStore } from '../construction/runtime-store-contract' +import { hostDictationTerms } from './runtime-dictation-terms' import { DictationTranscript, type DictationTranscriptSnapshot @@ -42,6 +43,7 @@ export class RuntimeMobileDictationController { async start(params: { dictationId: string modelId?: string + worktreeId?: string clientId?: string }): Promise<{ dictationId: string; modelId: string; dedupesResentChunks: true }> { const store = this.requireStore() @@ -89,7 +91,7 @@ export class RuntimeMobileDictationController { await getSpeechSttService(store).startDictation( modelId, (event) => acceptEvent(session, event), - undefined, + hostDictationTerms(store, params.worktreeId), owner, voice ) diff --git a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts index 17c63be4f29..fa042e5be68 100644 --- a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts +++ b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts @@ -18,6 +18,7 @@ import { getSpeechModelDeletionErrorCode } from '../../speech/speech-model-deletion' import type { RuntimeStore } from '../construction/runtime-store-contract' +import { hostDictationTerms } from './runtime-dictation-terms' import { clearOpenAiSpeechApiKey, getOpenAiSpeechApiKeyProtection, @@ -34,7 +35,8 @@ export type OpenAiSpeechKeyStatus = { export class RuntimeMobileSpeechCatalog { constructor(private readonly getStore: () => RuntimeStore | null) {} - async list(): Promise { + /** `worktreeId` picks the repo terms a phone recognizing on its own spells with. */ + async list(worktreeId?: string): Promise { const store = this.requireStore() const voice = store.getSettings().voice ?? getDefaultVoiceSettings() const states = await getSpeechModelManager(store).getModelStates() @@ -67,6 +69,7 @@ export class RuntimeMobileSpeechCatalog { dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle', dictationHoldToRecord: voice.dictationHoldToRecord !== false, removeFillerWords: voice.removeFillerWords !== false, + dictationTerms: hostDictationTerms(store, worktreeId), models } } diff --git a/src/main/speech/stt-recognizer.ts b/src/main/speech/stt-recognizer.ts index 2191fdf3ce0..66379b1ff3d 100644 --- a/src/main/speech/stt-recognizer.ts +++ b/src/main/speech/stt-recognizer.ts @@ -4,7 +4,6 @@ export type SttRecognizerInit = { modelDir: string files: string[] sampleRate: number - hotwordsFilePath?: string /** Absent = auto-detect. */ language?: string } @@ -17,6 +16,8 @@ export type SttRecognizerPost = (event: SttRecognizerEvent) => void /** One loaded model in the speech host child, shared by every dictation stream; calls never overlap. */ export type SttRecognizer = { + /** Starts a stream that expects these spellings; a stream fed without one expects none. */ + open(streamId: string, vocabulary: readonly string[]): void feed(streamId: string, samples: Float32Array, sampleRate: number): Promise /** Decode the stream's buffered audio, post its text and drop the stream; may throw after posting. */ finish(streamId: string): Promise diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index 02593f5dde2..aa2e80fd3a9 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -269,6 +269,35 @@ describe('SttService', () => { ]) }) + it('spells each dictation with its own terms on one shared recognizer', async () => { + const service = new SttService(readyModels(), spawnMockWorker) + const desktop = vi.fn() + const phone = vi.fn() + + await service.startDictation('model-a', desktop, [{ term: 'pnpm' }], 'desktop') + await service.startDictation( + 'model-a', + phone, + [{ term: 'kubectl', heardAs: 'cube control' }], + 'mobile:phone-a:1' + ) + const worker = getLastWorker()! + const opens = worker.messages.filter((message) => message.type === 'open') + for (const { streamId } of opens) { + worker.emit('message', { type: 'final', streamId, text: 'Run PNPM and cube control.' }) + } + + expect(getCreatedWorkerCount()).toBe(1) + expect(opens).toEqual([ + { type: 'open', streamId: opens[0].streamId, vocabulary: ['pnpm'] }, + { type: 'open', streamId: opens[1].streamId, vocabulary: ['kubectl'] } + ]) + expect(desktop.mock.calls.at(-1)).toEqual([ + { type: 'final', text: 'Run pnpm and cube control.' } + ]) + expect(phone.mock.calls.at(-1)).toEqual([{ type: 'final', text: 'Run PNPM and kubectl.' }]) + }) + it("hands a renderer's audio port only to its own on-device stream", async () => { const service = new SttService(readyModels(), spawnMockWorker) await service.startDictation('model-a', vi.fn(), undefined, 'mobile:phone-a:1') @@ -606,12 +635,17 @@ describe('SttService', () => { it('does not treat internal warm-worker replacement as startup cancellation', async () => { const service = new SttService(readyModels(), spawnMockWorker) - await service.startDictation('model-a', vi.fn(), '/tmp/hotwords-a.txt', 'desktop:1') + await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop:1', { + dictationLanguage: 'ja' + }) await service.stopDictation('desktop:1') await expect( - service.startDictation('model-a', vi.fn(), '/tmp/hotwords-b.txt', 'desktop:1') + service.startDictation('whisper-model', vi.fn(), undefined, 'desktop:1', { + dictationLanguage: 'ko' + }) ).resolves.toBe(undefined) + expect(getCreatedWorkerCount()).toBe(2) }) it('removes lifecycle listeners when the active worker errors', async () => { diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 6aa14d9e396..df0233b4a64 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -1,4 +1,5 @@ import type { MessagePortMain } from 'electron' +import { dictationVocabulary, type DictationTerm } from '../../shared/dictation-dictionary' import { createDictationTextPipeline, type DictationTextSettings @@ -44,19 +45,20 @@ export class SttService { this.state = createSttSessionState(modelManager, spawnWorker) } + /** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */ startDictation( modelId: string, sink: SttEventSink, - hotwordsFilePath?: string, + terms: readonly DictationTerm[] = [], owner = 'desktop', voice?: Partial ): Promise { - const clean = createDictationTextPipeline(getCatalogModel(modelId), voice) + const clean = createDictationTextPipeline(getCatalogModel(modelId), voice, terms) return startSttDictation( this.state, modelId, clean ? cleanTranscriptEvents(sink, clean) : sink, - hotwordsFilePath, + dictationVocabulary(terms), owner, voice?.dictationLanguage ) diff --git a/src/main/speech/stt-session-prewarm.ts b/src/main/speech/stt-session-prewarm.ts index afc4f7633e0..a4c0b6451e2 100644 --- a/src/main/speech/stt-session-prewarm.ts +++ b/src/main/speech/stt-session-prewarm.ts @@ -30,14 +30,9 @@ export async function prewarmSttModel( return } const manifest = getCatalogModel(modelId) - // Desktop and phone starts pass no hotwords, so this matches the recognizer they would load. const config: SttRecognizerConfig | null = manifest && isLocalSpeechModel(manifest) - ? { - modelId, - hotwordsFilePath: undefined, - language: engineDictationLanguage(manifest, language) - } + ? { modelId, language: engineDictationLanguage(manifest, language) } : null if (config && state.engine && sameSttRecognizerConfig(state.engine, config)) { return diff --git a/src/main/speech/stt-session-start.ts b/src/main/speech/stt-session-start.ts index 841396bf72a..5f80b0cd2f9 100644 --- a/src/main/speech/stt-session-start.ts +++ b/src/main/speech/stt-session-start.ts @@ -33,7 +33,7 @@ export async function startSttDictation( state: SttSessionState, modelId: string, sink: SttEventSink, - hotwordsFilePath?: string, + vocabulary: string[] = [], owner = 'desktop', language?: string ): Promise { @@ -48,7 +48,7 @@ export async function startSttDictation( } if (existing?.phase === 'stopping') { await existing.stopPromise - return startSttDictation(state, modelId, sink, hotwordsFilePath, owner, language) + return startSttDictation(state, modelId, sink, vocabulary, owner, language) } const manifest = getCatalogModel(modelId) @@ -58,7 +58,7 @@ export async function startSttDictation( const config: SttRecognizerConfig | null = manifest.provider !== 'local' ? null - : { modelId, hotwordsFilePath, language: engineDictationLanguage(manifest, language) } + : { modelId, language: engineDictationLanguage(manifest, language) } if (state.sessions.size >= MAX_STT_SESSIONS) { throw new Error('dictation_session_limit') } @@ -72,6 +72,7 @@ export async function startSttDictation( sink, config, modelId, + vocabulary, engine: null, cloud: null, phase: 'starting', @@ -153,18 +154,29 @@ async function acquireSttEngine( // Bound in the same step it is chosen, so an idle teardown never sees it unused. if (current && sameSttRecognizerConfig(current, config)) { clearSttIdleTeardownTimer(state) - session.engine = current - return current + return bindSttEngine(session, current) } if (!current) { - session.engine = spawnSttEngine(state, config, manifest) - return session.engine + return bindSttEngine(session, spawnSttEngine(state, config, manifest)) } // No session uses a recognizer with another config, so it is idle. await teardownSttEngine(state, current) } } +// The stream opens before this session can feed it, so all of its audio decodes with its vocabulary. +function bindSttEngine(session: SttSession, engine: SttEngine): SttEngine { + session.engine = engine + if (session.vocabulary.length > 0) { + engine.worker.postMessage({ + type: 'open', + streamId: session.streamId, + vocabulary: session.vocabulary + }) + } + return engine +} + export function spawnSttEngine( state: SttSessionState, config: SttRecognizerConfig, @@ -209,7 +221,6 @@ export function spawnSttEngine( modelDir: state.modelManager.getModelDir(config.modelId), sampleRate: manifest.sampleRate, files: manifest.files ?? [], - hotwordsFilePath: config.hotwordsFilePath, language: config.language }) return engine diff --git a/src/main/speech/stt-session-state.ts b/src/main/speech/stt-session-state.ts index 01a96842f93..21c081dacf9 100644 --- a/src/main/speech/stt-session-state.ts +++ b/src/main/speech/stt-session-state.ts @@ -9,7 +9,6 @@ export const MAX_STT_SESSIONS = 4 export type SttRecognizerConfig = { modelId: string - hotwordsFilePath: string | undefined language: string | undefined } @@ -31,6 +30,8 @@ export type SttSession = { /** null for cloud models, which load no recognizer. */ config: SttRecognizerConfig | null modelId: string + /** Spellings this dictation's stream tells the engine to expect. */ + vocabulary: string[] engine: SttEngine | null cloud: OpenAiTranscriptionSession | null phase: 'starting' | 'active' | 'stopping' @@ -70,11 +71,7 @@ export function createSttSessionState( } export function sameSttRecognizerConfig(a: SttRecognizerConfig, b: SttRecognizerConfig): boolean { - return ( - a.modelId === b.modelId && - a.hotwordsFilePath === b.hotwordsFilePath && - a.language === b.language - ) + return a.modelId === b.modelId && a.language === b.language } export function sttEngineUsers(state: SttSessionState, engine: SttEngine): SttSession[] { diff --git a/src/main/speech/stt-transcribe-cpp-recognizer.ts b/src/main/speech/stt-transcribe-cpp-recognizer.ts index f0bfefd74b0..24bed924fc2 100644 --- a/src/main/speech/stt-transcribe-cpp-recognizer.ts +++ b/src/main/speech/stt-transcribe-cpp-recognizer.ts @@ -1,4 +1,3 @@ -import { readFile } from 'node:fs/promises' import { availableParallelism } from 'node:os' import { join } from 'node:path' import { pathToFileURL } from 'node:url' @@ -14,18 +13,9 @@ function performanceCoreThreads(): number { return Math.min(logical, 8, Math.max(4, Math.floor(logical / 2))) } -// One term per line. The host deletes the file once dictation starts, so a CPU fallback's reload -// finds it gone and runs without. -async function readVocabulary(path: string | undefined): Promise { - const text = path ? await readFile(path, 'utf8').catch(() => '') : '' - return text - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) -} - // Totals, not the nominal rate ratio: per-feed resampling rounds, and the drift would replay decoded audio. type StreamCoverage = { + options: TranscribeCpp.TranscribeOptions chunker: OfflineAudioChunker inputSamples: number resampledSamples: number @@ -45,22 +35,29 @@ export async function createTranscribeCppRecognizer( const model = await transcribe.TranscribeModel.load(join(init.modelDir, gguf)) // The child runs one call at a time, so every stream can share one session. const session = model.createSession({ nThreads: performanceCoreThreads() }) - const vocabulary = await readVocabulary(init.hotwordsFilePath) const { languages } = model.capabilities // The setting is ISO-639-1; Nemotron names locales (ko-KR) and rejects the bare code. const language = init.language ? (languages.find((code) => code === init.language) ?? languages.find((code) => code.startsWith(`${init.language}-`))) : undefined - const options: TranscribeCpp.TranscribeOptions = { + const baseOptions: TranscribeCpp.TranscribeOptions = { timestamps: 'none', - ...(language ? { language } : {}), - ...(vocabulary.length > 0 && model.supports('vocabulary') ? { vocabulary } : {}) + ...(language ? { language } : {}) } + // A plain list as context, not the `vocabulary` option: in the 2026-11 dev bench that dropped + // Whisper's sentence-final periods. Whisper keeps a long prompt's end, so the first terms go last. + const streamOptions = (vocabulary: readonly string[]): TranscribeCpp.TranscribeOptions => + vocabulary.length > 0 && model.supports('context_prompt') + ? { ...baseOptions, prompt: `${vocabulary.toReversed().join(', ')}.` } + : baseOptions const modelRate = model.capabilities.nativeSampleRate || init.sampleRate const streams = new Map() - const transcribeChunk = async (chunk: Float32Array): Promise => { + const transcribeChunk = async ( + chunk: Float32Array, + options: TranscribeCpp.TranscribeOptions + ): Promise => { try { return (await session.run(chunk, options)).text } catch (error) { @@ -73,11 +70,11 @@ export async function createTranscribeCppRecognizer( } // The first run compiles GPU pipelines (seconds on Metal); pay it while loading, not on the first stop. - await transcribeChunk(new Float32Array(Math.round(modelRate / 2))) + await transcribeChunk(new Float32Array(Math.round(modelRate / 2)), baseOptions) // Coverage is in input-rate samples so the host can trim the audio it keeps for a CPU replay. const decode = async (streamId: string, stream: StreamCoverage, chunk: Float32Array) => { - const text = await transcribeChunk(chunk) + const text = await transcribeChunk(chunk, stream.options) stream.decodedSamples += chunk.length post({ type: 'chunk', @@ -87,18 +84,24 @@ export async function createTranscribeCppRecognizer( }) } + const openStream = (streamId: string, vocabulary: readonly string[]): StreamCoverage => { + const stream: StreamCoverage = { + options: streamOptions(vocabulary), + chunker: new OfflineAudioChunker(modelRate), + inputSamples: 0, + resampledSamples: 0, + decodedSamples: 0 + } + streams.set(streamId, stream) + return stream + } + return { + open(streamId, vocabulary) { + openStream(streamId, vocabulary) + }, async feed(streamId, samples, sampleRate) { - let stream = streams.get(streamId) - if (!stream) { - stream = { - chunker: new OfflineAudioChunker(modelRate), - inputSamples: 0, - resampledSamples: 0, - decodedSamples: 0 - } - streams.set(streamId, stream) - } + const stream = streams.get(streamId) ?? openStream(streamId, []) // The binding borrows PCM until the run settles; resampleToRate may return the caller's buffer. const resampled = new Float32Array( resampleToRate(samples, sampleRate || modelRate, modelRate) diff --git a/src/main/speech/stt-worker-cpu-fallback.ts b/src/main/speech/stt-worker-cpu-fallback.ts index 2f360f64951..3b32588abd7 100644 --- a/src/main/speech/stt-worker-cpu-fallback.ts +++ b/src/main/speech/stt-worker-cpu-fallback.ts @@ -12,6 +12,7 @@ export type SttGpuPolicy = { } type Feed = Extract +type Open = Extract type ChildMessage = | { type: 'chunk'; streamId: string; text: string; samples: number } | { @@ -23,6 +24,7 @@ type ChildMessage = /** A dictation's audio the GPU has not decoded yet, kept for a CPU replay. */ type StreamBacklog = { + open: Open | null feeds: Feed[] /** Input samples before feeds[0], in the stream's own count. */ start: number @@ -61,6 +63,8 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess { postMessage(msg: SttWorkerRequest): void { if (msg.type === 'init') { this.init = msg + } else if (msg.type === 'open' && this.canFallBack()) { + this.backlog(msg.streamId).open = msg } else if (msg.type === 'feed' && this.canFallBack()) { const stream = this.backlog(msg.streamId) stream.feeds.push(msg) @@ -90,7 +94,7 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess { private backlog(streamId: string): StreamBacklog { let stream = this.streams.get(streamId) if (!stream) { - stream = { feeds: [], start: 0, seconds: 0, pending: null } + stream = { open: null, feeds: [], start: 0, seconds: 0, pending: null } this.streams.set(streamId, stream) } return stream @@ -171,6 +175,9 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess { for (const [streamId, stream] of this.streams) { // A canceled dictation wants no text, so its audio is not replayed. if (stream.pending !== 'discard') { + if (stream.open) { + this.child.postMessage(stream.open) + } for (const feed of stream.feeds) { this.child.postMessage(feed) } diff --git a/src/main/speech/stt-worker-process.ts b/src/main/speech/stt-worker-process.ts index f3cfd895b27..b9a4eadea93 100644 --- a/src/main/speech/stt-worker-process.ts +++ b/src/main/speech/stt-worker-process.ts @@ -11,6 +11,7 @@ export const STT_WORKER_HUNG_ERROR = 'The speech engine stopped responding. Try export type SttWorkerRequest = | ({ type: 'init' } & SttRecognizerInit) + | { type: 'open'; streamId: string; vocabulary: string[] } | { type: 'feed'; streamId: string; samples: Float32Array; sampleRate: number } | { type: 'stop'; streamId: string } | { type: 'discard'; streamId: string } diff --git a/src/main/speech/stt-worker.ts b/src/main/speech/stt-worker.ts index 6922c958303..4e9eb66a22e 100644 --- a/src/main/speech/stt-worker.ts +++ b/src/main/speech/stt-worker.ts @@ -106,6 +106,8 @@ function handleMessage( switch (msg.type) { case 'init': return handleInit(msg) + case 'open': + return recognizer?.open(msg.streamId, msg.vocabulary) case 'feed': return handleFeed(msg) case 'stop': diff --git a/src/preload/api/speech-api.ts b/src/preload/api/speech-api.ts index dc5d63bb3ca..9197ed1883b 100644 --- a/src/preload/api/speech-api.ts +++ b/src/preload/api/speech-api.ts @@ -1,5 +1,6 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' import type { + DictationContext, OpenAiCompatibleSpeechProvider, OpenAiCompatibleSpeechProviderSummary, SpeechErrorEvent, @@ -28,7 +29,7 @@ export type SpeechApi = { deleteModel: (modelId: string) => Promise startDictation: ( modelId: string, - hotwords: string[] | undefined, + context: DictationContext | undefined, sessionId: string ) => Promise /** Resolves true when this chunk filled a cloud session's upload cap. */ diff --git a/src/preload/api/speech-bridge.ts b/src/preload/api/speech-bridge.ts index 4e751f13d75..4ec6ad72ac2 100644 --- a/src/preload/api/speech-bridge.ts +++ b/src/preload/api/speech-bridge.ts @@ -1,6 +1,7 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' import { ipcRenderer } from 'electron' import type { + DictationContext, OpenAiCompatibleSpeechProvider, OpenAiCompatibleSpeechProviderSummary, SpeechAudioPortMessage, @@ -40,7 +41,7 @@ export const speechApi = { ipcRenderer.invoke('speech:deleteModel', modelId), startDictation: async ( modelId: string, - hotwords: string[] | undefined, + context: DictationContext | undefined, sessionId: string ): Promise => { const { port1, port2 } = new MessageChannel() @@ -51,7 +52,7 @@ export const speechApi = { const result: { directAudio?: boolean } | undefined = await ipcRenderer.invoke( 'speech:startDictation', modelId, - hotwords, + context, sessionId ) directAudio = result?.directAudio === true diff --git a/src/renderer/src/components/dictation/DictationController.tsx b/src/renderer/src/components/dictation/DictationController.tsx index f4dc3ab93ea..9c4957044bc 100644 --- a/src/renderer/src/components/dictation/DictationController.tsx +++ b/src/renderer/src/components/dictation/DictationController.tsx @@ -3,6 +3,7 @@ import { useAppStore } from '@/store' import { useAudioCapture } from '@/hooks/use-audio-capture' import { DictationIndicator } from './DictationIndicator' import { captureInsertionTarget } from './dictation-insertion-target' +import { captureDictationContext } from './dictation-context' import { dictationPillAnchorFor, type DictationPillAnchor } from './dictation-pill-anchor' import { canRetryDictation, @@ -207,7 +208,8 @@ export function DictationController() { const insertionTarget = captureInsertionTarget() const deliver = createDictationTranscriptDelivery() deliverRef.current = (text) => deliver(text, insertionTarget) - const kept = createDictationKeptRecording(insertionTarget, deliver) + const context = captureDictationContext() + const kept = createDictationKeptRecording(insertionTarget, deliver, context) keptRef.current = kept setRecordingKey((key) => key + 1) setCaptureLive(false) @@ -270,7 +272,7 @@ export function DictationController() { } setCaptureLive(true) - await window.api.speech.startDictation(modelId, undefined, sessionId) + await window.api.speech.startDictation(modelId, context, sessionId) if (!isCurrent()) { await releaseStart(true) return diff --git a/src/renderer/src/components/dictation/dictation-context.ts b/src/renderer/src/components/dictation/dictation-context.ts new file mode 100644 index 00000000000..6d42a188fa2 --- /dev/null +++ b/src/renderer/src/components/dictation/dictation-context.ts @@ -0,0 +1,18 @@ +import { useAppStore } from '@/store' +import { findRepoForHost } from '@/store/repos/repo-host-identity' +import { getRepoIdFromWorktreeId } from '../../../../shared/worktree/id' +import type { DictationContext } from '../../../../shared/speech-types' + +/** The worktree a dictation starts in, with its repo's terms for an engine running in this app. */ +export function captureDictationContext(): DictationContext { + const state = useAppStore.getState() + const worktreeId = state.activeWorktreeId + if (!worktreeId) { + return {} + } + const repo = findRepoForHost(state.repos, getRepoIdFromWorktreeId(worktreeId), { + hostId: state.activeWorkspaceExecutionHostId, + settings: state.settings + }) + return { worktreeId, ...(repo?.dictationTerms ? { repoTerms: repo.dictationTerms } : {}) } +} diff --git a/src/renderer/src/components/dictation/dictation-kept-recording.ts b/src/renderer/src/components/dictation/dictation-kept-recording.ts index 9853bb5b438..f0b7ca42b0d 100644 --- a/src/renderer/src/components/dictation/dictation-kept-recording.ts +++ b/src/renderer/src/components/dictation/dictation-kept-recording.ts @@ -2,6 +2,7 @@ import { CLOUD_DICTATION_MAX_SECONDS, DICTATION_SAMPLE_RATE } from '../../../../shared/rpc-contract/speech-params' +import type { DictationContext } from '../../../../shared/speech-types' import type { DictationInsertionTarget } from './dictation-insertion-target' import type { DictationTranscriptDelivery } from './dictation-transcript-delivery' import { trimLandedOverlap } from './dictation-retry-overlap' @@ -17,11 +18,14 @@ export type DictationKeptRecording = { complete: boolean target: DictationInsertionTarget | null deliver: DictationTranscriptDelivery + /** A retry spells with the dictionary of the place it started in. */ + context: DictationContext } export function createDictationKeptRecording( target: DictationInsertionTarget | null, - deliver: DictationTranscriptDelivery + deliver: DictationTranscriptDelivery, + context: DictationContext ): DictationKeptRecording & { keep: (samples: Float32Array) => void /** The host stopped accepting audio at its own cap, so the first ten minutes are all it would hear. */ @@ -35,6 +39,7 @@ export function createDictationKeptRecording( complete: true, target, deliver, + context, keep: (frame: Float32Array): void => { const room = MAX_KEPT_SAMPLES - samples if (room <= 0) { @@ -111,7 +116,7 @@ export async function transcribeKeptRecording(args: { const retried: string[] = [] args.routeFinals((text) => retried.push(text)) try { - await window.api.speech.startDictation(args.modelId, undefined, sessionId) + await window.api.speech.startDictation(args.modelId, args.recording.context, sessionId) for (const block of joinIntoBlocks(args.recording.chunks)) { if (!isCurrent()) { break diff --git a/src/renderer/src/store/repos/repo-state.ts b/src/renderer/src/store/repos/repo-state.ts index b03fd65d884..683001aec29 100644 --- a/src/renderer/src/store/repos/repo-state.ts +++ b/src/renderer/src/store/repos/repo-state.ts @@ -48,6 +48,7 @@ export type RepoUpdate = Partial< | 'worktreeBasePath' | 'kind' | 'symlinkPaths' + | 'dictationTerms' | 'issueSourcePreference' | 'defaultJiraProject' | 'forkSyncMode' diff --git a/src/renderer/src/web/preload-api/web-speech-api.ts b/src/renderer/src/web/preload-api/web-speech-api.ts index f90437d21ee..5716ae8e426 100644 --- a/src/renderer/src/web/preload-api/web-speech-api.ts +++ b/src/renderer/src/web/preload-api/web-speech-api.ts @@ -148,8 +148,8 @@ export function createWebSpeechApi(): Pick< } return { - // No model id, like the phone: the host dictates with the model it has selected now. - startDictation: async (_modelId, _hotwords, sessionId) => { + // No model id, like the phone: the host dictates with the model it has selected now, and its own terms for the worktree. + startDictation: async (_modelId, context, sessionId) => { const dictation: HostDictation = { sessionId, dictationId: `web-${createNonSecureContextUuid()}`, @@ -164,7 +164,10 @@ export function createWebSpeechApi(): Pick< } active = dictation try { - await callRuntimeResult('speech.dictation.start', { dictationId: dictation.dictationId }) + await callRuntimeResult('speech.dictation.start', { + dictationId: dictation.dictationId, + ...(context?.worktreeId ? { worktreeId: context.worktreeId } : {}) + }) // A cancel that landed while the host was starting couldn't reach this dictation yet. if (active !== dictation) { void cancelOnHost(dictation) diff --git a/src/shared/dictation-dictionary.test.ts b/src/shared/dictation-dictionary.test.ts new file mode 100644 index 00000000000..dc7d86ee74c --- /dev/null +++ b/src/shared/dictation-dictionary.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest' +import { createDictationDictionary, mergeDictationTerms } from './dictation-dictionary' +import { createDictationTextPipeline } from './dictation-text-pipeline' + +const spell = (terms: Parameters[0], text: string): string => + createDictationDictionary(terms)!(text) + +describe('createDictationDictionary', () => { + // Parakeet v3's real output for the dev bench's code-term clips. + it('joins the words a model splits a code term into back into the term', () => { + const terms = [ + { term: 'useAudioCapture' }, + { term: 'usePcmWorklet' }, + { term: 'stt-worker.ts' }, + { term: 'numThreads' }, + { term: 'useMemo' }, + { term: 'onChange' } + ] + expect( + spell(terms, 'Rename use audio capture to use PCM worklet in the dictation controller.') + ).toBe('Rename useAudioCapture to usePcmWorklet in the dictation controller.') + expect(spell(terms, 'Open stt worker TS and set num threads to 4.')).toBe( + 'Open stt-worker.ts and set numThreads to 4.' + ) + expect(spell(terms, 'Wrap it in use memo and pass the on-change call back.')).toBe( + 'Wrap it in useMemo and pass the onChange call back.' + ) + }) + + it('replaces what the model hears instead, and fixes the spelling of the term itself', () => { + const terms = [{ term: 'kubectl', heardAs: 'Quebecal' }, { term: 'Orca' }, { term: 'pnpm' }] + expect(spell(terms, 'Use Quebecal to scale it.')).toBe('Use kubectl to scale it.') + expect(spell(terms, "Open or ca's settings and run PNPM.")).toBe( + "Open Orca's settings and run pnpm." + ) + }) + + it('leaves longer words and words split by a sentence end alone', () => { + const terms = [{ term: 'React' }, { term: 'onChange' }] + expect(spell(terms, 'Reactive state. Turn it on. Change it.')).toBe( + 'Reactive state. Turn it on. Change it.' + ) + }) + + it('replaces a "heard as" inside text written without spaces', () => { + expect(spell([{ term: '光輝', heardAs: 'こうき' }], 'こうきさんに送って')).toBe( + '光輝さんに送って' + ) + }) + + it("prefers a repo's spelling over a personal one", () => { + const terms = mergeDictationTerms([{ term: 'RPC' }], [{ term: 'rpc' }]) + expect(spell(terms, 'Call the rpc.')).toBe('Call the RPC.') + }) +}) + +describe('createDictationTextPipeline with terms', () => { + it('spells a lowercase term after a sentence-start filler goes', () => { + const clean = createDictationTextPipeline({ language: 'en' }, {}, [{ term: 'pnpm' }])! + expect(clean('Um, pnpm test.')).toBe('pnpm test.') + }) +}) diff --git a/src/shared/dictation-dictionary.ts b/src/shared/dictation-dictionary.ts new file mode 100644 index 00000000000..b4ea439e9a4 --- /dev/null +++ b/src/shared/dictation-dictionary.ts @@ -0,0 +1,119 @@ +/** A word or phrase dictation should spell `term`'s way, optionally also replacing what the model hears instead. */ +export type DictationTerm = { term: string; heardAs?: string } + +export const MAX_DICTATION_TERMS = 500 +export const MAX_DICTATION_TERM_LENGTH = 100 + +const WORD_RE = /[\p{L}\p{M}\p{N}]+/gu +// Spaces, or one joiner as in "on-change" or "stt-worker.ts": what may sit between the words of one term. +const JOINING_GAP_RE = /^(?:\s+|[-_./])$/u +// Scripts written without spaces, where a phrase can only be found inside a longer run of letters. +const UNSPACED_SCRIPT_RE = + /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u +const MAX_WORDS_PER_TERM = 8 + +// "Use Audio Capture", "use-audio-capture" and "useAudioCapture" all key as "useaudiocapture". +function wordKey(text: string): string { + return (text.normalize('NFKC').toLowerCase().match(WORD_RE) ?? []).join('') +} + +/** Well-formed entries only, trimmed and capped; undefined when the value isn't a list. */ +export function normalizeDictationTerms(value: unknown): DictationTerm[] | undefined { + if (!Array.isArray(value)) { + return undefined + } + const terms: DictationTerm[] = [] + for (const entry of value) { + const term: unknown = entry?.term + const heardAs: unknown = entry?.heardAs + if (typeof term !== 'string' || !term.trim()) { + continue + } + const alias = typeof heardAs === 'string' ? heardAs.trim() : '' + terms.push({ + term: term.trim().slice(0, MAX_DICTATION_TERM_LENGTH), + ...(alias ? { heardAs: alias.slice(0, MAX_DICTATION_TERM_LENGTH) } : {}) + }) + } + return terms.slice(0, MAX_DICTATION_TERMS) +} + +/** Repo terms first, so a repo's spelling wins over a personal one for the same words. */ +export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown): DictationTerm[] { + return [ + ...(normalizeDictationTerms(repoTerms) ?? []), + ...(normalizeDictationTerms(personalTerms) ?? []) + ] +} + +/** Rewrites words that key the same as a term (or its "heard as") to the term's spelling; null when there are none. */ +export function createDictationDictionary( + terms: readonly DictationTerm[] | undefined +): ((text: string) => string) | null { + const byKey = new Map() + const unspacedAliases: [heardAs: string, term: string][] = [] + // A "heard as" is what the user asked for, so it beats another term spelled the same. + for (const { term, heardAs } of terms ?? []) { + if (heardAs && UNSPACED_SCRIPT_RE.test(heardAs)) { + unspacedAliases.push([heardAs, term]) + } else if (heardAs && wordKey(heardAs) && !byKey.has(wordKey(heardAs))) { + byKey.set(wordKey(heardAs), term) + } + } + for (const { term } of terms ?? []) { + const key = wordKey(term) + if (key && !byKey.has(key)) { + byKey.set(key, term) + } + } + if (byKey.size === 0 && unspacedAliases.length === 0) { + return null + } + const longestKey = Math.max(0, ...[...byKey.keys()].map((key) => key.length)) + return (text) => { + for (const [alias, term] of unspacedAliases) { + text = text.split(alias).join(term) + } + return byKey.size > 0 ? replaceWordRuns(text, byKey, longestKey) : text + } +} + +// At each word, the longest run of joined words whose key is a term's becomes that term. +function replaceWordRuns(text: string, byKey: Map, longestKey: number): string { + const words = [...text.matchAll(WORD_RE)] + let out = '' + let cursor = 0 + for (let start = 0; start < words.length;) { + let key = '' + let match: { end: number; term: string } | null = null + for (let end = start; end < words.length && end - start < MAX_WORDS_PER_TERM; end++) { + if (end > start) { + const gapStart = words[end - 1].index + words[end - 1][0].length + if (!JOINING_GAP_RE.test(text.slice(gapStart, words[end].index))) { + break + } + } + key += wordKey(words[end][0]) + if (key.length > longestKey) { + break + } + const term = byKey.get(key) + if (term !== undefined) { + match = { end, term } + } + } + if (!match) { + start++ + continue + } + out += text.slice(cursor, words[start].index) + match.term + cursor = words[match.end].index + words[match.end][0].length + start = match.end + 1 + } + return out + text.slice(cursor) +} + +/** The distinct spellings an engine that takes a context prompt is told to expect. */ +export function dictationVocabulary(terms: readonly DictationTerm[] | undefined): string[] { + return [...new Set((terms ?? []).map((entry) => entry.term))] +} diff --git a/src/shared/dictation-text-pipeline.ts b/src/shared/dictation-text-pipeline.ts index a1fbb39d319..b348cf00909 100644 --- a/src/shared/dictation-text-pipeline.ts +++ b/src/shared/dictation-text-pipeline.ts @@ -1,3 +1,4 @@ +import { createDictationDictionary, type DictationTerm } from './dictation-dictionary' import { engineDictationLanguage } from './dictation-language' import { removeFillerWords } from './dictation-filler-words' import type { SpeechModelManifest, VoiceSettings } from './speech-types' @@ -26,11 +27,18 @@ function dictatedTextLanguages( /** The passes dictated text goes through before anyone sees it, or null when every pass is off. */ export function createDictationTextPipeline( model: DictationTextModel | undefined, - settings: Partial | undefined + settings: Partial | undefined, + terms?: readonly DictationTerm[] ): ((text: string) => string) | null { - if (settings?.removeFillerWords === false) { - return null - } const languages = dictatedTextLanguages(model, settings?.dictationLanguage) - return (text) => removeFillerWords(text, languages) + const dropFillers = + settings?.removeFillerWords === false + ? null + : (text: string) => removeFillerWords(text, languages) + const spellTerms = createDictationDictionary(terms) + if (!dropFillers || !spellTerms) { + return dropFillers ?? spellTerms + } + // Terms last, so a filler removed at a sentence start can't recapitalize a lowercase term like "pnpm". + return (text) => spellTerms(dropFillers(text)) } diff --git a/src/shared/repo-types.ts b/src/shared/repo-types.ts index 2c32a25aeb5..74e54601be9 100644 --- a/src/shared/repo-types.ts +++ b/src/shared/repo-types.ts @@ -1,3 +1,4 @@ +import type { DictationTerm } from './dictation-dictionary' import type { RepoIcon } from './repo-icon' import type { GhAccountBinding } from './github/account-binding' import type { GitHubRepositoryIdentity } from './github/pull-request-types' @@ -103,6 +104,8 @@ export type Repo = { projectHostSetupMethod?: RepoProjectHostSetupMethod /** Local row a WSL host now owns, kept for a downgrade; set only by the mover's retention pass. */ orcadRetained?: OrcadRetainedMarker + /** Dictation terms used only in this repo, kept in Orca (on the repo's host), never in the checkout. */ + dictationTerms?: DictationTerm[] } /** Shared IPC shape for repos:getBaseRefDefault across main/preload/renderer; remoteCount avoids a second hint lookup. */ diff --git a/src/shared/rpc-contract/repo-update-params.ts b/src/shared/rpc-contract/repo-update-params.ts index 2ebefc43506..4ec980ee244 100644 --- a/src/shared/rpc-contract/repo-update-params.ts +++ b/src/shared/rpc-contract/repo-update-params.ts @@ -10,6 +10,7 @@ import { } from '../worktree/visibility-sources' import { OptionalFiniteNumber, OptionalString } from './rpc-param-primitives' import { editorFormatterSchema } from '../editor-formatter' +import { normalizeDictationTerms } from '../dictation-dictionary' export const RepoSourceControlAiOverrides = z .unknown() @@ -57,6 +58,7 @@ export function createRepoUpdateSchema