feat(speech): spell dictionary terms in dictation, with repo terms and Whisper prompt

This commit is contained in:
Kelvin Amoaba
2026-11-09 19:08:05 +00:00
parent d291670a01
commit b265a96d8b
48 changed files with 484 additions and 144 deletions
@@ -36,6 +36,8 @@ export const dictationSetupSchema = z.looseObject({
dictationHoldToRecord: salvagedOptional('dictationHoldToRecord', z.boolean()),
// Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default.
removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()),
// Absent from older hosts; the phone normalizes it before spelling with it.
dictationTerms: salvagedOptional('dictationTerms', z.unknown()),
models: salvagingArray(speechModelSchema)
})
@@ -2,6 +2,7 @@ import { createContext, useContext, useEffect, useState, useSyncExternalStore }
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
import type { MobileDictationAwayTarget } from './mobile-dictation-away-insert'
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
export type MobileDictationHostToast = { message: string; open?: () => void }
@@ -20,6 +21,10 @@ export type MobileDictationOwner = {
readonly paneKey: string | undefined
readonly dictationMode: string | undefined
readonly removeFillerWords: boolean
// The computer's terms for this worktree, for words the phone recognizes itself.
readonly dictationTerms: readonly DictationTerm[] | undefined
// The computer spells its own words with this worktree's repo terms.
readonly worktreeId: string
}
// A mounted session screen, as the dictation host sees it.
@@ -100,6 +100,8 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
client: owner?.client ?? null,
enabled: true,
removeFillerWords: owner?.removeFillerWords ?? true,
dictationTerms: owner?.dictationTerms,
worktreeId: owner?.worktreeId,
onTranscript: (text) => {
const current = ownerRef.current
if (!current) {
+12 -5
View File
@@ -126,8 +126,12 @@ export function isDictationSetupRequiredError(message: string): boolean {
return SETUP_REQUIRED_CODES.has(message) || message.startsWith('voice_model_not_ready:')
}
export async function fetchDictationSetup(client: RpcOperationSender): Promise<MobileSpeechSetup> {
const reply = await requestDictationSetupReply(client)
/** `worktreeId` asks for that repo's dictionary terms too. */
export async function fetchDictationSetup(
client: RpcOperationSender,
worktreeId?: string
): Promise<MobileSpeechSetup> {
const reply = await requestDictationSetupReply(client, worktreeId ? { worktreeId } : null)
if (isLegacyDesktopSpeechSetupReply(reply)) {
throw new Error(
translate(
@@ -145,15 +149,18 @@ export async function fetchDictationSetup(client: RpcOperationSender): Promise<M
)
}
async function requestDictationSetupReply(client: RpcOperationSender): Promise<RpcResponse> {
async function requestDictationSetupReply(
client: RpcOperationSender,
params: { worktreeId: string } | null
): Promise<RpcResponse> {
try {
return await dictationSetupRead.request(client, null)
return await dictationSetupRead.request(client, params)
} catch (error) {
if (!(error instanceof LogicalClientCutoverError)) {
throw error
}
// Replay this read on authenticated replacement; mutations must surface cutover to avoid replaying unknown commits.
return dictationSetupRead.request(client, null)
return dictationSetupRead.request(client, params)
}
}
@@ -15,6 +15,7 @@ import type { RpcClient } from '../transport/rpc/rpc-client-contract'
type StartMobileDictationDesktopSessionOptions = {
client: RpcClient
dictationId: string
worktreeId: string | undefined
generation: number
getCurrentGeneration: () => number
getEnabled: () => boolean
@@ -102,7 +103,7 @@ export function startMobileDictationDesktopSession(
}
async function startOnHost(options: StartMobileDictationDesktopSessionOptions): Promise<boolean> {
const { client, dictationId } = options
const { client, dictationId, worktreeId } = options
let hostDedupesResends = false
try {
@@ -111,7 +112,7 @@ async function startOnHost(options: StartMobileDictationDesktopSessionOptions):
client,
dictationSessionStart.request(
client,
{ dictationId },
{ dictationId, ...(worktreeId ? { worktreeId } : {}) },
{ timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS }
)
).catch(() => {
@@ -48,6 +48,7 @@ export type MobileDictationRun = {
readonly finish: RefObject<(emptyTranscriptMessage: string) => Promise<void>>
readonly onTranscript: RefObject<(text: string) => void>
readonly onAudioLimitReached: RefObject<(() => void) | undefined>
readonly worktreeId: RefObject<string | undefined>
readonly keptTranscript: MobileDictationTranscriptSoFar
// keepText: leave the words so far on screen (recording, finishing, or a failure kept for Retry).
readonly applyStatus: (next: DictationStatus, keepText?: boolean) => void
@@ -22,6 +22,7 @@ export async function startMobileDictationOnHost(
await startMobileDictationDesktopSession({
client,
dictationId,
worktreeId: run.worktreeId.current,
generation,
getCurrentGeneration: () => run.generation.current,
getEnabled: () => run.enabled.current,
@@ -20,6 +20,10 @@ export type UseMobileDictationOptions = {
onError?: (error: Error, outcome: { kept: boolean }) => void
// The computer's filler switch, for text the phone recognizes itself; unknown means on.
removeFillerWords?: boolean
// The computer's dictionary for this worktree, unvalidated, for text the phone recognizes itself.
dictationTerms?: unknown
// Lets the computer pick this worktree's repo terms for text it recognizes.
worktreeId?: string
// The host's cloud upload is full; the dictation stops and inserts what was kept.
onAudioLimitReached?: () => void
}
+6 -2
View File
@@ -36,9 +36,10 @@ export type { UseMobileDictationResult } from './mobile-dictation-session-state'
export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult {
useMobileTranslation()
const { client, enabled, onTranscript, onError, removeFillerWords, onAudioLimitReached } = options
const { dictationTerms, worktreeId } = options
// Both capture seams hold the screen while the microphone is open; native uses the device directly and page uses shell audio verbs.
const capture = useDictationCapture()
const speech = useOnDeviceSpeech(removeFillerWords)
const speech = useOnDeviceSpeech(removeFillerWords, dictationTerms)
const [status, setStatus] = useState<DictationStatus>('idle')
const [error, setError] = useState<string | null>(null)
// A failure whose recording is kept for Retry; cleared by Retry, discard and the next dictation.
@@ -61,6 +62,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
const onTranscriptRef = useRef(onTranscript)
const onErrorRef = useRef(onError)
const onAudioLimitReachedRef = useRef(onAudioLimitReached)
const worktreeIdRef = useRef(worktreeId)
const pendingChunksRef = useRef<Set<Promise<void>>>(new Set())
const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget())
const audioPhaseRef = useRef<MobileDictationAudioPhase>('closed')
@@ -79,7 +81,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
onTranscriptRef.current = onTranscript
onErrorRef.current = onError
onAudioLimitReachedRef.current = onAudioLimitReached
}, [client, speech, enabled, onTranscript, onError, onAudioLimitReached])
worktreeIdRef.current = worktreeId
}, [client, speech, enabled, onTranscript, onError, onAudioLimitReached, worktreeId])
const applyStatus = useCallback(
// Kept through processing so the text so far stays in place until the final transcript replaces it in the same render.
@@ -146,6 +149,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
finish: finishRef,
onTranscript: onTranscriptRef,
onAudioLimitReached: onAudioLimitReachedRef,
worktreeId: worktreeIdRef,
keptTranscript,
applyStatus,
showEngine,
+4 -1
View File
@@ -1,6 +1,9 @@
import type { OnDeviceSpeech } from './on-device-speech-contract'
// The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer.
export function useOnDeviceSpeech(_removeFillerWords?: boolean): OnDeviceSpeech | null {
export function useOnDeviceSpeech(
_removeFillerWords?: boolean,
_dictationTerms?: unknown
): OnDeviceSpeech | null {
return null
}
+20 -12
View File
@@ -1,4 +1,5 @@
import { useLayoutEffect, useMemo, useRef } from 'react'
import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary'
import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline'
import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
import type { OnDeviceSpeech } from './on-device-speech-contract'
@@ -9,25 +10,29 @@ const keepText = (text: string): string => text
// The phone's recognizer transcribes only the locale it started in, so that locale's language picks the fillers, as the host does for its model.
function phoneDictationTextCleaner(
locale: string | null,
removeFillerWords: boolean
removeFillerWords: boolean,
dictationTerms: unknown
): (text: string) => string {
const language = locale?.split('-')[0].toLowerCase()
const clean = createDictationTextPipeline(language ? { language } : undefined, {
removeFillerWords
})
const clean = createDictationTextPipeline(
language ? { language } : undefined,
{ removeFillerWords },
normalizeDictationTerms(dictationTerms)
)
return clean ?? keepText
}
function createPageOnDeviceSpeech(
verbs: NativeVerbs,
removeFillerWords: () => boolean
textSettings: () => { removeFillerWords: boolean; dictationTerms: unknown }
): OnDeviceSpeech {
let clean = keepText
return {
status: () => verbs.speechStatus(onDeviceSpeechLocale()),
start: async () => {
const started = await verbs.speechStart(onDeviceSpeechLocale())
clean = phoneDictationTextCleaner(started.locale, removeFillerWords())
const { removeFillerWords, dictationTerms } = textSettings()
clean = phoneDictationTextCleaner(started.locale, removeFillerWords, dictationTerms)
return started
},
read: async () => {
@@ -46,17 +51,20 @@ function createPageOnDeviceSpeech(
}
// Null on a shell without the verb, so dictation stays on the computer exactly as before.
// removeFillerWords is the computer's switch, read at each start.
export function useOnDeviceSpeech(removeFillerWords = true): OnDeviceSpeech | null {
// The computer's filler switch and dictionary for this worktree, read at each start.
export function useOnDeviceSpeech(
removeFillerWords = true,
dictationTerms?: unknown
): OnDeviceSpeech | null {
const verbs = useNativeVerbs()
const removeFillerWordsRef = useRef(removeFillerWords)
const textSettingsRef = useRef({ removeFillerWords, dictationTerms })
useLayoutEffect(() => {
removeFillerWordsRef.current = removeFillerWords
}, [removeFillerWords])
textSettingsRef.current = { removeFillerWords, dictationTerms }
}, [removeFillerWords, dictationTerms])
return useMemo(
() =>
verbs.canRecognizeSpeech
? createPageOnDeviceSpeech(verbs, () => removeFillerWordsRef.current)
? createPageOnDeviceSpeech(verbs, () => textSettingsRef.current)
: null,
[verbs]
)
@@ -8,14 +8,17 @@ import {
fetchDictationSetup
} from '../dictation/mobile-dictation-setup'
import type { MobileSessionFeedbackCapabilitiesModel } from './use-mobile-session-feedback-capabilities'
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
// The computer's dictation settings as this session reads them: mic shown, mode, filler switch, and whether setup is needed first.
export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapabilitiesModel) {
const { client, connState, setShowDictationSetup, setDictationMode, setDictationEnabled } = scope
const { client, connState, worktreeId, setShowDictationSetup, setDictationMode } = scope
const { setDictationEnabled } = scope
// The phone's own recognizer needs no computer setup, and chat inserts its words on the phone.
const phoneDictation = usePhoneDictationReady()
// The host's filler switch, kept from its last setup read; on until one says otherwise.
// The host's filler switch and this worktree's terms, kept from its last setup read; fillers on until one says otherwise.
const [removeFillerWords, setRemoveFillerWords] = useState(true)
const [dictationTerms, setDictationTerms] = useState<DictationTerm[] | undefined>(undefined)
// Only a host setup read sets this; while false, the host's start refusal decides (older hosts, first tap before the read).
const dictationNeedsSetupRef = useRef(false)
@@ -27,17 +30,18 @@ export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapab
return
}
try {
const setup = await fetchDictationSetup(client)
const setup = await fetchDictationSetup(client, worktreeId)
if (read === dictationSettingsReadRef.current) {
setDictationMode(setup.dictationMode)
setRemoveFillerWords(setup.removeFillerWords !== false)
setDictationTerms(setup.dictationTerms)
setDictationEnabled(!dictationMicHidden(setup))
dictationNeedsSetupRef.current = dictationNeedsSetup(setup)
}
} catch {
// Keep the last settings when an older host cannot report them.
}
}, [client, connState, phoneDictation.refresh, setDictationEnabled, setDictationMode])
}, [client, connState, phoneDictation.refresh, setDictationEnabled, setDictationMode, worktreeId])
// The sheet may have changed host setup: let the host judge taps until the fresh read lands.
const closeDictationSetup = useCallback(() => {
@@ -65,6 +69,7 @@ export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapab
return {
phoneDictation,
removeFillerWords,
dictationTerms,
dictationNeedsSetupRef,
closeDictationSetup,
refreshDictationMode
@@ -185,6 +185,7 @@ export function useMobileSessionDictation(
const {
phoneDictation,
removeFillerWords,
dictationTerms,
dictationNeedsSetupRef,
closeDictationSetup,
refreshDictationMode
@@ -234,7 +235,9 @@ export function useMobileSessionDictation(
executionHost
),
dictationMode: scope.dictationMode,
removeFillerWords
removeFillerWords,
dictationTerms,
worktreeId
})
}, [
activeHandle,
@@ -250,6 +253,7 @@ export function useMobileSessionDictation(
instance,
phoneDictation.readyRef,
removeFillerWords,
dictationTerms,
scope.dictationMode,
sessionKey,
setShowDictationSetup,
+10
View File
@@ -17,6 +17,7 @@ import {
} from '../../../shared/worktree/visibility-sources'
import { prepareLocalWorktreeRootForRepo } from '../../workspaces/creation/worktree-root-preparation'
import { commitProvenRepoKindChange } from '../../workspaces/catalog/folder-repo-git-conversion'
import { normalizeDictationTerms } from '../../../shared/dictation-dictionary'
import { invalidateAuthorizedRootsCache } from '../worktrees/removal/registered-worktree-roots-cache'
import { notifyReposChanged } from './repos-changed-notification'
import {
@@ -45,6 +46,7 @@ export function registerRepoUpdateHandler(mainWindow: BrowserWindow, store: Stor
| 'worktreeBasePath'
| 'kind'
| 'symlinkPaths'
| 'dictationTerms'
| 'issueSourcePreference'
| 'forkSyncMode'
| 'agentWorkspaceTrust'
@@ -110,6 +112,14 @@ export function registerRepoUpdateHandler(mainWindow: BrowserWindow, store: Stor
delete updates.symlinkPaths
}
}
if ('dictationTerms' in updates) {
const terms = normalizeDictationTerms(updates.dictationTerms)
if (terms) {
updates.dictationTerms = terms
} else {
delete updates.dictationTerms
}
}
if ('worktreeBasePath' in updates && updates.worktreeBasePath !== undefined) {
const v = updates.worktreeBasePath as unknown
if (typeof v !== 'string') {
+4 -39
View File
@@ -6,9 +6,8 @@ import {
type MessagePortMain
} from 'electron'
import { abortWhenRendererGone } from './renderer-lifetime-abort'
import { join } from 'node:path'
import { writeFile, unlink } from 'node:fs/promises'
import { createHash } from 'node:crypto'
import { mergeDictationTerms } from '../../shared/dictation-dictionary'
import type { DictationContext } from '../../shared/speech-types'
import { SPEECH_MODEL_CATALOG } from '../speech/model-catalog'
import { deleteLocalSpeechModel } from '../speech/speech-model-deletion'
import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service'
@@ -112,29 +111,6 @@ export function registerSpeechHandlers(store: Store): void {
})
})
const getHotwordsFilePath = (content: string): string => {
const digest = createHash('sha256').update(content).digest('hex').slice(0, 12)
// Native speech libraries may open paths with ANSI Windows APIs; keep hotwords in the ASCII-safe model cache.
return join(getSpeechModelManager(store).getModelsDir(), `speech-hotwords-${digest}.txt`)
}
const hotwordFileUsers = new Map<string, { users: number; written: boolean }>()
const retainHotwordsFile = (path: string): ((written: boolean) => void) => {
const entry = hotwordFileUsers.get(path) ?? { users: 0, written: false }
entry.users += 1
hotwordFileUsers.set(path, entry)
return (written) => {
entry.written ||= written
entry.users -= 1
if (entry.users === 0) {
hotwordFileUsers.delete(path)
if (entry.written) {
unlink(path).catch(() => {})
}
}
}
}
// Renderer run ids restart per document; a retired document's owner must not be reused by its successor.
const retiredDocumentCounts = new Map<number, number>()
const getDesktopOwner = (senderId: number, sessionId: string): string =>
@@ -153,7 +129,7 @@ export function registerSpeechHandlers(store: Store): void {
ipcMain.handle(
'speech:startDictation',
async (event, modelId: string, hotwords?: string[], sessionId = 'desktop') => {
async (event, modelId: string, context?: DictationContext, sessionId = 'desktop') => {
const senderId = event.sender.id
const owner = getDesktopOwner(senderId, sessionId)
const audioPort = pendingAudioPorts.get(owner)
@@ -164,8 +140,6 @@ export function registerSpeechHandlers(store: Store): void {
audioPort?.close()
return { directAudio }
}
let resolvedHotwordsPath: string | undefined
let releaseHotwords: ((written: boolean) => void) | undefined
let rendererRetired = false
const rendererLifetime = abortWhenRendererGone(event.sender)
let sessionListenerRemoved = false
@@ -207,14 +181,6 @@ export function registerSpeechHandlers(store: Store): void {
}
}
if (hotwords && hotwords.length > 0) {
const content = `${hotwords.join('\n')}\n`
const hotwordsFilePath = getHotwordsFilePath(content)
releaseHotwords = retainHotwordsFile(hotwordsFilePath)
await writeFile(hotwordsFilePath, content, 'utf-8')
resolvedHotwordsPath = hotwordsFilePath
}
if (rendererRetired || window.isDestroyed()) {
cleanupSessionListener()
return { directAudio }
@@ -252,7 +218,7 @@ export function registerSpeechHandlers(store: Store): void {
break
}
},
resolvedHotwordsPath,
mergeDictationTerms(context?.repoTerms, store.getSettings().voice?.dictationTerms),
owner,
store.getSettings().voice
)
@@ -272,7 +238,6 @@ export function registerSpeechHandlers(store: Store): void {
cleanupSessionListener()
throw err
} finally {
releaseHotwords?.(resolvedHotwordsPath !== undefined)
if (!directAudio) {
audioPort?.close()
}
@@ -187,6 +187,7 @@ export class RepoLifecycleOperations {
| 'folderUpgradeGitRootPath'
| 'executionHostId'
| 'symlinkPaths'
| 'dictationTerms'
| 'issueSourcePreference'
| 'forkSyncMode'
| 'agentWorkspaceTrust'
@@ -101,6 +101,7 @@ export class RepoUpdatePersistenceOperations {
| 'folderUpgradeGitRootPath'
| 'executionHostId'
| 'symlinkPaths'
| 'dictationTerms'
| 'issueSourcePreference'
| 'defaultJiraProject'
| 'forkSyncMode'
@@ -41,6 +41,7 @@ type RepositoryUpdates = Partial<
| 'worktreeBasePath'
| 'kind'
| 'symlinkPaths'
| 'dictationTerms'
| 'issueSourcePreference'
| 'defaultJiraProject'
| 'externalWorktreeVisibility'
@@ -5,15 +5,16 @@ import {
DictationSetup,
DictationStart,
OpenAiSpeechKeySet,
SpeechModelAction
SpeechModelAction,
SpeechSetupRead
} from '../../../../../shared/rpc-contract/speech-params'
export const SPEECH_METHODS = [
defineMethod({
name: 'speech.models.list',
permission: 'workspace',
params: null,
handler: async (_params, { runtime }) => runtime.listMobileSpeechModels()
params: SpeechSetupRead,
handler: async (params, { runtime }) => runtime.listMobileSpeechModels(params.worktreeId)
}),
defineMethod({
name: 'speech.models.download',
@@ -0,0 +1,12 @@
import { mergeDictationTerms, type DictationTerm } from '../../../shared/dictation-dictionary'
import { getRepoIdFromWorktreeId } from '../../../shared/worktree/id'
import type { RuntimeStore } from '../construction/runtime-store-contract'
/** This host's dictionary for a worktree: its repo's terms, then the personal ones. A folder workspace has no repo row, so it gets the personal ones. */
export function hostDictationTerms(
store: RuntimeStore,
worktreeId: string | undefined
): DictationTerm[] {
const repo = worktreeId ? store.getRepo(getRepoIdFromWorktreeId(worktreeId)) : undefined
return mergeDictationTerms(repo?.dictationTerms, store.getSettings().voice?.dictationTerms)
}
@@ -12,6 +12,7 @@ import {
type MobileDictationChunk
} from './mobile-dictation-chunk-order'
import type { RuntimeStore } from '../construction/runtime-store-contract'
import { hostDictationTerms } from './runtime-dictation-terms'
import {
DictationTranscript,
type DictationTranscriptSnapshot
@@ -42,6 +43,7 @@ export class RuntimeMobileDictationController {
async start(params: {
dictationId: string
modelId?: string
worktreeId?: string
clientId?: string
}): Promise<{ dictationId: string; modelId: string; dedupesResentChunks: true }> {
const store = this.requireStore()
@@ -89,7 +91,7 @@ export class RuntimeMobileDictationController {
await getSpeechSttService(store).startDictation(
modelId,
(event) => acceptEvent(session, event),
undefined,
hostDictationTerms(store, params.worktreeId),
owner,
voice
)
@@ -18,6 +18,7 @@ import {
getSpeechModelDeletionErrorCode
} from '../../speech/speech-model-deletion'
import type { RuntimeStore } from '../construction/runtime-store-contract'
import { hostDictationTerms } from './runtime-dictation-terms'
import {
clearOpenAiSpeechApiKey,
getOpenAiSpeechApiKeyProtection,
@@ -34,7 +35,8 @@ export type OpenAiSpeechKeyStatus = {
export class RuntimeMobileSpeechCatalog {
constructor(private readonly getStore: () => RuntimeStore | null) {}
async list(): Promise<RuntimeSpeechSetupState> {
/** `worktreeId` picks the repo terms a phone recognizing on its own spells with. */
async list(worktreeId?: string): Promise<RuntimeSpeechSetupState> {
const store = this.requireStore()
const voice = store.getSettings().voice ?? getDefaultVoiceSettings()
const states = await getSpeechModelManager(store).getModelStates()
@@ -67,6 +69,7 @@ export class RuntimeMobileSpeechCatalog {
dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle',
dictationHoldToRecord: voice.dictationHoldToRecord !== false,
removeFillerWords: voice.removeFillerWords !== false,
dictationTerms: hostDictationTerms(store, worktreeId),
models
}
}
+2 -1
View File
@@ -4,7 +4,6 @@ export type SttRecognizerInit = {
modelDir: string
files: string[]
sampleRate: number
hotwordsFilePath?: string
/** Absent = auto-detect. */
language?: string
}
@@ -17,6 +16,8 @@ export type SttRecognizerPost = (event: SttRecognizerEvent) => void
/** One loaded model in the speech host child, shared by every dictation stream; calls never overlap. */
export type SttRecognizer = {
/** Starts a stream that expects these spellings; a stream fed without one expects none. */
open(streamId: string, vocabulary: readonly string[]): void
feed(streamId: string, samples: Float32Array, sampleRate: number): Promise<void>
/** Decode the stream's buffered audio, post its text and drop the stream; may throw after posting. */
finish(streamId: string): Promise<void>
+36 -2
View File
@@ -269,6 +269,35 @@ describe('SttService', () => {
])
})
it('spells each dictation with its own terms on one shared recognizer', async () => {
const service = new SttService(readyModels(), spawnMockWorker)
const desktop = vi.fn<SttEventSink>()
const phone = vi.fn<SttEventSink>()
await service.startDictation('model-a', desktop, [{ term: 'pnpm' }], 'desktop')
await service.startDictation(
'model-a',
phone,
[{ term: 'kubectl', heardAs: 'cube control' }],
'mobile:phone-a:1'
)
const worker = getLastWorker()!
const opens = worker.messages.filter((message) => message.type === 'open')
for (const { streamId } of opens) {
worker.emit('message', { type: 'final', streamId, text: 'Run PNPM and cube control.' })
}
expect(getCreatedWorkerCount()).toBe(1)
expect(opens).toEqual([
{ type: 'open', streamId: opens[0].streamId, vocabulary: ['pnpm'] },
{ type: 'open', streamId: opens[1].streamId, vocabulary: ['kubectl'] }
])
expect(desktop.mock.calls.at(-1)).toEqual([
{ type: 'final', text: 'Run pnpm and cube control.' }
])
expect(phone.mock.calls.at(-1)).toEqual([{ type: 'final', text: 'Run PNPM and kubectl.' }])
})
it("hands a renderer's audio port only to its own on-device stream", async () => {
const service = new SttService(readyModels(), spawnMockWorker)
await service.startDictation('model-a', vi.fn(), undefined, 'mobile:phone-a:1')
@@ -606,12 +635,17 @@ describe('SttService', () => {
it('does not treat internal warm-worker replacement as startup cancellation', async () => {
const service = new SttService(readyModels(), spawnMockWorker)
await service.startDictation('model-a', vi.fn(), '/tmp/hotwords-a.txt', 'desktop:1')
await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop:1', {
dictationLanguage: 'ja'
})
await service.stopDictation('desktop:1')
await expect(
service.startDictation('model-a', vi.fn(), '/tmp/hotwords-b.txt', 'desktop:1')
service.startDictation('whisper-model', vi.fn(), undefined, 'desktop:1', {
dictationLanguage: 'ko'
})
).resolves.toBe(undefined)
expect(getCreatedWorkerCount()).toBe(2)
})
it('removes lifecycle listeners when the active worker errors', async () => {
+5 -3
View File
@@ -1,4 +1,5 @@
import type { MessagePortMain } from 'electron'
import { dictationVocabulary, type DictationTerm } from '../../shared/dictation-dictionary'
import {
createDictationTextPipeline,
type DictationTextSettings
@@ -44,19 +45,20 @@ export class SttService {
this.state = createSttSessionState(modelManager, spawnWorker)
}
/** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */
startDictation(
modelId: string,
sink: SttEventSink,
hotwordsFilePath?: string,
terms: readonly DictationTerm[] = [],
owner = 'desktop',
voice?: Partial<DictationTextSettings>
): Promise<void> {
const clean = createDictationTextPipeline(getCatalogModel(modelId), voice)
const clean = createDictationTextPipeline(getCatalogModel(modelId), voice, terms)
return startSttDictation(
this.state,
modelId,
clean ? cleanTranscriptEvents(sink, clean) : sink,
hotwordsFilePath,
dictationVocabulary(terms),
owner,
voice?.dictationLanguage
)
+1 -6
View File
@@ -30,14 +30,9 @@ export async function prewarmSttModel(
return
}
const manifest = getCatalogModel(modelId)
// Desktop and phone starts pass no hotwords, so this matches the recognizer they would load.
const config: SttRecognizerConfig | null =
manifest && isLocalSpeechModel(manifest)
? {
modelId,
hotwordsFilePath: undefined,
language: engineDictationLanguage(manifest, language)
}
? { modelId, language: engineDictationLanguage(manifest, language) }
: null
if (config && state.engine && sameSttRecognizerConfig(state.engine, config)) {
return
+19 -8
View File
@@ -33,7 +33,7 @@ export async function startSttDictation(
state: SttSessionState,
modelId: string,
sink: SttEventSink,
hotwordsFilePath?: string,
vocabulary: string[] = [],
owner = 'desktop',
language?: string
): Promise<void> {
@@ -48,7 +48,7 @@ export async function startSttDictation(
}
if (existing?.phase === 'stopping') {
await existing.stopPromise
return startSttDictation(state, modelId, sink, hotwordsFilePath, owner, language)
return startSttDictation(state, modelId, sink, vocabulary, owner, language)
}
const manifest = getCatalogModel(modelId)
@@ -58,7 +58,7 @@ export async function startSttDictation(
const config: SttRecognizerConfig | null =
manifest.provider !== 'local'
? null
: { modelId, hotwordsFilePath, language: engineDictationLanguage(manifest, language) }
: { modelId, language: engineDictationLanguage(manifest, language) }
if (state.sessions.size >= MAX_STT_SESSIONS) {
throw new Error('dictation_session_limit')
}
@@ -72,6 +72,7 @@ export async function startSttDictation(
sink,
config,
modelId,
vocabulary,
engine: null,
cloud: null,
phase: 'starting',
@@ -153,18 +154,29 @@ async function acquireSttEngine(
// Bound in the same step it is chosen, so an idle teardown never sees it unused.
if (current && sameSttRecognizerConfig(current, config)) {
clearSttIdleTeardownTimer(state)
session.engine = current
return current
return bindSttEngine(session, current)
}
if (!current) {
session.engine = spawnSttEngine(state, config, manifest)
return session.engine
return bindSttEngine(session, spawnSttEngine(state, config, manifest))
}
// No session uses a recognizer with another config, so it is idle.
await teardownSttEngine(state, current)
}
}
// The stream opens before this session can feed it, so all of its audio decodes with its vocabulary.
function bindSttEngine(session: SttSession, engine: SttEngine): SttEngine {
session.engine = engine
if (session.vocabulary.length > 0) {
engine.worker.postMessage({
type: 'open',
streamId: session.streamId,
vocabulary: session.vocabulary
})
}
return engine
}
export function spawnSttEngine(
state: SttSessionState,
config: SttRecognizerConfig,
@@ -209,7 +221,6 @@ export function spawnSttEngine(
modelDir: state.modelManager.getModelDir(config.modelId),
sampleRate: manifest.sampleRate,
files: manifest.files ?? [],
hotwordsFilePath: config.hotwordsFilePath,
language: config.language
})
return engine
+3 -6
View File
@@ -9,7 +9,6 @@ export const MAX_STT_SESSIONS = 4
export type SttRecognizerConfig = {
modelId: string
hotwordsFilePath: string | undefined
language: string | undefined
}
@@ -31,6 +30,8 @@ export type SttSession = {
/** null for cloud models, which load no recognizer. */
config: SttRecognizerConfig | null
modelId: string
/** Spellings this dictation's stream tells the engine to expect. */
vocabulary: string[]
engine: SttEngine | null
cloud: OpenAiTranscriptionSession | null
phase: 'starting' | 'active' | 'stopping'
@@ -70,11 +71,7 @@ export function createSttSessionState(
}
export function sameSttRecognizerConfig(a: SttRecognizerConfig, b: SttRecognizerConfig): boolean {
return (
a.modelId === b.modelId &&
a.hotwordsFilePath === b.hotwordsFilePath &&
a.language === b.language
)
return a.modelId === b.modelId && a.language === b.language
}
export function sttEngineUsers(state: SttSessionState, engine: SttEngine): SttSession[] {
@@ -1,4 +1,3 @@
import { readFile } from 'node:fs/promises'
import { availableParallelism } from 'node:os'
import { join } from 'node:path'
import { pathToFileURL } from 'node:url'
@@ -14,18 +13,9 @@ function performanceCoreThreads(): number {
return Math.min(logical, 8, Math.max(4, Math.floor(logical / 2)))
}
// One term per line. The host deletes the file once dictation starts, so a CPU fallback's reload
// finds it gone and runs without.
async function readVocabulary(path: string | undefined): Promise<string[]> {
const text = path ? await readFile(path, 'utf8').catch(() => '') : ''
return text
.split('\n')
.map((line) => line.trim())
.filter(Boolean)
}
// Totals, not the nominal rate ratio: per-feed resampling rounds, and the drift would replay decoded audio.
type StreamCoverage = {
options: TranscribeCpp.TranscribeOptions
chunker: OfflineAudioChunker
inputSamples: number
resampledSamples: number
@@ -45,22 +35,29 @@ export async function createTranscribeCppRecognizer(
const model = await transcribe.TranscribeModel.load(join(init.modelDir, gguf))
// The child runs one call at a time, so every stream can share one session.
const session = model.createSession({ nThreads: performanceCoreThreads() })
const vocabulary = await readVocabulary(init.hotwordsFilePath)
const { languages } = model.capabilities
// The setting is ISO-639-1; Nemotron names locales (ko-KR) and rejects the bare code.
const language = init.language
? (languages.find((code) => code === init.language) ??
languages.find((code) => code.startsWith(`${init.language}-`)))
: undefined
const options: TranscribeCpp.TranscribeOptions = {
const baseOptions: TranscribeCpp.TranscribeOptions = {
timestamps: 'none',
...(language ? { language } : {}),
...(vocabulary.length > 0 && model.supports('vocabulary') ? { vocabulary } : {})
...(language ? { language } : {})
}
// A plain list as context, not the `vocabulary` option: in the 2026-11 dev bench that dropped
// Whisper's sentence-final periods. Whisper keeps a long prompt's end, so the first terms go last.
const streamOptions = (vocabulary: readonly string[]): TranscribeCpp.TranscribeOptions =>
vocabulary.length > 0 && model.supports('context_prompt')
? { ...baseOptions, prompt: `${vocabulary.toReversed().join(', ')}.` }
: baseOptions
const modelRate = model.capabilities.nativeSampleRate || init.sampleRate
const streams = new Map<string, StreamCoverage>()
const transcribeChunk = async (chunk: Float32Array): Promise<string> => {
const transcribeChunk = async (
chunk: Float32Array,
options: TranscribeCpp.TranscribeOptions
): Promise<string> => {
try {
return (await session.run(chunk, options)).text
} catch (error) {
@@ -73,11 +70,11 @@ export async function createTranscribeCppRecognizer(
}
// The first run compiles GPU pipelines (seconds on Metal); pay it while loading, not on the first stop.
await transcribeChunk(new Float32Array(Math.round(modelRate / 2)))
await transcribeChunk(new Float32Array(Math.round(modelRate / 2)), baseOptions)
// Coverage is in input-rate samples so the host can trim the audio it keeps for a CPU replay.
const decode = async (streamId: string, stream: StreamCoverage, chunk: Float32Array) => {
const text = await transcribeChunk(chunk)
const text = await transcribeChunk(chunk, stream.options)
stream.decodedSamples += chunk.length
post({
type: 'chunk',
@@ -87,18 +84,24 @@ export async function createTranscribeCppRecognizer(
})
}
const openStream = (streamId: string, vocabulary: readonly string[]): StreamCoverage => {
const stream: StreamCoverage = {
options: streamOptions(vocabulary),
chunker: new OfflineAudioChunker(modelRate),
inputSamples: 0,
resampledSamples: 0,
decodedSamples: 0
}
streams.set(streamId, stream)
return stream
}
return {
open(streamId, vocabulary) {
openStream(streamId, vocabulary)
},
async feed(streamId, samples, sampleRate) {
let stream = streams.get(streamId)
if (!stream) {
stream = {
chunker: new OfflineAudioChunker(modelRate),
inputSamples: 0,
resampledSamples: 0,
decodedSamples: 0
}
streams.set(streamId, stream)
}
const stream = streams.get(streamId) ?? openStream(streamId, [])
// The binding borrows PCM until the run settles; resampleToRate may return the caller's buffer.
const resampled = new Float32Array(
resampleToRate(samples, sampleRate || modelRate, modelRate)
+8 -1
View File
@@ -12,6 +12,7 @@ export type SttGpuPolicy = {
}
type Feed = Extract<SttWorkerRequest, { type: 'feed' }>
type Open = Extract<SttWorkerRequest, { type: 'open' }>
type ChildMessage =
| { type: 'chunk'; streamId: string; text: string; samples: number }
| {
@@ -23,6 +24,7 @@ type ChildMessage =
/** A dictation's audio the GPU has not decoded yet, kept for a CPU replay. */
type StreamBacklog = {
open: Open | null
feeds: Feed[]
/** Input samples before feeds[0], in the stream's own count. */
start: number
@@ -61,6 +63,8 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
postMessage(msg: SttWorkerRequest): void {
if (msg.type === 'init') {
this.init = msg
} else if (msg.type === 'open' && this.canFallBack()) {
this.backlog(msg.streamId).open = msg
} else if (msg.type === 'feed' && this.canFallBack()) {
const stream = this.backlog(msg.streamId)
stream.feeds.push(msg)
@@ -90,7 +94,7 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
private backlog(streamId: string): StreamBacklog {
let stream = this.streams.get(streamId)
if (!stream) {
stream = { feeds: [], start: 0, seconds: 0, pending: null }
stream = { open: null, feeds: [], start: 0, seconds: 0, pending: null }
this.streams.set(streamId, stream)
}
return stream
@@ -171,6 +175,9 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
for (const [streamId, stream] of this.streams) {
// A canceled dictation wants no text, so its audio is not replayed.
if (stream.pending !== 'discard') {
if (stream.open) {
this.child.postMessage(stream.open)
}
for (const feed of stream.feeds) {
this.child.postMessage(feed)
}
+1
View File
@@ -11,6 +11,7 @@ export const STT_WORKER_HUNG_ERROR = 'The speech engine stopped responding. Try
export type SttWorkerRequest =
| ({ type: 'init' } & SttRecognizerInit)
| { type: 'open'; streamId: string; vocabulary: string[] }
| { type: 'feed'; streamId: string; samples: Float32Array; sampleRate: number }
| { type: 'stop'; streamId: string }
| { type: 'discard'; streamId: string }
+2
View File
@@ -106,6 +106,8 @@ function handleMessage(
switch (msg.type) {
case 'init':
return handleInit(msg)
case 'open':
return recognizer?.open(msg.streamId, msg.vocabulary)
case 'feed':
return handleFeed(msg)
case 'stop':
+2 -1
View File
@@ -1,5 +1,6 @@
import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection'
import type {
DictationContext,
OpenAiCompatibleSpeechProvider,
OpenAiCompatibleSpeechProviderSummary,
SpeechErrorEvent,
@@ -28,7 +29,7 @@ export type SpeechApi = {
deleteModel: (modelId: string) => Promise<void>
startDictation: (
modelId: string,
hotwords: string[] | undefined,
context: DictationContext | undefined,
sessionId: string
) => Promise<void>
/** Resolves true when this chunk filled a cloud session's upload cap. */
+3 -2
View File
@@ -1,6 +1,7 @@
import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection'
import { ipcRenderer } from 'electron'
import type {
DictationContext,
OpenAiCompatibleSpeechProvider,
OpenAiCompatibleSpeechProviderSummary,
SpeechAudioPortMessage,
@@ -40,7 +41,7 @@ export const speechApi = {
ipcRenderer.invoke('speech:deleteModel', modelId),
startDictation: async (
modelId: string,
hotwords: string[] | undefined,
context: DictationContext | undefined,
sessionId: string
): Promise<void> => {
const { port1, port2 } = new MessageChannel()
@@ -51,7 +52,7 @@ export const speechApi = {
const result: { directAudio?: boolean } | undefined = await ipcRenderer.invoke(
'speech:startDictation',
modelId,
hotwords,
context,
sessionId
)
directAudio = result?.directAudio === true
@@ -3,6 +3,7 @@ import { useAppStore } from '@/store'
import { useAudioCapture } from '@/hooks/use-audio-capture'
import { DictationIndicator } from './DictationIndicator'
import { captureInsertionTarget } from './dictation-insertion-target'
import { captureDictationContext } from './dictation-context'
import { dictationPillAnchorFor, type DictationPillAnchor } from './dictation-pill-anchor'
import {
canRetryDictation,
@@ -207,7 +208,8 @@ export function DictationController() {
const insertionTarget = captureInsertionTarget()
const deliver = createDictationTranscriptDelivery()
deliverRef.current = (text) => deliver(text, insertionTarget)
const kept = createDictationKeptRecording(insertionTarget, deliver)
const context = captureDictationContext()
const kept = createDictationKeptRecording(insertionTarget, deliver, context)
keptRef.current = kept
setRecordingKey((key) => key + 1)
setCaptureLive(false)
@@ -270,7 +272,7 @@ export function DictationController() {
}
setCaptureLive(true)
await window.api.speech.startDictation(modelId, undefined, sessionId)
await window.api.speech.startDictation(modelId, context, sessionId)
if (!isCurrent()) {
await releaseStart(true)
return
@@ -0,0 +1,18 @@
import { useAppStore } from '@/store'
import { findRepoForHost } from '@/store/repos/repo-host-identity'
import { getRepoIdFromWorktreeId } from '../../../../shared/worktree/id'
import type { DictationContext } from '../../../../shared/speech-types'
/** The worktree a dictation starts in, with its repo's terms for an engine running in this app. */
export function captureDictationContext(): DictationContext {
const state = useAppStore.getState()
const worktreeId = state.activeWorktreeId
if (!worktreeId) {
return {}
}
const repo = findRepoForHost(state.repos, getRepoIdFromWorktreeId(worktreeId), {
hostId: state.activeWorkspaceExecutionHostId,
settings: state.settings
})
return { worktreeId, ...(repo?.dictationTerms ? { repoTerms: repo.dictationTerms } : {}) }
}
@@ -2,6 +2,7 @@ import {
CLOUD_DICTATION_MAX_SECONDS,
DICTATION_SAMPLE_RATE
} from '../../../../shared/rpc-contract/speech-params'
import type { DictationContext } from '../../../../shared/speech-types'
import type { DictationInsertionTarget } from './dictation-insertion-target'
import type { DictationTranscriptDelivery } from './dictation-transcript-delivery'
import { trimLandedOverlap } from './dictation-retry-overlap'
@@ -17,11 +18,14 @@ export type DictationKeptRecording = {
complete: boolean
target: DictationInsertionTarget | null
deliver: DictationTranscriptDelivery
/** A retry spells with the dictionary of the place it started in. */
context: DictationContext
}
export function createDictationKeptRecording(
target: DictationInsertionTarget | null,
deliver: DictationTranscriptDelivery
deliver: DictationTranscriptDelivery,
context: DictationContext
): DictationKeptRecording & {
keep: (samples: Float32Array) => void
/** The host stopped accepting audio at its own cap, so the first ten minutes are all it would hear. */
@@ -35,6 +39,7 @@ export function createDictationKeptRecording(
complete: true,
target,
deliver,
context,
keep: (frame: Float32Array): void => {
const room = MAX_KEPT_SAMPLES - samples
if (room <= 0) {
@@ -111,7 +116,7 @@ export async function transcribeKeptRecording(args: {
const retried: string[] = []
args.routeFinals((text) => retried.push(text))
try {
await window.api.speech.startDictation(args.modelId, undefined, sessionId)
await window.api.speech.startDictation(args.modelId, args.recording.context, sessionId)
for (const block of joinIntoBlocks(args.recording.chunks)) {
if (!isCurrent()) {
break
@@ -48,6 +48,7 @@ export type RepoUpdate = Partial<
| 'worktreeBasePath'
| 'kind'
| 'symlinkPaths'
| 'dictationTerms'
| 'issueSourcePreference'
| 'defaultJiraProject'
| 'forkSyncMode'
@@ -148,8 +148,8 @@ export function createWebSpeechApi(): Pick<
}
return {
// No model id, like the phone: the host dictates with the model it has selected now.
startDictation: async (_modelId, _hotwords, sessionId) => {
// No model id, like the phone: the host dictates with the model it has selected now, and its own terms for the worktree.
startDictation: async (_modelId, context, sessionId) => {
const dictation: HostDictation = {
sessionId,
dictationId: `web-${createNonSecureContextUuid()}`,
@@ -164,7 +164,10 @@ export function createWebSpeechApi(): Pick<
}
active = dictation
try {
await callRuntimeResult('speech.dictation.start', { dictationId: dictation.dictationId })
await callRuntimeResult('speech.dictation.start', {
dictationId: dictation.dictationId,
...(context?.worktreeId ? { worktreeId: context.worktreeId } : {})
})
// A cancel that landed while the host was starting couldn't reach this dictation yet.
if (active !== dictation) {
void cancelOnHost(dictation)
+62
View File
@@ -0,0 +1,62 @@
import { describe, expect, it } from 'vitest'
import { createDictationDictionary, mergeDictationTerms } from './dictation-dictionary'
import { createDictationTextPipeline } from './dictation-text-pipeline'
const spell = (terms: Parameters<typeof createDictationDictionary>[0], text: string): string =>
createDictationDictionary(terms)!(text)
describe('createDictationDictionary', () => {
// Parakeet v3's real output for the dev bench's code-term clips.
it('joins the words a model splits a code term into back into the term', () => {
const terms = [
{ term: 'useAudioCapture' },
{ term: 'usePcmWorklet' },
{ term: 'stt-worker.ts' },
{ term: 'numThreads' },
{ term: 'useMemo' },
{ term: 'onChange' }
]
expect(
spell(terms, 'Rename use audio capture to use PCM worklet in the dictation controller.')
).toBe('Rename useAudioCapture to usePcmWorklet in the dictation controller.')
expect(spell(terms, 'Open stt worker TS and set num threads to 4.')).toBe(
'Open stt-worker.ts and set numThreads to 4.'
)
expect(spell(terms, 'Wrap it in use memo and pass the on-change call back.')).toBe(
'Wrap it in useMemo and pass the onChange call back.'
)
})
it('replaces what the model hears instead, and fixes the spelling of the term itself', () => {
const terms = [{ term: 'kubectl', heardAs: 'Quebecal' }, { term: 'Orca' }, { term: 'pnpm' }]
expect(spell(terms, 'Use Quebecal to scale it.')).toBe('Use kubectl to scale it.')
expect(spell(terms, "Open or ca's settings and run PNPM.")).toBe(
"Open Orca's settings and run pnpm."
)
})
it('leaves longer words and words split by a sentence end alone', () => {
const terms = [{ term: 'React' }, { term: 'onChange' }]
expect(spell(terms, 'Reactive state. Turn it on. Change it.')).toBe(
'Reactive state. Turn it on. Change it.'
)
})
it('replaces a "heard as" inside text written without spaces', () => {
expect(spell([{ term: '光輝', heardAs: 'こうき' }], 'こうきさんに送って')).toBe(
'光輝さんに送って'
)
})
it("prefers a repo's spelling over a personal one", () => {
const terms = mergeDictationTerms([{ term: 'RPC' }], [{ term: 'rpc' }])
expect(spell(terms, 'Call the rpc.')).toBe('Call the RPC.')
})
})
describe('createDictationTextPipeline with terms', () => {
it('spells a lowercase term after a sentence-start filler goes', () => {
const clean = createDictationTextPipeline({ language: 'en' }, {}, [{ term: 'pnpm' }])!
expect(clean('Um, pnpm test.')).toBe('pnpm test.')
})
})
+119
View File
@@ -0,0 +1,119 @@
/** A word or phrase dictation should spell `term`'s way, optionally also replacing what the model hears instead. */
export type DictationTerm = { term: string; heardAs?: string }
export const MAX_DICTATION_TERMS = 500
export const MAX_DICTATION_TERM_LENGTH = 100
const WORD_RE = /[\p{L}\p{M}\p{N}]+/gu
// Spaces, or one joiner as in "on-change" or "stt-worker.ts": what may sit between the words of one term.
const JOINING_GAP_RE = /^(?:\s+|[-_./])$/u
// Scripts written without spaces, where a phrase can only be found inside a longer run of letters.
const UNSPACED_SCRIPT_RE =
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u
const MAX_WORDS_PER_TERM = 8
// "Use Audio Capture", "use-audio-capture" and "useAudioCapture" all key as "useaudiocapture".
function wordKey(text: string): string {
return (text.normalize('NFKC').toLowerCase().match(WORD_RE) ?? []).join('')
}
/** Well-formed entries only, trimmed and capped; undefined when the value isn't a list. */
export function normalizeDictationTerms(value: unknown): DictationTerm[] | undefined {
if (!Array.isArray(value)) {
return undefined
}
const terms: DictationTerm[] = []
for (const entry of value) {
const term: unknown = entry?.term
const heardAs: unknown = entry?.heardAs
if (typeof term !== 'string' || !term.trim()) {
continue
}
const alias = typeof heardAs === 'string' ? heardAs.trim() : ''
terms.push({
term: term.trim().slice(0, MAX_DICTATION_TERM_LENGTH),
...(alias ? { heardAs: alias.slice(0, MAX_DICTATION_TERM_LENGTH) } : {})
})
}
return terms.slice(0, MAX_DICTATION_TERMS)
}
/** Repo terms first, so a repo's spelling wins over a personal one for the same words. */
export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown): DictationTerm[] {
return [
...(normalizeDictationTerms(repoTerms) ?? []),
...(normalizeDictationTerms(personalTerms) ?? [])
]
}
/** Rewrites words that key the same as a term (or its "heard as") to the term's spelling; null when there are none. */
export function createDictationDictionary(
terms: readonly DictationTerm[] | undefined
): ((text: string) => string) | null {
const byKey = new Map<string, string>()
const unspacedAliases: [heardAs: string, term: string][] = []
// A "heard as" is what the user asked for, so it beats another term spelled the same.
for (const { term, heardAs } of terms ?? []) {
if (heardAs && UNSPACED_SCRIPT_RE.test(heardAs)) {
unspacedAliases.push([heardAs, term])
} else if (heardAs && wordKey(heardAs) && !byKey.has(wordKey(heardAs))) {
byKey.set(wordKey(heardAs), term)
}
}
for (const { term } of terms ?? []) {
const key = wordKey(term)
if (key && !byKey.has(key)) {
byKey.set(key, term)
}
}
if (byKey.size === 0 && unspacedAliases.length === 0) {
return null
}
const longestKey = Math.max(0, ...[...byKey.keys()].map((key) => key.length))
return (text) => {
for (const [alias, term] of unspacedAliases) {
text = text.split(alias).join(term)
}
return byKey.size > 0 ? replaceWordRuns(text, byKey, longestKey) : text
}
}
// At each word, the longest run of joined words whose key is a term's becomes that term.
function replaceWordRuns(text: string, byKey: Map<string, string>, longestKey: number): string {
const words = [...text.matchAll(WORD_RE)]
let out = ''
let cursor = 0
for (let start = 0; start < words.length;) {
let key = ''
let match: { end: number; term: string } | null = null
for (let end = start; end < words.length && end - start < MAX_WORDS_PER_TERM; end++) {
if (end > start) {
const gapStart = words[end - 1].index + words[end - 1][0].length
if (!JOINING_GAP_RE.test(text.slice(gapStart, words[end].index))) {
break
}
}
key += wordKey(words[end][0])
if (key.length > longestKey) {
break
}
const term = byKey.get(key)
if (term !== undefined) {
match = { end, term }
}
}
if (!match) {
start++
continue
}
out += text.slice(cursor, words[start].index) + match.term
cursor = words[match.end].index + words[match.end][0].length
start = match.end + 1
}
return out + text.slice(cursor)
}
/** The distinct spellings an engine that takes a context prompt is told to expect. */
export function dictationVocabulary(terms: readonly DictationTerm[] | undefined): string[] {
return [...new Set((terms ?? []).map((entry) => entry.term))]
}
+13 -5
View File
@@ -1,3 +1,4 @@
import { createDictationDictionary, type DictationTerm } from './dictation-dictionary'
import { engineDictationLanguage } from './dictation-language'
import { removeFillerWords } from './dictation-filler-words'
import type { SpeechModelManifest, VoiceSettings } from './speech-types'
@@ -26,11 +27,18 @@ function dictatedTextLanguages(
/** The passes dictated text goes through before anyone sees it, or null when every pass is off. */
export function createDictationTextPipeline(
model: DictationTextModel | undefined,
settings: Partial<DictationTextSettings> | undefined
settings: Partial<DictationTextSettings> | undefined,
terms?: readonly DictationTerm[]
): ((text: string) => string) | null {
if (settings?.removeFillerWords === false) {
return null
}
const languages = dictatedTextLanguages(model, settings?.dictationLanguage)
return (text) => removeFillerWords(text, languages)
const dropFillers =
settings?.removeFillerWords === false
? null
: (text: string) => removeFillerWords(text, languages)
const spellTerms = createDictationDictionary(terms)
if (!dropFillers || !spellTerms) {
return dropFillers ?? spellTerms
}
// Terms last, so a filler removed at a sentence start can't recapitalize a lowercase term like "pnpm".
return (text) => spellTerms(dropFillers(text))
}
+3
View File
@@ -1,3 +1,4 @@
import type { DictationTerm } from './dictation-dictionary'
import type { RepoIcon } from './repo-icon'
import type { GhAccountBinding } from './github/account-binding'
import type { GitHubRepositoryIdentity } from './github/pull-request-types'
@@ -103,6 +104,8 @@ export type Repo = {
projectHostSetupMethod?: RepoProjectHostSetupMethod
/** Local row a WSL host now owns, kept for a downgrade; set only by the mover's retention pass. */
orcadRetained?: OrcadRetainedMarker
/** Dictation terms used only in this repo, kept in Orca (on the repo's host), never in the checkout. */
dictationTerms?: DictationTerm[]
}
/** Shared IPC shape for repos:getBaseRefDefault across main/preload/renderer; remoteCount avoids a second hint lookup. */
@@ -10,6 +10,7 @@ import {
} from '../worktree/visibility-sources'
import { OptionalFiniteNumber, OptionalString } from './rpc-param-primitives'
import { editorFormatterSchema } from '../editor-formatter'
import { normalizeDictationTerms } from '../dictation-dictionary'
export const RepoSourceControlAiOverrides = z
.unknown()
@@ -57,6 +58,7 @@ export function createRepoUpdateSchema<T extends Readonly<Record<string, z.ZodTy
worktreeBasePath: OptionalString,
kind: z.enum(['git', 'folder']).optional(),
symlinkPaths: z.array(z.string()).optional(),
dictationTerms: z.unknown().transform(normalizeDictationTerms).optional(),
issueSourcePreference: z.enum(['auto', 'upstream', 'origin']).optional(),
defaultJiraProject: z.unknown().optional().transform(normalizeJiraProjectReference),
ghAccount: z
+3 -2
View File
@@ -616,7 +616,8 @@ import {
DictationSetup,
DictationStart,
OpenAiSpeechKeySet,
SpeechModelAction
SpeechModelAction,
SpeechSetupRead
} from './speech-params'
import {
RepoAddRemote,
@@ -1525,7 +1526,7 @@ export const RPC_PARAMS_BY_METHOD = {
'speech.dictation.start': DictationStart,
'speech.models.delete': SpeechModelAction,
'speech.models.download': SpeechModelAction,
'speech.models.list': null,
'speech.models.list': SpeechSetupRead,
'speech.openAiKey.clear': null,
'speech.openAiKey.set': OpenAiSpeechKeySet,
'speech.openAiKey.status': null,
+7 -1
View File
@@ -31,7 +31,9 @@ export function isValidAudioBase64(value: string): boolean {
export const DictationStart = z.object({
dictationId: requiredString('Missing dictation ID'),
modelId: OptionalString
modelId: OptionalString,
/** Picks the repo's dictionary terms; older hosts strip it. */
worktreeId: OptionalString
})
export const DictationChunk = z.object({
@@ -53,6 +55,10 @@ export const DictationHandle = z.object({
dictationId: requiredString('Missing dictation ID')
})
export const SpeechSetupRead = z.object({
worktreeId: OptionalString
})
export const SpeechModelAction = z.object({
modelId: requiredString('Missing model ID')
})
+3
View File
@@ -1,3 +1,4 @@
import type { DictationTerm } from './dictation-dictionary'
import type { SmartTier } from './sidebar-list/sort-stamps'
import type {
AgentStatusState,
@@ -177,6 +178,8 @@ export type RuntimeSpeechSetupState = {
dictationHoldToRecord: boolean
/** Phones that recognize speech themselves clean their own text by this switch. */
removeFillerWords: boolean
/** ...and spell these terms, repo terms first; absent from older hosts. */
dictationTerms?: DictationTerm[]
models: RuntimeSpeechModelSummary[]
}
+9
View File
@@ -1,3 +1,4 @@
import type { DictationTerm } from './dictation-dictionary'
export type SpeechModelType =
| 'transducer'
| 'paraformer'
@@ -106,4 +107,12 @@ export type VoiceSettings = {
unloadModelAfterMinutes: number | null
/** 'off' keeps on-device recognition on the CPU, for machines whose GPU driver hard-crashes. Absent = 'auto'. */
speechGpu?: SpeechGpuMode
/** Personal dictionary, used in every repo; each repo adds its own Repo.dictationTerms. */
dictationTerms?: DictationTerm[]
}
/** Where a dictation goes: an engine in the desktop's own process takes repoTerms, a host resolves worktreeId itself. */
export type DictationContext = {
worktreeId?: string
repoTerms?: DictationTerm[]
}