mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
feat(speech): spell dictionary terms in dictation, with repo terms and Whisper prompt
This commit is contained in:
@@ -36,6 +36,8 @@ export const dictationSetupSchema = z.looseObject({
|
||||
dictationHoldToRecord: salvagedOptional('dictationHoldToRecord', z.boolean()),
|
||||
// Absent from older hosts, which remove no fillers either; the phone then keeps them on, the host default.
|
||||
removeFillerWords: salvagedOptional('removeFillerWords', z.boolean()),
|
||||
// Absent from older hosts; the phone normalizes it before spelling with it.
|
||||
dictationTerms: salvagedOptional('dictationTerms', z.unknown()),
|
||||
models: salvagingArray(speechModelSchema)
|
||||
})
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { createContext, useContext, useEffect, useState, useSyncExternalStore }
|
||||
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
|
||||
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
|
||||
import type { MobileDictationAwayTarget } from './mobile-dictation-away-insert'
|
||||
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
|
||||
|
||||
export type MobileDictationHostToast = { message: string; open?: () => void }
|
||||
|
||||
@@ -20,6 +21,10 @@ export type MobileDictationOwner = {
|
||||
readonly paneKey: string | undefined
|
||||
readonly dictationMode: string | undefined
|
||||
readonly removeFillerWords: boolean
|
||||
// The computer's terms for this worktree, for words the phone recognizes itself.
|
||||
readonly dictationTerms: readonly DictationTerm[] | undefined
|
||||
// The computer spells its own words with this worktree's repo terms.
|
||||
readonly worktreeId: string
|
||||
}
|
||||
|
||||
// A mounted session screen, as the dictation host sees it.
|
||||
|
||||
@@ -100,6 +100,8 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
client: owner?.client ?? null,
|
||||
enabled: true,
|
||||
removeFillerWords: owner?.removeFillerWords ?? true,
|
||||
dictationTerms: owner?.dictationTerms,
|
||||
worktreeId: owner?.worktreeId,
|
||||
onTranscript: (text) => {
|
||||
const current = ownerRef.current
|
||||
if (!current) {
|
||||
|
||||
@@ -126,8 +126,12 @@ export function isDictationSetupRequiredError(message: string): boolean {
|
||||
return SETUP_REQUIRED_CODES.has(message) || message.startsWith('voice_model_not_ready:')
|
||||
}
|
||||
|
||||
export async function fetchDictationSetup(client: RpcOperationSender): Promise<MobileSpeechSetup> {
|
||||
const reply = await requestDictationSetupReply(client)
|
||||
/** `worktreeId` asks for that repo's dictionary terms too. */
|
||||
export async function fetchDictationSetup(
|
||||
client: RpcOperationSender,
|
||||
worktreeId?: string
|
||||
): Promise<MobileSpeechSetup> {
|
||||
const reply = await requestDictationSetupReply(client, worktreeId ? { worktreeId } : null)
|
||||
if (isLegacyDesktopSpeechSetupReply(reply)) {
|
||||
throw new Error(
|
||||
translate(
|
||||
@@ -145,15 +149,18 @@ export async function fetchDictationSetup(client: RpcOperationSender): Promise<M
|
||||
)
|
||||
}
|
||||
|
||||
async function requestDictationSetupReply(client: RpcOperationSender): Promise<RpcResponse> {
|
||||
async function requestDictationSetupReply(
|
||||
client: RpcOperationSender,
|
||||
params: { worktreeId: string } | null
|
||||
): Promise<RpcResponse> {
|
||||
try {
|
||||
return await dictationSetupRead.request(client, null)
|
||||
return await dictationSetupRead.request(client, params)
|
||||
} catch (error) {
|
||||
if (!(error instanceof LogicalClientCutoverError)) {
|
||||
throw error
|
||||
}
|
||||
// Replay this read on authenticated replacement; mutations must surface cutover to avoid replaying unknown commits.
|
||||
return dictationSetupRead.request(client, null)
|
||||
return dictationSetupRead.request(client, params)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import type { RpcClient } from '../transport/rpc/rpc-client-contract'
|
||||
type StartMobileDictationDesktopSessionOptions = {
|
||||
client: RpcClient
|
||||
dictationId: string
|
||||
worktreeId: string | undefined
|
||||
generation: number
|
||||
getCurrentGeneration: () => number
|
||||
getEnabled: () => boolean
|
||||
@@ -102,7 +103,7 @@ export function startMobileDictationDesktopSession(
|
||||
}
|
||||
|
||||
async function startOnHost(options: StartMobileDictationDesktopSessionOptions): Promise<boolean> {
|
||||
const { client, dictationId } = options
|
||||
const { client, dictationId, worktreeId } = options
|
||||
let hostDedupesResends = false
|
||||
|
||||
try {
|
||||
@@ -111,7 +112,7 @@ async function startOnHost(options: StartMobileDictationDesktopSessionOptions):
|
||||
client,
|
||||
dictationSessionStart.request(
|
||||
client,
|
||||
{ dictationId },
|
||||
{ dictationId, ...(worktreeId ? { worktreeId } : {}) },
|
||||
{ timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS }
|
||||
)
|
||||
).catch(() => {
|
||||
|
||||
@@ -48,6 +48,7 @@ export type MobileDictationRun = {
|
||||
readonly finish: RefObject<(emptyTranscriptMessage: string) => Promise<void>>
|
||||
readonly onTranscript: RefObject<(text: string) => void>
|
||||
readonly onAudioLimitReached: RefObject<(() => void) | undefined>
|
||||
readonly worktreeId: RefObject<string | undefined>
|
||||
readonly keptTranscript: MobileDictationTranscriptSoFar
|
||||
// keepText: leave the words so far on screen (recording, finishing, or a failure kept for Retry).
|
||||
readonly applyStatus: (next: DictationStatus, keepText?: boolean) => void
|
||||
|
||||
@@ -22,6 +22,7 @@ export async function startMobileDictationOnHost(
|
||||
await startMobileDictationDesktopSession({
|
||||
client,
|
||||
dictationId,
|
||||
worktreeId: run.worktreeId.current,
|
||||
generation,
|
||||
getCurrentGeneration: () => run.generation.current,
|
||||
getEnabled: () => run.enabled.current,
|
||||
|
||||
@@ -20,6 +20,10 @@ export type UseMobileDictationOptions = {
|
||||
onError?: (error: Error, outcome: { kept: boolean }) => void
|
||||
// The computer's filler switch, for text the phone recognizes itself; unknown means on.
|
||||
removeFillerWords?: boolean
|
||||
// The computer's dictionary for this worktree, unvalidated, for text the phone recognizes itself.
|
||||
dictationTerms?: unknown
|
||||
// Lets the computer pick this worktree's repo terms for text it recognizes.
|
||||
worktreeId?: string
|
||||
// The host's cloud upload is full; the dictation stops and inserts what was kept.
|
||||
onAudioLimitReached?: () => void
|
||||
}
|
||||
|
||||
@@ -36,9 +36,10 @@ export type { UseMobileDictationResult } from './mobile-dictation-session-state'
|
||||
export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult {
|
||||
useMobileTranslation()
|
||||
const { client, enabled, onTranscript, onError, removeFillerWords, onAudioLimitReached } = options
|
||||
const { dictationTerms, worktreeId } = options
|
||||
// Both capture seams hold the screen while the microphone is open; native uses the device directly and page uses shell audio verbs.
|
||||
const capture = useDictationCapture()
|
||||
const speech = useOnDeviceSpeech(removeFillerWords)
|
||||
const speech = useOnDeviceSpeech(removeFillerWords, dictationTerms)
|
||||
const [status, setStatus] = useState<DictationStatus>('idle')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
// A failure whose recording is kept for Retry; cleared by Retry, discard and the next dictation.
|
||||
@@ -61,6 +62,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
const onTranscriptRef = useRef(onTranscript)
|
||||
const onErrorRef = useRef(onError)
|
||||
const onAudioLimitReachedRef = useRef(onAudioLimitReached)
|
||||
const worktreeIdRef = useRef(worktreeId)
|
||||
const pendingChunksRef = useRef<Set<Promise<void>>>(new Set())
|
||||
const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget())
|
||||
const audioPhaseRef = useRef<MobileDictationAudioPhase>('closed')
|
||||
@@ -79,7 +81,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
onTranscriptRef.current = onTranscript
|
||||
onErrorRef.current = onError
|
||||
onAudioLimitReachedRef.current = onAudioLimitReached
|
||||
}, [client, speech, enabled, onTranscript, onError, onAudioLimitReached])
|
||||
worktreeIdRef.current = worktreeId
|
||||
}, [client, speech, enabled, onTranscript, onError, onAudioLimitReached, worktreeId])
|
||||
|
||||
const applyStatus = useCallback(
|
||||
// Kept through processing so the text so far stays in place until the final transcript replaces it in the same render.
|
||||
@@ -146,6 +149,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
finish: finishRef,
|
||||
onTranscript: onTranscriptRef,
|
||||
onAudioLimitReached: onAudioLimitReachedRef,
|
||||
worktreeId: worktreeIdRef,
|
||||
keptTranscript,
|
||||
applyStatus,
|
||||
showEngine,
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import type { OnDeviceSpeech } from './on-device-speech-contract'
|
||||
|
||||
// The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer.
|
||||
export function useOnDeviceSpeech(_removeFillerWords?: boolean): OnDeviceSpeech | null {
|
||||
export function useOnDeviceSpeech(
|
||||
_removeFillerWords?: boolean,
|
||||
_dictationTerms?: unknown
|
||||
): OnDeviceSpeech | null {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useLayoutEffect, useMemo, useRef } from 'react'
|
||||
import { normalizeDictationTerms } from '../../../src/shared/dictation-dictionary'
|
||||
import { createDictationTextPipeline } from '../../../src/shared/dictation-text-pipeline'
|
||||
import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
|
||||
import type { OnDeviceSpeech } from './on-device-speech-contract'
|
||||
@@ -9,25 +10,29 @@ const keepText = (text: string): string => text
|
||||
// The phone's recognizer transcribes only the locale it started in, so that locale's language picks the fillers, as the host does for its model.
|
||||
function phoneDictationTextCleaner(
|
||||
locale: string | null,
|
||||
removeFillerWords: boolean
|
||||
removeFillerWords: boolean,
|
||||
dictationTerms: unknown
|
||||
): (text: string) => string {
|
||||
const language = locale?.split('-')[0].toLowerCase()
|
||||
const clean = createDictationTextPipeline(language ? { language } : undefined, {
|
||||
removeFillerWords
|
||||
})
|
||||
const clean = createDictationTextPipeline(
|
||||
language ? { language } : undefined,
|
||||
{ removeFillerWords },
|
||||
normalizeDictationTerms(dictationTerms)
|
||||
)
|
||||
return clean ?? keepText
|
||||
}
|
||||
|
||||
function createPageOnDeviceSpeech(
|
||||
verbs: NativeVerbs,
|
||||
removeFillerWords: () => boolean
|
||||
textSettings: () => { removeFillerWords: boolean; dictationTerms: unknown }
|
||||
): OnDeviceSpeech {
|
||||
let clean = keepText
|
||||
return {
|
||||
status: () => verbs.speechStatus(onDeviceSpeechLocale()),
|
||||
start: async () => {
|
||||
const started = await verbs.speechStart(onDeviceSpeechLocale())
|
||||
clean = phoneDictationTextCleaner(started.locale, removeFillerWords())
|
||||
const { removeFillerWords, dictationTerms } = textSettings()
|
||||
clean = phoneDictationTextCleaner(started.locale, removeFillerWords, dictationTerms)
|
||||
return started
|
||||
},
|
||||
read: async () => {
|
||||
@@ -46,17 +51,20 @@ function createPageOnDeviceSpeech(
|
||||
}
|
||||
|
||||
// Null on a shell without the verb, so dictation stays on the computer exactly as before.
|
||||
// removeFillerWords is the computer's switch, read at each start.
|
||||
export function useOnDeviceSpeech(removeFillerWords = true): OnDeviceSpeech | null {
|
||||
// The computer's filler switch and dictionary for this worktree, read at each start.
|
||||
export function useOnDeviceSpeech(
|
||||
removeFillerWords = true,
|
||||
dictationTerms?: unknown
|
||||
): OnDeviceSpeech | null {
|
||||
const verbs = useNativeVerbs()
|
||||
const removeFillerWordsRef = useRef(removeFillerWords)
|
||||
const textSettingsRef = useRef({ removeFillerWords, dictationTerms })
|
||||
useLayoutEffect(() => {
|
||||
removeFillerWordsRef.current = removeFillerWords
|
||||
}, [removeFillerWords])
|
||||
textSettingsRef.current = { removeFillerWords, dictationTerms }
|
||||
}, [removeFillerWords, dictationTerms])
|
||||
return useMemo(
|
||||
() =>
|
||||
verbs.canRecognizeSpeech
|
||||
? createPageOnDeviceSpeech(verbs, () => removeFillerWordsRef.current)
|
||||
? createPageOnDeviceSpeech(verbs, () => textSettingsRef.current)
|
||||
: null,
|
||||
[verbs]
|
||||
)
|
||||
|
||||
@@ -8,14 +8,17 @@ import {
|
||||
fetchDictationSetup
|
||||
} from '../dictation/mobile-dictation-setup'
|
||||
import type { MobileSessionFeedbackCapabilitiesModel } from './use-mobile-session-feedback-capabilities'
|
||||
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
|
||||
|
||||
// The computer's dictation settings as this session reads them: mic shown, mode, filler switch, and whether setup is needed first.
|
||||
export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapabilitiesModel) {
|
||||
const { client, connState, setShowDictationSetup, setDictationMode, setDictationEnabled } = scope
|
||||
const { client, connState, worktreeId, setShowDictationSetup, setDictationMode } = scope
|
||||
const { setDictationEnabled } = scope
|
||||
// The phone's own recognizer needs no computer setup, and chat inserts its words on the phone.
|
||||
const phoneDictation = usePhoneDictationReady()
|
||||
// The host's filler switch, kept from its last setup read; on until one says otherwise.
|
||||
// The host's filler switch and this worktree's terms, kept from its last setup read; fillers on until one says otherwise.
|
||||
const [removeFillerWords, setRemoveFillerWords] = useState(true)
|
||||
const [dictationTerms, setDictationTerms] = useState<DictationTerm[] | undefined>(undefined)
|
||||
// Only a host setup read sets this; while false, the host's start refusal decides (older hosts, first tap before the read).
|
||||
const dictationNeedsSetupRef = useRef(false)
|
||||
|
||||
@@ -27,17 +30,18 @@ export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapab
|
||||
return
|
||||
}
|
||||
try {
|
||||
const setup = await fetchDictationSetup(client)
|
||||
const setup = await fetchDictationSetup(client, worktreeId)
|
||||
if (read === dictationSettingsReadRef.current) {
|
||||
setDictationMode(setup.dictationMode)
|
||||
setRemoveFillerWords(setup.removeFillerWords !== false)
|
||||
setDictationTerms(setup.dictationTerms)
|
||||
setDictationEnabled(!dictationMicHidden(setup))
|
||||
dictationNeedsSetupRef.current = dictationNeedsSetup(setup)
|
||||
}
|
||||
} catch {
|
||||
// Keep the last settings when an older host cannot report them.
|
||||
}
|
||||
}, [client, connState, phoneDictation.refresh, setDictationEnabled, setDictationMode])
|
||||
}, [client, connState, phoneDictation.refresh, setDictationEnabled, setDictationMode, worktreeId])
|
||||
|
||||
// The sheet may have changed host setup: let the host judge taps until the fresh read lands.
|
||||
const closeDictationSetup = useCallback(() => {
|
||||
@@ -65,6 +69,7 @@ export function useMobileSessionDictationSetup(scope: MobileSessionFeedbackCapab
|
||||
return {
|
||||
phoneDictation,
|
||||
removeFillerWords,
|
||||
dictationTerms,
|
||||
dictationNeedsSetupRef,
|
||||
closeDictationSetup,
|
||||
refreshDictationMode
|
||||
|
||||
@@ -185,6 +185,7 @@ export function useMobileSessionDictation(
|
||||
const {
|
||||
phoneDictation,
|
||||
removeFillerWords,
|
||||
dictationTerms,
|
||||
dictationNeedsSetupRef,
|
||||
closeDictationSetup,
|
||||
refreshDictationMode
|
||||
@@ -234,7 +235,9 @@ export function useMobileSessionDictation(
|
||||
executionHost
|
||||
),
|
||||
dictationMode: scope.dictationMode,
|
||||
removeFillerWords
|
||||
removeFillerWords,
|
||||
dictationTerms,
|
||||
worktreeId
|
||||
})
|
||||
}, [
|
||||
activeHandle,
|
||||
@@ -250,6 +253,7 @@ export function useMobileSessionDictation(
|
||||
instance,
|
||||
phoneDictation.readyRef,
|
||||
removeFillerWords,
|
||||
dictationTerms,
|
||||
scope.dictationMode,
|
||||
sessionKey,
|
||||
setShowDictationSetup,
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
} from '../../../shared/worktree/visibility-sources'
|
||||
import { prepareLocalWorktreeRootForRepo } from '../../workspaces/creation/worktree-root-preparation'
|
||||
import { commitProvenRepoKindChange } from '../../workspaces/catalog/folder-repo-git-conversion'
|
||||
import { normalizeDictationTerms } from '../../../shared/dictation-dictionary'
|
||||
import { invalidateAuthorizedRootsCache } from '../worktrees/removal/registered-worktree-roots-cache'
|
||||
import { notifyReposChanged } from './repos-changed-notification'
|
||||
import {
|
||||
@@ -45,6 +46,7 @@ export function registerRepoUpdateHandler(mainWindow: BrowserWindow, store: Stor
|
||||
| 'worktreeBasePath'
|
||||
| 'kind'
|
||||
| 'symlinkPaths'
|
||||
| 'dictationTerms'
|
||||
| 'issueSourcePreference'
|
||||
| 'forkSyncMode'
|
||||
| 'agentWorkspaceTrust'
|
||||
@@ -110,6 +112,14 @@ export function registerRepoUpdateHandler(mainWindow: BrowserWindow, store: Stor
|
||||
delete updates.symlinkPaths
|
||||
}
|
||||
}
|
||||
if ('dictationTerms' in updates) {
|
||||
const terms = normalizeDictationTerms(updates.dictationTerms)
|
||||
if (terms) {
|
||||
updates.dictationTerms = terms
|
||||
} else {
|
||||
delete updates.dictationTerms
|
||||
}
|
||||
}
|
||||
if ('worktreeBasePath' in updates && updates.worktreeBasePath !== undefined) {
|
||||
const v = updates.worktreeBasePath as unknown
|
||||
if (typeof v !== 'string') {
|
||||
|
||||
+4
-39
@@ -6,9 +6,8 @@ import {
|
||||
type MessagePortMain
|
||||
} from 'electron'
|
||||
import { abortWhenRendererGone } from './renderer-lifetime-abort'
|
||||
import { join } from 'node:path'
|
||||
import { writeFile, unlink } from 'node:fs/promises'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mergeDictationTerms } from '../../shared/dictation-dictionary'
|
||||
import type { DictationContext } from '../../shared/speech-types'
|
||||
import { SPEECH_MODEL_CATALOG } from '../speech/model-catalog'
|
||||
import { deleteLocalSpeechModel } from '../speech/speech-model-deletion'
|
||||
import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service'
|
||||
@@ -112,29 +111,6 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
})
|
||||
})
|
||||
|
||||
const getHotwordsFilePath = (content: string): string => {
|
||||
const digest = createHash('sha256').update(content).digest('hex').slice(0, 12)
|
||||
// Native speech libraries may open paths with ANSI Windows APIs; keep hotwords in the ASCII-safe model cache.
|
||||
return join(getSpeechModelManager(store).getModelsDir(), `speech-hotwords-${digest}.txt`)
|
||||
}
|
||||
|
||||
const hotwordFileUsers = new Map<string, { users: number; written: boolean }>()
|
||||
const retainHotwordsFile = (path: string): ((written: boolean) => void) => {
|
||||
const entry = hotwordFileUsers.get(path) ?? { users: 0, written: false }
|
||||
entry.users += 1
|
||||
hotwordFileUsers.set(path, entry)
|
||||
return (written) => {
|
||||
entry.written ||= written
|
||||
entry.users -= 1
|
||||
if (entry.users === 0) {
|
||||
hotwordFileUsers.delete(path)
|
||||
if (entry.written) {
|
||||
unlink(path).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Renderer run ids restart per document; a retired document's owner must not be reused by its successor.
|
||||
const retiredDocumentCounts = new Map<number, number>()
|
||||
const getDesktopOwner = (senderId: number, sessionId: string): string =>
|
||||
@@ -153,7 +129,7 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
|
||||
ipcMain.handle(
|
||||
'speech:startDictation',
|
||||
async (event, modelId: string, hotwords?: string[], sessionId = 'desktop') => {
|
||||
async (event, modelId: string, context?: DictationContext, sessionId = 'desktop') => {
|
||||
const senderId = event.sender.id
|
||||
const owner = getDesktopOwner(senderId, sessionId)
|
||||
const audioPort = pendingAudioPorts.get(owner)
|
||||
@@ -164,8 +140,6 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
audioPort?.close()
|
||||
return { directAudio }
|
||||
}
|
||||
let resolvedHotwordsPath: string | undefined
|
||||
let releaseHotwords: ((written: boolean) => void) | undefined
|
||||
let rendererRetired = false
|
||||
const rendererLifetime = abortWhenRendererGone(event.sender)
|
||||
let sessionListenerRemoved = false
|
||||
@@ -207,14 +181,6 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
}
|
||||
}
|
||||
|
||||
if (hotwords && hotwords.length > 0) {
|
||||
const content = `${hotwords.join('\n')}\n`
|
||||
const hotwordsFilePath = getHotwordsFilePath(content)
|
||||
releaseHotwords = retainHotwordsFile(hotwordsFilePath)
|
||||
await writeFile(hotwordsFilePath, content, 'utf-8')
|
||||
resolvedHotwordsPath = hotwordsFilePath
|
||||
}
|
||||
|
||||
if (rendererRetired || window.isDestroyed()) {
|
||||
cleanupSessionListener()
|
||||
return { directAudio }
|
||||
@@ -252,7 +218,7 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
break
|
||||
}
|
||||
},
|
||||
resolvedHotwordsPath,
|
||||
mergeDictationTerms(context?.repoTerms, store.getSettings().voice?.dictationTerms),
|
||||
owner,
|
||||
store.getSettings().voice
|
||||
)
|
||||
@@ -272,7 +238,6 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
cleanupSessionListener()
|
||||
throw err
|
||||
} finally {
|
||||
releaseHotwords?.(resolvedHotwordsPath !== undefined)
|
||||
if (!directAudio) {
|
||||
audioPort?.close()
|
||||
}
|
||||
|
||||
@@ -187,6 +187,7 @@ export class RepoLifecycleOperations {
|
||||
| 'folderUpgradeGitRootPath'
|
||||
| 'executionHostId'
|
||||
| 'symlinkPaths'
|
||||
| 'dictationTerms'
|
||||
| 'issueSourcePreference'
|
||||
| 'forkSyncMode'
|
||||
| 'agentWorkspaceTrust'
|
||||
|
||||
@@ -101,6 +101,7 @@ export class RepoUpdatePersistenceOperations {
|
||||
| 'folderUpgradeGitRootPath'
|
||||
| 'executionHostId'
|
||||
| 'symlinkPaths'
|
||||
| 'dictationTerms'
|
||||
| 'issueSourcePreference'
|
||||
| 'defaultJiraProject'
|
||||
| 'forkSyncMode'
|
||||
|
||||
@@ -41,6 +41,7 @@ type RepositoryUpdates = Partial<
|
||||
| 'worktreeBasePath'
|
||||
| 'kind'
|
||||
| 'symlinkPaths'
|
||||
| 'dictationTerms'
|
||||
| 'issueSourcePreference'
|
||||
| 'defaultJiraProject'
|
||||
| 'externalWorktreeVisibility'
|
||||
|
||||
@@ -5,15 +5,16 @@ import {
|
||||
DictationSetup,
|
||||
DictationStart,
|
||||
OpenAiSpeechKeySet,
|
||||
SpeechModelAction
|
||||
SpeechModelAction,
|
||||
SpeechSetupRead
|
||||
} from '../../../../../shared/rpc-contract/speech-params'
|
||||
|
||||
export const SPEECH_METHODS = [
|
||||
defineMethod({
|
||||
name: 'speech.models.list',
|
||||
permission: 'workspace',
|
||||
params: null,
|
||||
handler: async (_params, { runtime }) => runtime.listMobileSpeechModels()
|
||||
params: SpeechSetupRead,
|
||||
handler: async (params, { runtime }) => runtime.listMobileSpeechModels(params.worktreeId)
|
||||
}),
|
||||
defineMethod({
|
||||
name: 'speech.models.download',
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { mergeDictationTerms, type DictationTerm } from '../../../shared/dictation-dictionary'
|
||||
import { getRepoIdFromWorktreeId } from '../../../shared/worktree/id'
|
||||
import type { RuntimeStore } from '../construction/runtime-store-contract'
|
||||
|
||||
/** This host's dictionary for a worktree: its repo's terms, then the personal ones. A folder workspace has no repo row, so it gets the personal ones. */
|
||||
export function hostDictationTerms(
|
||||
store: RuntimeStore,
|
||||
worktreeId: string | undefined
|
||||
): DictationTerm[] {
|
||||
const repo = worktreeId ? store.getRepo(getRepoIdFromWorktreeId(worktreeId)) : undefined
|
||||
return mergeDictationTerms(repo?.dictationTerms, store.getSettings().voice?.dictationTerms)
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
type MobileDictationChunk
|
||||
} from './mobile-dictation-chunk-order'
|
||||
import type { RuntimeStore } from '../construction/runtime-store-contract'
|
||||
import { hostDictationTerms } from './runtime-dictation-terms'
|
||||
import {
|
||||
DictationTranscript,
|
||||
type DictationTranscriptSnapshot
|
||||
@@ -42,6 +43,7 @@ export class RuntimeMobileDictationController {
|
||||
async start(params: {
|
||||
dictationId: string
|
||||
modelId?: string
|
||||
worktreeId?: string
|
||||
clientId?: string
|
||||
}): Promise<{ dictationId: string; modelId: string; dedupesResentChunks: true }> {
|
||||
const store = this.requireStore()
|
||||
@@ -89,7 +91,7 @@ export class RuntimeMobileDictationController {
|
||||
await getSpeechSttService(store).startDictation(
|
||||
modelId,
|
||||
(event) => acceptEvent(session, event),
|
||||
undefined,
|
||||
hostDictationTerms(store, params.worktreeId),
|
||||
owner,
|
||||
voice
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
getSpeechModelDeletionErrorCode
|
||||
} from '../../speech/speech-model-deletion'
|
||||
import type { RuntimeStore } from '../construction/runtime-store-contract'
|
||||
import { hostDictationTerms } from './runtime-dictation-terms'
|
||||
import {
|
||||
clearOpenAiSpeechApiKey,
|
||||
getOpenAiSpeechApiKeyProtection,
|
||||
@@ -34,7 +35,8 @@ export type OpenAiSpeechKeyStatus = {
|
||||
export class RuntimeMobileSpeechCatalog {
|
||||
constructor(private readonly getStore: () => RuntimeStore | null) {}
|
||||
|
||||
async list(): Promise<RuntimeSpeechSetupState> {
|
||||
/** `worktreeId` picks the repo terms a phone recognizing on its own spells with. */
|
||||
async list(worktreeId?: string): Promise<RuntimeSpeechSetupState> {
|
||||
const store = this.requireStore()
|
||||
const voice = store.getSettings().voice ?? getDefaultVoiceSettings()
|
||||
const states = await getSpeechModelManager(store).getModelStates()
|
||||
@@ -67,6 +69,7 @@ export class RuntimeMobileSpeechCatalog {
|
||||
dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle',
|
||||
dictationHoldToRecord: voice.dictationHoldToRecord !== false,
|
||||
removeFillerWords: voice.removeFillerWords !== false,
|
||||
dictationTerms: hostDictationTerms(store, worktreeId),
|
||||
models
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ export type SttRecognizerInit = {
|
||||
modelDir: string
|
||||
files: string[]
|
||||
sampleRate: number
|
||||
hotwordsFilePath?: string
|
||||
/** Absent = auto-detect. */
|
||||
language?: string
|
||||
}
|
||||
@@ -17,6 +16,8 @@ export type SttRecognizerPost = (event: SttRecognizerEvent) => void
|
||||
|
||||
/** One loaded model in the speech host child, shared by every dictation stream; calls never overlap. */
|
||||
export type SttRecognizer = {
|
||||
/** Starts a stream that expects these spellings; a stream fed without one expects none. */
|
||||
open(streamId: string, vocabulary: readonly string[]): void
|
||||
feed(streamId: string, samples: Float32Array, sampleRate: number): Promise<void>
|
||||
/** Decode the stream's buffered audio, post its text and drop the stream; may throw after posting. */
|
||||
finish(streamId: string): Promise<void>
|
||||
|
||||
@@ -269,6 +269,35 @@ describe('SttService', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('spells each dictation with its own terms on one shared recognizer', async () => {
|
||||
const service = new SttService(readyModels(), spawnMockWorker)
|
||||
const desktop = vi.fn<SttEventSink>()
|
||||
const phone = vi.fn<SttEventSink>()
|
||||
|
||||
await service.startDictation('model-a', desktop, [{ term: 'pnpm' }], 'desktop')
|
||||
await service.startDictation(
|
||||
'model-a',
|
||||
phone,
|
||||
[{ term: 'kubectl', heardAs: 'cube control' }],
|
||||
'mobile:phone-a:1'
|
||||
)
|
||||
const worker = getLastWorker()!
|
||||
const opens = worker.messages.filter((message) => message.type === 'open')
|
||||
for (const { streamId } of opens) {
|
||||
worker.emit('message', { type: 'final', streamId, text: 'Run PNPM and cube control.' })
|
||||
}
|
||||
|
||||
expect(getCreatedWorkerCount()).toBe(1)
|
||||
expect(opens).toEqual([
|
||||
{ type: 'open', streamId: opens[0].streamId, vocabulary: ['pnpm'] },
|
||||
{ type: 'open', streamId: opens[1].streamId, vocabulary: ['kubectl'] }
|
||||
])
|
||||
expect(desktop.mock.calls.at(-1)).toEqual([
|
||||
{ type: 'final', text: 'Run pnpm and cube control.' }
|
||||
])
|
||||
expect(phone.mock.calls.at(-1)).toEqual([{ type: 'final', text: 'Run PNPM and kubectl.' }])
|
||||
})
|
||||
|
||||
it("hands a renderer's audio port only to its own on-device stream", async () => {
|
||||
const service = new SttService(readyModels(), spawnMockWorker)
|
||||
await service.startDictation('model-a', vi.fn(), undefined, 'mobile:phone-a:1')
|
||||
@@ -606,12 +635,17 @@ describe('SttService', () => {
|
||||
it('does not treat internal warm-worker replacement as startup cancellation', async () => {
|
||||
const service = new SttService(readyModels(), spawnMockWorker)
|
||||
|
||||
await service.startDictation('model-a', vi.fn(), '/tmp/hotwords-a.txt', 'desktop:1')
|
||||
await service.startDictation('whisper-model', vi.fn(), undefined, 'desktop:1', {
|
||||
dictationLanguage: 'ja'
|
||||
})
|
||||
await service.stopDictation('desktop:1')
|
||||
|
||||
await expect(
|
||||
service.startDictation('model-a', vi.fn(), '/tmp/hotwords-b.txt', 'desktop:1')
|
||||
service.startDictation('whisper-model', vi.fn(), undefined, 'desktop:1', {
|
||||
dictationLanguage: 'ko'
|
||||
})
|
||||
).resolves.toBe(undefined)
|
||||
expect(getCreatedWorkerCount()).toBe(2)
|
||||
})
|
||||
|
||||
it('removes lifecycle listeners when the active worker errors', async () => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { MessagePortMain } from 'electron'
|
||||
import { dictationVocabulary, type DictationTerm } from '../../shared/dictation-dictionary'
|
||||
import {
|
||||
createDictationTextPipeline,
|
||||
type DictationTextSettings
|
||||
@@ -44,19 +45,20 @@ export class SttService {
|
||||
this.state = createSttSessionState(modelManager, spawnWorker)
|
||||
}
|
||||
|
||||
/** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */
|
||||
startDictation(
|
||||
modelId: string,
|
||||
sink: SttEventSink,
|
||||
hotwordsFilePath?: string,
|
||||
terms: readonly DictationTerm[] = [],
|
||||
owner = 'desktop',
|
||||
voice?: Partial<DictationTextSettings>
|
||||
): Promise<void> {
|
||||
const clean = createDictationTextPipeline(getCatalogModel(modelId), voice)
|
||||
const clean = createDictationTextPipeline(getCatalogModel(modelId), voice, terms)
|
||||
return startSttDictation(
|
||||
this.state,
|
||||
modelId,
|
||||
clean ? cleanTranscriptEvents(sink, clean) : sink,
|
||||
hotwordsFilePath,
|
||||
dictationVocabulary(terms),
|
||||
owner,
|
||||
voice?.dictationLanguage
|
||||
)
|
||||
|
||||
@@ -30,14 +30,9 @@ export async function prewarmSttModel(
|
||||
return
|
||||
}
|
||||
const manifest = getCatalogModel(modelId)
|
||||
// Desktop and phone starts pass no hotwords, so this matches the recognizer they would load.
|
||||
const config: SttRecognizerConfig | null =
|
||||
manifest && isLocalSpeechModel(manifest)
|
||||
? {
|
||||
modelId,
|
||||
hotwordsFilePath: undefined,
|
||||
language: engineDictationLanguage(manifest, language)
|
||||
}
|
||||
? { modelId, language: engineDictationLanguage(manifest, language) }
|
||||
: null
|
||||
if (config && state.engine && sameSttRecognizerConfig(state.engine, config)) {
|
||||
return
|
||||
|
||||
@@ -33,7 +33,7 @@ export async function startSttDictation(
|
||||
state: SttSessionState,
|
||||
modelId: string,
|
||||
sink: SttEventSink,
|
||||
hotwordsFilePath?: string,
|
||||
vocabulary: string[] = [],
|
||||
owner = 'desktop',
|
||||
language?: string
|
||||
): Promise<void> {
|
||||
@@ -48,7 +48,7 @@ export async function startSttDictation(
|
||||
}
|
||||
if (existing?.phase === 'stopping') {
|
||||
await existing.stopPromise
|
||||
return startSttDictation(state, modelId, sink, hotwordsFilePath, owner, language)
|
||||
return startSttDictation(state, modelId, sink, vocabulary, owner, language)
|
||||
}
|
||||
|
||||
const manifest = getCatalogModel(modelId)
|
||||
@@ -58,7 +58,7 @@ export async function startSttDictation(
|
||||
const config: SttRecognizerConfig | null =
|
||||
manifest.provider !== 'local'
|
||||
? null
|
||||
: { modelId, hotwordsFilePath, language: engineDictationLanguage(manifest, language) }
|
||||
: { modelId, language: engineDictationLanguage(manifest, language) }
|
||||
if (state.sessions.size >= MAX_STT_SESSIONS) {
|
||||
throw new Error('dictation_session_limit')
|
||||
}
|
||||
@@ -72,6 +72,7 @@ export async function startSttDictation(
|
||||
sink,
|
||||
config,
|
||||
modelId,
|
||||
vocabulary,
|
||||
engine: null,
|
||||
cloud: null,
|
||||
phase: 'starting',
|
||||
@@ -153,18 +154,29 @@ async function acquireSttEngine(
|
||||
// Bound in the same step it is chosen, so an idle teardown never sees it unused.
|
||||
if (current && sameSttRecognizerConfig(current, config)) {
|
||||
clearSttIdleTeardownTimer(state)
|
||||
session.engine = current
|
||||
return current
|
||||
return bindSttEngine(session, current)
|
||||
}
|
||||
if (!current) {
|
||||
session.engine = spawnSttEngine(state, config, manifest)
|
||||
return session.engine
|
||||
return bindSttEngine(session, spawnSttEngine(state, config, manifest))
|
||||
}
|
||||
// No session uses a recognizer with another config, so it is idle.
|
||||
await teardownSttEngine(state, current)
|
||||
}
|
||||
}
|
||||
|
||||
// The stream opens before this session can feed it, so all of its audio decodes with its vocabulary.
|
||||
function bindSttEngine(session: SttSession, engine: SttEngine): SttEngine {
|
||||
session.engine = engine
|
||||
if (session.vocabulary.length > 0) {
|
||||
engine.worker.postMessage({
|
||||
type: 'open',
|
||||
streamId: session.streamId,
|
||||
vocabulary: session.vocabulary
|
||||
})
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
export function spawnSttEngine(
|
||||
state: SttSessionState,
|
||||
config: SttRecognizerConfig,
|
||||
@@ -209,7 +221,6 @@ export function spawnSttEngine(
|
||||
modelDir: state.modelManager.getModelDir(config.modelId),
|
||||
sampleRate: manifest.sampleRate,
|
||||
files: manifest.files ?? [],
|
||||
hotwordsFilePath: config.hotwordsFilePath,
|
||||
language: config.language
|
||||
})
|
||||
return engine
|
||||
|
||||
@@ -9,7 +9,6 @@ export const MAX_STT_SESSIONS = 4
|
||||
|
||||
export type SttRecognizerConfig = {
|
||||
modelId: string
|
||||
hotwordsFilePath: string | undefined
|
||||
language: string | undefined
|
||||
}
|
||||
|
||||
@@ -31,6 +30,8 @@ export type SttSession = {
|
||||
/** null for cloud models, which load no recognizer. */
|
||||
config: SttRecognizerConfig | null
|
||||
modelId: string
|
||||
/** Spellings this dictation's stream tells the engine to expect. */
|
||||
vocabulary: string[]
|
||||
engine: SttEngine | null
|
||||
cloud: OpenAiTranscriptionSession | null
|
||||
phase: 'starting' | 'active' | 'stopping'
|
||||
@@ -70,11 +71,7 @@ export function createSttSessionState(
|
||||
}
|
||||
|
||||
export function sameSttRecognizerConfig(a: SttRecognizerConfig, b: SttRecognizerConfig): boolean {
|
||||
return (
|
||||
a.modelId === b.modelId &&
|
||||
a.hotwordsFilePath === b.hotwordsFilePath &&
|
||||
a.language === b.language
|
||||
)
|
||||
return a.modelId === b.modelId && a.language === b.language
|
||||
}
|
||||
|
||||
export function sttEngineUsers(state: SttSessionState, engine: SttEngine): SttSession[] {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { availableParallelism } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
@@ -14,18 +13,9 @@ function performanceCoreThreads(): number {
|
||||
return Math.min(logical, 8, Math.max(4, Math.floor(logical / 2)))
|
||||
}
|
||||
|
||||
// One term per line. The host deletes the file once dictation starts, so a CPU fallback's reload
|
||||
// finds it gone and runs without.
|
||||
async function readVocabulary(path: string | undefined): Promise<string[]> {
|
||||
const text = path ? await readFile(path, 'utf8').catch(() => '') : ''
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
// Totals, not the nominal rate ratio: per-feed resampling rounds, and the drift would replay decoded audio.
|
||||
type StreamCoverage = {
|
||||
options: TranscribeCpp.TranscribeOptions
|
||||
chunker: OfflineAudioChunker
|
||||
inputSamples: number
|
||||
resampledSamples: number
|
||||
@@ -45,22 +35,29 @@ export async function createTranscribeCppRecognizer(
|
||||
const model = await transcribe.TranscribeModel.load(join(init.modelDir, gguf))
|
||||
// The child runs one call at a time, so every stream can share one session.
|
||||
const session = model.createSession({ nThreads: performanceCoreThreads() })
|
||||
const vocabulary = await readVocabulary(init.hotwordsFilePath)
|
||||
const { languages } = model.capabilities
|
||||
// The setting is ISO-639-1; Nemotron names locales (ko-KR) and rejects the bare code.
|
||||
const language = init.language
|
||||
? (languages.find((code) => code === init.language) ??
|
||||
languages.find((code) => code.startsWith(`${init.language}-`)))
|
||||
: undefined
|
||||
const options: TranscribeCpp.TranscribeOptions = {
|
||||
const baseOptions: TranscribeCpp.TranscribeOptions = {
|
||||
timestamps: 'none',
|
||||
...(language ? { language } : {}),
|
||||
...(vocabulary.length > 0 && model.supports('vocabulary') ? { vocabulary } : {})
|
||||
...(language ? { language } : {})
|
||||
}
|
||||
// A plain list as context, not the `vocabulary` option: in the 2026-11 dev bench that dropped
|
||||
// Whisper's sentence-final periods. Whisper keeps a long prompt's end, so the first terms go last.
|
||||
const streamOptions = (vocabulary: readonly string[]): TranscribeCpp.TranscribeOptions =>
|
||||
vocabulary.length > 0 && model.supports('context_prompt')
|
||||
? { ...baseOptions, prompt: `${vocabulary.toReversed().join(', ')}.` }
|
||||
: baseOptions
|
||||
const modelRate = model.capabilities.nativeSampleRate || init.sampleRate
|
||||
const streams = new Map<string, StreamCoverage>()
|
||||
|
||||
const transcribeChunk = async (chunk: Float32Array): Promise<string> => {
|
||||
const transcribeChunk = async (
|
||||
chunk: Float32Array,
|
||||
options: TranscribeCpp.TranscribeOptions
|
||||
): Promise<string> => {
|
||||
try {
|
||||
return (await session.run(chunk, options)).text
|
||||
} catch (error) {
|
||||
@@ -73,11 +70,11 @@ export async function createTranscribeCppRecognizer(
|
||||
}
|
||||
|
||||
// The first run compiles GPU pipelines (seconds on Metal); pay it while loading, not on the first stop.
|
||||
await transcribeChunk(new Float32Array(Math.round(modelRate / 2)))
|
||||
await transcribeChunk(new Float32Array(Math.round(modelRate / 2)), baseOptions)
|
||||
|
||||
// Coverage is in input-rate samples so the host can trim the audio it keeps for a CPU replay.
|
||||
const decode = async (streamId: string, stream: StreamCoverage, chunk: Float32Array) => {
|
||||
const text = await transcribeChunk(chunk)
|
||||
const text = await transcribeChunk(chunk, stream.options)
|
||||
stream.decodedSamples += chunk.length
|
||||
post({
|
||||
type: 'chunk',
|
||||
@@ -87,18 +84,24 @@ export async function createTranscribeCppRecognizer(
|
||||
})
|
||||
}
|
||||
|
||||
const openStream = (streamId: string, vocabulary: readonly string[]): StreamCoverage => {
|
||||
const stream: StreamCoverage = {
|
||||
options: streamOptions(vocabulary),
|
||||
chunker: new OfflineAudioChunker(modelRate),
|
||||
inputSamples: 0,
|
||||
resampledSamples: 0,
|
||||
decodedSamples: 0
|
||||
}
|
||||
streams.set(streamId, stream)
|
||||
return stream
|
||||
}
|
||||
|
||||
return {
|
||||
open(streamId, vocabulary) {
|
||||
openStream(streamId, vocabulary)
|
||||
},
|
||||
async feed(streamId, samples, sampleRate) {
|
||||
let stream = streams.get(streamId)
|
||||
if (!stream) {
|
||||
stream = {
|
||||
chunker: new OfflineAudioChunker(modelRate),
|
||||
inputSamples: 0,
|
||||
resampledSamples: 0,
|
||||
decodedSamples: 0
|
||||
}
|
||||
streams.set(streamId, stream)
|
||||
}
|
||||
const stream = streams.get(streamId) ?? openStream(streamId, [])
|
||||
// The binding borrows PCM until the run settles; resampleToRate may return the caller's buffer.
|
||||
const resampled = new Float32Array(
|
||||
resampleToRate(samples, sampleRate || modelRate, modelRate)
|
||||
|
||||
@@ -12,6 +12,7 @@ export type SttGpuPolicy = {
|
||||
}
|
||||
|
||||
type Feed = Extract<SttWorkerRequest, { type: 'feed' }>
|
||||
type Open = Extract<SttWorkerRequest, { type: 'open' }>
|
||||
type ChildMessage =
|
||||
| { type: 'chunk'; streamId: string; text: string; samples: number }
|
||||
| {
|
||||
@@ -23,6 +24,7 @@ type ChildMessage =
|
||||
|
||||
/** A dictation's audio the GPU has not decoded yet, kept for a CPU replay. */
|
||||
type StreamBacklog = {
|
||||
open: Open | null
|
||||
feeds: Feed[]
|
||||
/** Input samples before feeds[0], in the stream's own count. */
|
||||
start: number
|
||||
@@ -61,6 +63,8 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
|
||||
postMessage(msg: SttWorkerRequest): void {
|
||||
if (msg.type === 'init') {
|
||||
this.init = msg
|
||||
} else if (msg.type === 'open' && this.canFallBack()) {
|
||||
this.backlog(msg.streamId).open = msg
|
||||
} else if (msg.type === 'feed' && this.canFallBack()) {
|
||||
const stream = this.backlog(msg.streamId)
|
||||
stream.feeds.push(msg)
|
||||
@@ -90,7 +94,7 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
|
||||
private backlog(streamId: string): StreamBacklog {
|
||||
let stream = this.streams.get(streamId)
|
||||
if (!stream) {
|
||||
stream = { feeds: [], start: 0, seconds: 0, pending: null }
|
||||
stream = { open: null, feeds: [], start: 0, seconds: 0, pending: null }
|
||||
this.streams.set(streamId, stream)
|
||||
}
|
||||
return stream
|
||||
@@ -171,6 +175,9 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
|
||||
for (const [streamId, stream] of this.streams) {
|
||||
// A canceled dictation wants no text, so its audio is not replayed.
|
||||
if (stream.pending !== 'discard') {
|
||||
if (stream.open) {
|
||||
this.child.postMessage(stream.open)
|
||||
}
|
||||
for (const feed of stream.feeds) {
|
||||
this.child.postMessage(feed)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ export const STT_WORKER_HUNG_ERROR = 'The speech engine stopped responding. Try
|
||||
|
||||
export type SttWorkerRequest =
|
||||
| ({ type: 'init' } & SttRecognizerInit)
|
||||
| { type: 'open'; streamId: string; vocabulary: string[] }
|
||||
| { type: 'feed'; streamId: string; samples: Float32Array; sampleRate: number }
|
||||
| { type: 'stop'; streamId: string }
|
||||
| { type: 'discard'; streamId: string }
|
||||
|
||||
@@ -106,6 +106,8 @@ function handleMessage(
|
||||
switch (msg.type) {
|
||||
case 'init':
|
||||
return handleInit(msg)
|
||||
case 'open':
|
||||
return recognizer?.open(msg.streamId, msg.vocabulary)
|
||||
case 'feed':
|
||||
return handleFeed(msg)
|
||||
case 'stop':
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection'
|
||||
import type {
|
||||
DictationContext,
|
||||
OpenAiCompatibleSpeechProvider,
|
||||
OpenAiCompatibleSpeechProviderSummary,
|
||||
SpeechErrorEvent,
|
||||
@@ -28,7 +29,7 @@ export type SpeechApi = {
|
||||
deleteModel: (modelId: string) => Promise<void>
|
||||
startDictation: (
|
||||
modelId: string,
|
||||
hotwords: string[] | undefined,
|
||||
context: DictationContext | undefined,
|
||||
sessionId: string
|
||||
) => Promise<void>
|
||||
/** Resolves true when this chunk filled a cloud session's upload cap. */
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection'
|
||||
import { ipcRenderer } from 'electron'
|
||||
import type {
|
||||
DictationContext,
|
||||
OpenAiCompatibleSpeechProvider,
|
||||
OpenAiCompatibleSpeechProviderSummary,
|
||||
SpeechAudioPortMessage,
|
||||
@@ -40,7 +41,7 @@ export const speechApi = {
|
||||
ipcRenderer.invoke('speech:deleteModel', modelId),
|
||||
startDictation: async (
|
||||
modelId: string,
|
||||
hotwords: string[] | undefined,
|
||||
context: DictationContext | undefined,
|
||||
sessionId: string
|
||||
): Promise<void> => {
|
||||
const { port1, port2 } = new MessageChannel()
|
||||
@@ -51,7 +52,7 @@ export const speechApi = {
|
||||
const result: { directAudio?: boolean } | undefined = await ipcRenderer.invoke(
|
||||
'speech:startDictation',
|
||||
modelId,
|
||||
hotwords,
|
||||
context,
|
||||
sessionId
|
||||
)
|
||||
directAudio = result?.directAudio === true
|
||||
|
||||
@@ -3,6 +3,7 @@ import { useAppStore } from '@/store'
|
||||
import { useAudioCapture } from '@/hooks/use-audio-capture'
|
||||
import { DictationIndicator } from './DictationIndicator'
|
||||
import { captureInsertionTarget } from './dictation-insertion-target'
|
||||
import { captureDictationContext } from './dictation-context'
|
||||
import { dictationPillAnchorFor, type DictationPillAnchor } from './dictation-pill-anchor'
|
||||
import {
|
||||
canRetryDictation,
|
||||
@@ -207,7 +208,8 @@ export function DictationController() {
|
||||
const insertionTarget = captureInsertionTarget()
|
||||
const deliver = createDictationTranscriptDelivery()
|
||||
deliverRef.current = (text) => deliver(text, insertionTarget)
|
||||
const kept = createDictationKeptRecording(insertionTarget, deliver)
|
||||
const context = captureDictationContext()
|
||||
const kept = createDictationKeptRecording(insertionTarget, deliver, context)
|
||||
keptRef.current = kept
|
||||
setRecordingKey((key) => key + 1)
|
||||
setCaptureLive(false)
|
||||
@@ -270,7 +272,7 @@ export function DictationController() {
|
||||
}
|
||||
setCaptureLive(true)
|
||||
|
||||
await window.api.speech.startDictation(modelId, undefined, sessionId)
|
||||
await window.api.speech.startDictation(modelId, context, sessionId)
|
||||
if (!isCurrent()) {
|
||||
await releaseStart(true)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { useAppStore } from '@/store'
|
||||
import { findRepoForHost } from '@/store/repos/repo-host-identity'
|
||||
import { getRepoIdFromWorktreeId } from '../../../../shared/worktree/id'
|
||||
import type { DictationContext } from '../../../../shared/speech-types'
|
||||
|
||||
/** The worktree a dictation starts in, with its repo's terms for an engine running in this app. */
|
||||
export function captureDictationContext(): DictationContext {
|
||||
const state = useAppStore.getState()
|
||||
const worktreeId = state.activeWorktreeId
|
||||
if (!worktreeId) {
|
||||
return {}
|
||||
}
|
||||
const repo = findRepoForHost(state.repos, getRepoIdFromWorktreeId(worktreeId), {
|
||||
hostId: state.activeWorkspaceExecutionHostId,
|
||||
settings: state.settings
|
||||
})
|
||||
return { worktreeId, ...(repo?.dictationTerms ? { repoTerms: repo.dictationTerms } : {}) }
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
CLOUD_DICTATION_MAX_SECONDS,
|
||||
DICTATION_SAMPLE_RATE
|
||||
} from '../../../../shared/rpc-contract/speech-params'
|
||||
import type { DictationContext } from '../../../../shared/speech-types'
|
||||
import type { DictationInsertionTarget } from './dictation-insertion-target'
|
||||
import type { DictationTranscriptDelivery } from './dictation-transcript-delivery'
|
||||
import { trimLandedOverlap } from './dictation-retry-overlap'
|
||||
@@ -17,11 +18,14 @@ export type DictationKeptRecording = {
|
||||
complete: boolean
|
||||
target: DictationInsertionTarget | null
|
||||
deliver: DictationTranscriptDelivery
|
||||
/** A retry spells with the dictionary of the place it started in. */
|
||||
context: DictationContext
|
||||
}
|
||||
|
||||
export function createDictationKeptRecording(
|
||||
target: DictationInsertionTarget | null,
|
||||
deliver: DictationTranscriptDelivery
|
||||
deliver: DictationTranscriptDelivery,
|
||||
context: DictationContext
|
||||
): DictationKeptRecording & {
|
||||
keep: (samples: Float32Array) => void
|
||||
/** The host stopped accepting audio at its own cap, so the first ten minutes are all it would hear. */
|
||||
@@ -35,6 +39,7 @@ export function createDictationKeptRecording(
|
||||
complete: true,
|
||||
target,
|
||||
deliver,
|
||||
context,
|
||||
keep: (frame: Float32Array): void => {
|
||||
const room = MAX_KEPT_SAMPLES - samples
|
||||
if (room <= 0) {
|
||||
@@ -111,7 +116,7 @@ export async function transcribeKeptRecording(args: {
|
||||
const retried: string[] = []
|
||||
args.routeFinals((text) => retried.push(text))
|
||||
try {
|
||||
await window.api.speech.startDictation(args.modelId, undefined, sessionId)
|
||||
await window.api.speech.startDictation(args.modelId, args.recording.context, sessionId)
|
||||
for (const block of joinIntoBlocks(args.recording.chunks)) {
|
||||
if (!isCurrent()) {
|
||||
break
|
||||
|
||||
@@ -48,6 +48,7 @@ export type RepoUpdate = Partial<
|
||||
| 'worktreeBasePath'
|
||||
| 'kind'
|
||||
| 'symlinkPaths'
|
||||
| 'dictationTerms'
|
||||
| 'issueSourcePreference'
|
||||
| 'defaultJiraProject'
|
||||
| 'forkSyncMode'
|
||||
|
||||
@@ -148,8 +148,8 @@ export function createWebSpeechApi(): Pick<
|
||||
}
|
||||
|
||||
return {
|
||||
// No model id, like the phone: the host dictates with the model it has selected now.
|
||||
startDictation: async (_modelId, _hotwords, sessionId) => {
|
||||
// No model id, like the phone: the host dictates with the model it has selected now, and its own terms for the worktree.
|
||||
startDictation: async (_modelId, context, sessionId) => {
|
||||
const dictation: HostDictation = {
|
||||
sessionId,
|
||||
dictationId: `web-${createNonSecureContextUuid()}`,
|
||||
@@ -164,7 +164,10 @@ export function createWebSpeechApi(): Pick<
|
||||
}
|
||||
active = dictation
|
||||
try {
|
||||
await callRuntimeResult('speech.dictation.start', { dictationId: dictation.dictationId })
|
||||
await callRuntimeResult('speech.dictation.start', {
|
||||
dictationId: dictation.dictationId,
|
||||
...(context?.worktreeId ? { worktreeId: context.worktreeId } : {})
|
||||
})
|
||||
// A cancel that landed while the host was starting couldn't reach this dictation yet.
|
||||
if (active !== dictation) {
|
||||
void cancelOnHost(dictation)
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createDictationDictionary, mergeDictationTerms } from './dictation-dictionary'
|
||||
import { createDictationTextPipeline } from './dictation-text-pipeline'
|
||||
|
||||
const spell = (terms: Parameters<typeof createDictationDictionary>[0], text: string): string =>
|
||||
createDictationDictionary(terms)!(text)
|
||||
|
||||
describe('createDictationDictionary', () => {
|
||||
// Parakeet v3's real output for the dev bench's code-term clips.
|
||||
it('joins the words a model splits a code term into back into the term', () => {
|
||||
const terms = [
|
||||
{ term: 'useAudioCapture' },
|
||||
{ term: 'usePcmWorklet' },
|
||||
{ term: 'stt-worker.ts' },
|
||||
{ term: 'numThreads' },
|
||||
{ term: 'useMemo' },
|
||||
{ term: 'onChange' }
|
||||
]
|
||||
expect(
|
||||
spell(terms, 'Rename use audio capture to use PCM worklet in the dictation controller.')
|
||||
).toBe('Rename useAudioCapture to usePcmWorklet in the dictation controller.')
|
||||
expect(spell(terms, 'Open stt worker TS and set num threads to 4.')).toBe(
|
||||
'Open stt-worker.ts and set numThreads to 4.'
|
||||
)
|
||||
expect(spell(terms, 'Wrap it in use memo and pass the on-change call back.')).toBe(
|
||||
'Wrap it in useMemo and pass the onChange call back.'
|
||||
)
|
||||
})
|
||||
|
||||
it('replaces what the model hears instead, and fixes the spelling of the term itself', () => {
|
||||
const terms = [{ term: 'kubectl', heardAs: 'Quebecal' }, { term: 'Orca' }, { term: 'pnpm' }]
|
||||
expect(spell(terms, 'Use Quebecal to scale it.')).toBe('Use kubectl to scale it.')
|
||||
expect(spell(terms, "Open or ca's settings and run PNPM.")).toBe(
|
||||
"Open Orca's settings and run pnpm."
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves longer words and words split by a sentence end alone', () => {
|
||||
const terms = [{ term: 'React' }, { term: 'onChange' }]
|
||||
expect(spell(terms, 'Reactive state. Turn it on. Change it.')).toBe(
|
||||
'Reactive state. Turn it on. Change it.'
|
||||
)
|
||||
})
|
||||
|
||||
it('replaces a "heard as" inside text written without spaces', () => {
|
||||
expect(spell([{ term: '光輝', heardAs: 'こうき' }], 'こうきさんに送って')).toBe(
|
||||
'光輝さんに送って'
|
||||
)
|
||||
})
|
||||
|
||||
it("prefers a repo's spelling over a personal one", () => {
|
||||
const terms = mergeDictationTerms([{ term: 'RPC' }], [{ term: 'rpc' }])
|
||||
expect(spell(terms, 'Call the rpc.')).toBe('Call the RPC.')
|
||||
})
|
||||
})
|
||||
|
||||
describe('createDictationTextPipeline with terms', () => {
|
||||
it('spells a lowercase term after a sentence-start filler goes', () => {
|
||||
const clean = createDictationTextPipeline({ language: 'en' }, {}, [{ term: 'pnpm' }])!
|
||||
expect(clean('Um, pnpm test.')).toBe('pnpm test.')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,119 @@
|
||||
/** A word or phrase dictation should spell `term`'s way, optionally also replacing what the model hears instead. */
|
||||
export type DictationTerm = { term: string; heardAs?: string }
|
||||
|
||||
export const MAX_DICTATION_TERMS = 500
|
||||
export const MAX_DICTATION_TERM_LENGTH = 100
|
||||
|
||||
const WORD_RE = /[\p{L}\p{M}\p{N}]+/gu
|
||||
// Spaces, or one joiner as in "on-change" or "stt-worker.ts": what may sit between the words of one term.
|
||||
const JOINING_GAP_RE = /^(?:\s+|[-_./])$/u
|
||||
// Scripts written without spaces, where a phrase can only be found inside a longer run of letters.
|
||||
const UNSPACED_SCRIPT_RE =
|
||||
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u
|
||||
const MAX_WORDS_PER_TERM = 8
|
||||
|
||||
// "Use Audio Capture", "use-audio-capture" and "useAudioCapture" all key as "useaudiocapture".
|
||||
function wordKey(text: string): string {
|
||||
return (text.normalize('NFKC').toLowerCase().match(WORD_RE) ?? []).join('')
|
||||
}
|
||||
|
||||
/** Well-formed entries only, trimmed and capped; undefined when the value isn't a list. */
|
||||
export function normalizeDictationTerms(value: unknown): DictationTerm[] | undefined {
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined
|
||||
}
|
||||
const terms: DictationTerm[] = []
|
||||
for (const entry of value) {
|
||||
const term: unknown = entry?.term
|
||||
const heardAs: unknown = entry?.heardAs
|
||||
if (typeof term !== 'string' || !term.trim()) {
|
||||
continue
|
||||
}
|
||||
const alias = typeof heardAs === 'string' ? heardAs.trim() : ''
|
||||
terms.push({
|
||||
term: term.trim().slice(0, MAX_DICTATION_TERM_LENGTH),
|
||||
...(alias ? { heardAs: alias.slice(0, MAX_DICTATION_TERM_LENGTH) } : {})
|
||||
})
|
||||
}
|
||||
return terms.slice(0, MAX_DICTATION_TERMS)
|
||||
}
|
||||
|
||||
/** Repo terms first, so a repo's spelling wins over a personal one for the same words. */
|
||||
export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown): DictationTerm[] {
|
||||
return [
|
||||
...(normalizeDictationTerms(repoTerms) ?? []),
|
||||
...(normalizeDictationTerms(personalTerms) ?? [])
|
||||
]
|
||||
}
|
||||
|
||||
/** Rewrites words that key the same as a term (or its "heard as") to the term's spelling; null when there are none. */
|
||||
export function createDictationDictionary(
|
||||
terms: readonly DictationTerm[] | undefined
|
||||
): ((text: string) => string) | null {
|
||||
const byKey = new Map<string, string>()
|
||||
const unspacedAliases: [heardAs: string, term: string][] = []
|
||||
// A "heard as" is what the user asked for, so it beats another term spelled the same.
|
||||
for (const { term, heardAs } of terms ?? []) {
|
||||
if (heardAs && UNSPACED_SCRIPT_RE.test(heardAs)) {
|
||||
unspacedAliases.push([heardAs, term])
|
||||
} else if (heardAs && wordKey(heardAs) && !byKey.has(wordKey(heardAs))) {
|
||||
byKey.set(wordKey(heardAs), term)
|
||||
}
|
||||
}
|
||||
for (const { term } of terms ?? []) {
|
||||
const key = wordKey(term)
|
||||
if (key && !byKey.has(key)) {
|
||||
byKey.set(key, term)
|
||||
}
|
||||
}
|
||||
if (byKey.size === 0 && unspacedAliases.length === 0) {
|
||||
return null
|
||||
}
|
||||
const longestKey = Math.max(0, ...[...byKey.keys()].map((key) => key.length))
|
||||
return (text) => {
|
||||
for (const [alias, term] of unspacedAliases) {
|
||||
text = text.split(alias).join(term)
|
||||
}
|
||||
return byKey.size > 0 ? replaceWordRuns(text, byKey, longestKey) : text
|
||||
}
|
||||
}
|
||||
|
||||
// At each word, the longest run of joined words whose key is a term's becomes that term.
|
||||
function replaceWordRuns(text: string, byKey: Map<string, string>, longestKey: number): string {
|
||||
const words = [...text.matchAll(WORD_RE)]
|
||||
let out = ''
|
||||
let cursor = 0
|
||||
for (let start = 0; start < words.length;) {
|
||||
let key = ''
|
||||
let match: { end: number; term: string } | null = null
|
||||
for (let end = start; end < words.length && end - start < MAX_WORDS_PER_TERM; end++) {
|
||||
if (end > start) {
|
||||
const gapStart = words[end - 1].index + words[end - 1][0].length
|
||||
if (!JOINING_GAP_RE.test(text.slice(gapStart, words[end].index))) {
|
||||
break
|
||||
}
|
||||
}
|
||||
key += wordKey(words[end][0])
|
||||
if (key.length > longestKey) {
|
||||
break
|
||||
}
|
||||
const term = byKey.get(key)
|
||||
if (term !== undefined) {
|
||||
match = { end, term }
|
||||
}
|
||||
}
|
||||
if (!match) {
|
||||
start++
|
||||
continue
|
||||
}
|
||||
out += text.slice(cursor, words[start].index) + match.term
|
||||
cursor = words[match.end].index + words[match.end][0].length
|
||||
start = match.end + 1
|
||||
}
|
||||
return out + text.slice(cursor)
|
||||
}
|
||||
|
||||
/** The distinct spellings an engine that takes a context prompt is told to expect. */
|
||||
export function dictationVocabulary(terms: readonly DictationTerm[] | undefined): string[] {
|
||||
return [...new Set((terms ?? []).map((entry) => entry.term))]
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createDictationDictionary, type DictationTerm } from './dictation-dictionary'
|
||||
import { engineDictationLanguage } from './dictation-language'
|
||||
import { removeFillerWords } from './dictation-filler-words'
|
||||
import type { SpeechModelManifest, VoiceSettings } from './speech-types'
|
||||
@@ -26,11 +27,18 @@ function dictatedTextLanguages(
|
||||
/** The passes dictated text goes through before anyone sees it, or null when every pass is off. */
|
||||
export function createDictationTextPipeline(
|
||||
model: DictationTextModel | undefined,
|
||||
settings: Partial<DictationTextSettings> | undefined
|
||||
settings: Partial<DictationTextSettings> | undefined,
|
||||
terms?: readonly DictationTerm[]
|
||||
): ((text: string) => string) | null {
|
||||
if (settings?.removeFillerWords === false) {
|
||||
return null
|
||||
}
|
||||
const languages = dictatedTextLanguages(model, settings?.dictationLanguage)
|
||||
return (text) => removeFillerWords(text, languages)
|
||||
const dropFillers =
|
||||
settings?.removeFillerWords === false
|
||||
? null
|
||||
: (text: string) => removeFillerWords(text, languages)
|
||||
const spellTerms = createDictationDictionary(terms)
|
||||
if (!dropFillers || !spellTerms) {
|
||||
return dropFillers ?? spellTerms
|
||||
}
|
||||
// Terms last, so a filler removed at a sentence start can't recapitalize a lowercase term like "pnpm".
|
||||
return (text) => spellTerms(dropFillers(text))
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DictationTerm } from './dictation-dictionary'
|
||||
import type { RepoIcon } from './repo-icon'
|
||||
import type { GhAccountBinding } from './github/account-binding'
|
||||
import type { GitHubRepositoryIdentity } from './github/pull-request-types'
|
||||
@@ -103,6 +104,8 @@ export type Repo = {
|
||||
projectHostSetupMethod?: RepoProjectHostSetupMethod
|
||||
/** Local row a WSL host now owns, kept for a downgrade; set only by the mover's retention pass. */
|
||||
orcadRetained?: OrcadRetainedMarker
|
||||
/** Dictation terms used only in this repo, kept in Orca (on the repo's host), never in the checkout. */
|
||||
dictationTerms?: DictationTerm[]
|
||||
}
|
||||
|
||||
/** Shared IPC shape for repos:getBaseRefDefault across main/preload/renderer; remoteCount avoids a second hint lookup. */
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
} from '../worktree/visibility-sources'
|
||||
import { OptionalFiniteNumber, OptionalString } from './rpc-param-primitives'
|
||||
import { editorFormatterSchema } from '../editor-formatter'
|
||||
import { normalizeDictationTerms } from '../dictation-dictionary'
|
||||
|
||||
export const RepoSourceControlAiOverrides = z
|
||||
.unknown()
|
||||
@@ -57,6 +58,7 @@ export function createRepoUpdateSchema<T extends Readonly<Record<string, z.ZodTy
|
||||
worktreeBasePath: OptionalString,
|
||||
kind: z.enum(['git', 'folder']).optional(),
|
||||
symlinkPaths: z.array(z.string()).optional(),
|
||||
dictationTerms: z.unknown().transform(normalizeDictationTerms).optional(),
|
||||
issueSourcePreference: z.enum(['auto', 'upstream', 'origin']).optional(),
|
||||
defaultJiraProject: z.unknown().optional().transform(normalizeJiraProjectReference),
|
||||
ghAccount: z
|
||||
|
||||
+3
-2
@@ -616,7 +616,8 @@ import {
|
||||
DictationSetup,
|
||||
DictationStart,
|
||||
OpenAiSpeechKeySet,
|
||||
SpeechModelAction
|
||||
SpeechModelAction,
|
||||
SpeechSetupRead
|
||||
} from './speech-params'
|
||||
import {
|
||||
RepoAddRemote,
|
||||
@@ -1525,7 +1526,7 @@ export const RPC_PARAMS_BY_METHOD = {
|
||||
'speech.dictation.start': DictationStart,
|
||||
'speech.models.delete': SpeechModelAction,
|
||||
'speech.models.download': SpeechModelAction,
|
||||
'speech.models.list': null,
|
||||
'speech.models.list': SpeechSetupRead,
|
||||
'speech.openAiKey.clear': null,
|
||||
'speech.openAiKey.set': OpenAiSpeechKeySet,
|
||||
'speech.openAiKey.status': null,
|
||||
|
||||
@@ -31,7 +31,9 @@ export function isValidAudioBase64(value: string): boolean {
|
||||
|
||||
export const DictationStart = z.object({
|
||||
dictationId: requiredString('Missing dictation ID'),
|
||||
modelId: OptionalString
|
||||
modelId: OptionalString,
|
||||
/** Picks the repo's dictionary terms; older hosts strip it. */
|
||||
worktreeId: OptionalString
|
||||
})
|
||||
|
||||
export const DictationChunk = z.object({
|
||||
@@ -53,6 +55,10 @@ export const DictationHandle = z.object({
|
||||
dictationId: requiredString('Missing dictation ID')
|
||||
})
|
||||
|
||||
export const SpeechSetupRead = z.object({
|
||||
worktreeId: OptionalString
|
||||
})
|
||||
|
||||
export const SpeechModelAction = z.object({
|
||||
modelId: requiredString('Missing model ID')
|
||||
})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DictationTerm } from './dictation-dictionary'
|
||||
import type { SmartTier } from './sidebar-list/sort-stamps'
|
||||
import type {
|
||||
AgentStatusState,
|
||||
@@ -177,6 +178,8 @@ export type RuntimeSpeechSetupState = {
|
||||
dictationHoldToRecord: boolean
|
||||
/** Phones that recognize speech themselves clean their own text by this switch. */
|
||||
removeFillerWords: boolean
|
||||
/** ...and spell these terms, repo terms first; absent from older hosts. */
|
||||
dictationTerms?: DictationTerm[]
|
||||
models: RuntimeSpeechModelSummary[]
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DictationTerm } from './dictation-dictionary'
|
||||
export type SpeechModelType =
|
||||
| 'transducer'
|
||||
| 'paraformer'
|
||||
@@ -106,4 +107,12 @@ export type VoiceSettings = {
|
||||
unloadModelAfterMinutes: number | null
|
||||
/** 'off' keeps on-device recognition on the CPU, for machines whose GPU driver hard-crashes. Absent = 'auto'. */
|
||||
speechGpu?: SpeechGpuMode
|
||||
/** Personal dictionary, used in every repo; each repo adds its own Repo.dictationTerms. */
|
||||
dictationTerms?: DictationTerm[]
|
||||
}
|
||||
|
||||
/** Where a dictation goes: an engine in the desktop's own process takes repoTerms, a host resolves worktreeId itself. */
|
||||
export type DictationContext = {
|
||||
worktreeId?: string
|
||||
repoTerms?: DictationTerm[]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user