From 313fb8733e37d4c6e10f725e720177ea7c022a78 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:18:06 +0000 Subject: [PATCH 1/3] fix(speech): reload the warm recognizer once the VAD download lands (#40477) A recognizer that loaded before the Silero VAD file existed kept cutting text only at stop until its next restart. When the VAD lands, the speech service now unloads that recognizer once no dictation uses it and warms it again, so pause cuts start on first use. A running dictation is never interrupted. --- src/main/speech/speech-vad-model.ts | 25 ++++++++++--- src/main/speech/stt-service.test.ts | 54 ++++++++++++++++++++++++---- src/main/speech/stt-service.ts | 22 ++++++++++++ src/main/speech/stt-session-start.ts | 3 +- src/main/speech/stt-session-state.ts | 2 ++ 5 files changed, 93 insertions(+), 13 deletions(-) diff --git a/src/main/speech/speech-vad-model.ts b/src/main/speech/speech-vad-model.ts index 5aa954a01b0..12403954e06 100644 --- a/src/main/speech/speech-vad-model.ts +++ b/src/main/speech/speech-vad-model.ts @@ -19,10 +19,7 @@ export function downloadSileroVadInBackground(models: Pick -): string | undefined { +function placedSileroVadPath(models: Pick): string | undefined { const [file] = SILERO_VAD_MODEL.downloadFiles const path = join(models.getModelDir(SILERO_VAD_MODEL.id), file.name) if (path === verifiedVadPath) { @@ -36,6 +33,24 @@ export function sileroVadModelPath( } catch { // Not downloaded yet, e.g. the model came before VAD support. } - downloadSileroVadInBackground(models) return undefined } + +/** Whether a model-state change for `modelId` means the Silero VAD file is now in place. */ +export function sileroVadLanded( + models: Pick, + modelId: string +): boolean { + return modelId === SILERO_VAD_MODEL.id && placedSileroVadPath(models) !== undefined +} + +/** The downloaded Silero VAD file; when it's missing, starts fetching it and returns undefined. */ +export function sileroVadModelPath( + models: Pick +): string | undefined { + const path = placedSileroVadPath(models) + if (!path) { + downloadSileroVadInBackground(models) + } + return path +} diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index cdd27198850..f397c55b819 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -1,6 +1,9 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import type { ModelManager } from './model-manager' import type { SpeechModelState } from '../../shared/speech-types' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.hoisted(() => { class HoistedMockWorker extends EventTarget { @@ -104,9 +107,16 @@ const stoppedReply = (worker: InstanceType) => ({ streamId: worker.messages.findLast((message) => message.type === 'stop')?.streamId }) -const testModels = (models: Pick): ModelManager => - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SttService reads getModelState and getModelDir, and fetches the VAD with downloadModel. - ({ ...models, downloadModel: vi.fn().mockResolvedValue(undefined) }) as never +const testModels = ( + models: Pick, + onModelStateChange: ModelManager['setProgressCallback'] = () => () => {} +): ModelManager => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SttService reads getModelState and getModelDir, fetches the VAD with downloadModel and watches it land. + ({ + ...models, + downloadModel: vi.fn().mockResolvedValue(undefined), + setProgressCallback: onModelStateChange + }) as never const readyModels = (): ModelManager => testModels({ @@ -122,6 +132,7 @@ async function flushMicrotasks(): Promise { } vi.mock('./model-catalog', () => ({ + isLocalSpeechModel: (manifest: { provider: string }) => manifest.provider === 'local', getCatalogModel: (id: string) => id === 'openai-model' ? { @@ -174,6 +185,31 @@ describe('SttService', () => { resetWorkers() }) + it('reloads a recognizer loaded before the VAD landed once the dictation using it ends', async () => { + const modelsDir = mkdtempSync(join(tmpdir(), 'stt-vad-')) + onTestFinished(() => rmSync(modelsDir, { recursive: true, force: true })) + const watchModels = vi.fn(() => () => {}) + const models = testModels( + { getModelState: readyModels().getModelState, getModelDir: (id) => join(modelsDir, id) }, + watchModels + ) + const service = new SttService(models, spawnMockWorker) + await service.startDictation('model-a', vi.fn(), undefined, 'desktop') + const initOf = (worker = getLastWorker()) => worker?.messages.find((m) => m.type === 'init') + expect(initOf()).toMatchObject({ vadModelPath: undefined }) + + const vadPath = join(modelsDir, 'silero-vad-v6.2', 'silero-vad-v6.2-F32.gguf') + mkdirSync(join(modelsDir, 'silero-vad-v6.2')) + writeFileSync(vadPath, Buffer.alloc(1_241_056)) + watchModels.mock.calls[0]?.[0]('silero-vad-v6.2', -1) + await flushMicrotasks() + expect(getCreatedWorkerCount()).toBe(1) + + await service.stopDictation('desktop') + await vi.waitFor(() => expect(getCreatedWorkerCount()).toBe(2)) + expect(initOf()).toMatchObject({ vadModelPath: vadPath }) + }) + it('gives each device dictating at once only its own text from one recognizer', async () => { const service = new SttService(readyModels(), spawnMockWorker) const desktop = vi.fn() @@ -739,15 +775,19 @@ describe('SttService', () => { async (event) => { vi.useFakeTimers() try { - const models: Pick = { + const models: Pick< + ModelManager, + 'getModelState' | 'getModelDir' | 'downloadModel' | 'setProgressCallback' + > = { getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), getModelDir: vi.fn().mockReturnValue('/tmp/model-a'), - downloadModel: vi.fn().mockResolvedValue(undefined) + downloadModel: vi.fn().mockResolvedValue(undefined), + setProgressCallback: () => () => {} } - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This lifecycle reads getModelState/getModelDir and downloadModel (VAD) on the model manager. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: This lifecycle reads getModelState/getModelDir, downloadModel (VAD) and setProgressCallback on the model manager. const service = new SttService(models as ModelManager, spawnMockWorker) const sink = vi.fn() await service.startDictation('model-a', sink, undefined, 'desktop') diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 048c4872135..2333dfae0ed 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -6,6 +6,7 @@ import { } from '../../shared/dictation-text-pipeline' import { getCatalogModel } from './model-catalog' import type { ModelManager } from './model-manager' +import { sileroVadLanded } from './speech-vad-model' import type { SpawnSttWorkerProcess } from './stt-worker-process' import { prewarmSttModel } from './stt-session-prewarm' import { startSttDictation } from './stt-session-start' @@ -45,6 +46,27 @@ export class SttService { constructor(modelManager: ModelManager, spawnWorker: SpawnSttWorkerProcess) { this.state = createSttSessionState(modelManager, spawnWorker) + modelManager.setProgressCallback((modelId) => { + if (sileroVadLanded(modelManager, modelId)) { + void this.reloadForVad() + } + }) + } + + /** A recognizer loaded before the VAD landed cuts text only at stop; reload it once no dictation uses it. */ + private async reloadForVad(): Promise { + const engine = this.state.engine + if (!engine || engine.retired || engine.vadModelPath) { + return + } + const { modelId, language } = engine + const unloaded = this.unloadModelWhenIdle() + const generation = this.state.prewarmGeneration + await unloaded + // A newer pre-warm or a dictation took over; loading is theirs now. + if (this.state.prewarmGeneration === generation && !this.state.engine) { + await this.prewarm(modelId, language) + } } /** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */ diff --git a/src/main/speech/stt-session-start.ts b/src/main/speech/stt-session-start.ts index cd815fbc920..ee7bd5b1833 100644 --- a/src/main/speech/stt-session-start.ts +++ b/src/main/speech/stt-session-start.ts @@ -187,7 +187,7 @@ export function spawnSttEngine( if (!speechModulesDir) { throw new Error(LOCAL_SPEECH_UNAVAILABLE_ERROR) } - // A VAD fetched now serves the next engine start. + // A VAD fetched now reloads this engine once it is idle (SttService). const vadModelPath = sileroVadModelPath(state.modelManager) clearSttIdleTeardownTimer(state) const worker = state.spawnWorker() @@ -210,6 +210,7 @@ export function spawnSttEngine( worker, detach, retired: false, + vadModelPath, onUnloaded: [], ready: readyPromise.catch((error: unknown) => { failSttEngine(state, engine, STT_WORKER_EXITED_ERROR) diff --git a/src/main/speech/stt-session-state.ts b/src/main/speech/stt-session-state.ts index 21c081dacf9..c54d4dd5eed 100644 --- a/src/main/speech/stt-session-state.ts +++ b/src/main/speech/stt-session-state.ts @@ -19,6 +19,8 @@ export type SttEngine = SttRecognizerConfig & { detach: () => void /** Torn down on purpose, so a load it cut short is not a failure of the model. */ retired: boolean + /** Absent when the VAD wasn't downloaded yet at load, so this process cuts text only at stop. */ + vadModelPath: string | undefined /** Called once this process is gone; set when it must not outlive its last dictation. */ onUnloaded: (() => void)[] } From 18996530407d9744890f196813f3d5cc8061cd4b Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:18:14 +0000 Subject: [PATCH 2/3] feat(settings): manage dictation vocabulary in Voice settings, per user and per repo (#40471) * feat(settings): add a Vocabulary group to Voice settings for personal and repo terms * feat(speech): use the repo name as a dictation hint without recasing one-word names * fix(settings): use Input and Button styling as the design system defines it * fix(settings): chain quick repo-term edits, report a host that drops them, keep the hint off phone replies * fix(settings): key a pending repo-term write by host too * refactor(settings): catch a dropped repo-term write in updateRepo and keep the name hint out of stored terms * fix(settings): one vocabulary chip per term with its aliases, and no repo suffix on the scope tab * fix(settings): skip a bare vocabulary term that already has an entry --- src/main/ipc/speech.ts | 7 +- .../runtime/speech/runtime-dictation-terms.ts | 6 +- .../speech/runtime-mobile-speech-catalog.ts | 4 +- src/main/speech/stt-service.ts | 2 +- .../components/dictation/dictation-context.ts | 6 +- .../components/settings/voice/VoicePane.tsx | 8 + .../settings/voice/VoiceVocabularySetting.tsx | 269 ++++++++++++++++++ .../settings/voice/voice-pane-search.ts | 18 ++ src/renderer/src/i18n/locales/de.json | 23 +- src/renderer/src/i18n/locales/en.json | 23 +- src/renderer/src/i18n/locales/es.json | 23 +- src/renderer/src/i18n/locales/fr.json | 23 +- src/renderer/src/i18n/locales/ja.json | 23 +- src/renderer/src/i18n/locales/ko.json | 23 +- src/renderer/src/i18n/locales/pt.json | 23 +- src/renderer/src/i18n/locales/ru.json | 23 +- src/renderer/src/i18n/locales/tr.json | 23 +- src/renderer/src/i18n/locales/zh-Hant.json | 23 +- src/renderer/src/i18n/locales/zh.json | 23 +- src/renderer/src/store/repos/repo-update.ts | 8 + src/shared/dictation-dictionary.test.ts | 21 +- src/shared/dictation-dictionary.ts | 47 ++- src/shared/speech-types.ts | 3 + 23 files changed, 619 insertions(+), 33 deletions(-) create mode 100644 src/renderer/src/components/settings/voice/VoiceVocabularySetting.tsx diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 2451c5acff7..98f7f52f9b0 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -6,7 +6,7 @@ import { type MessagePortMain } from 'electron' import { abortWhenRendererGone } from './renderer-lifetime-abort' -import { mergeDictationTerms } from '../../shared/dictation-dictionary' +import { dictationTermsFor } from '../../shared/dictation-dictionary' import { SPEECH_MODEL_CATALOG } from '../speech/model-catalog' import { deleteLocalSpeechModel } from '../speech/speech-model-deletion' import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service' @@ -220,7 +220,10 @@ export function registerSpeechHandlers(store: Store): void { break } }, - mergeDictationTerms(context?.repoTerms, store.getSettings().voice?.dictationTerms), + dictationTermsFor(store.getSettings().voice, { + dictationTerms: context?.repoTerms, + displayName: context?.repoName + }), owner, store.getSettings().voice ) diff --git a/src/main/runtime/speech/runtime-dictation-terms.ts b/src/main/runtime/speech/runtime-dictation-terms.ts index 13785a2cc2d..63838db9179 100644 --- a/src/main/runtime/speech/runtime-dictation-terms.ts +++ b/src/main/runtime/speech/runtime-dictation-terms.ts @@ -1,12 +1,12 @@ -import { mergeDictationTerms, type DictationTerm } from '../../../shared/dictation-dictionary' +import { dictationTermsFor, type DictationTerm } from '../../../shared/dictation-dictionary' import { getRepoIdFromWorktreeId } from '../../../shared/worktree/id' import type { RuntimeStore } from '../construction/runtime-store-contract' -/** This host's dictionary for a worktree: its repo's terms, then the personal ones. A folder workspace has no repo row, so it gets the personal ones. */ +/** This host's dictionary for a worktree; a folder workspace has no repo row, so it gets the personal terms. */ export function hostDictationTerms( store: RuntimeStore, worktreeId: string | undefined ): DictationTerm[] { const repo = worktreeId ? store.getRepo(getRepoIdFromWorktreeId(worktreeId)) : undefined - return mergeDictationTerms(repo?.dictationTerms, store.getSettings().voice?.dictationTerms) + return dictationTermsFor(store.getSettings().voice, repo) } diff --git a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts index fa042e5be68..6619e4f7e0b 100644 --- a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts +++ b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts @@ -69,7 +69,9 @@ export class RuntimeMobileSpeechCatalog { dictationMode: voice.dictationMode === 'hold' ? 'hold' : 'toggle', dictationHoldToRecord: voice.dictationHoldToRecord !== false, removeFillerWords: voice.removeFillerWords !== false, - dictationTerms: hostDictationTerms(store, worktreeId), + // The phone has no Whisper prompt to bias, and its text pass would only join the name; keeping the + // hint out also spares a phone page from before hints, which would treat the name as a full term. + dictationTerms: hostDictationTerms(store, worktreeId).filter((entry) => !entry.hint), models } } diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 2333dfae0ed..bacc4b753ed 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -69,7 +69,7 @@ export class SttService { } } - /** `terms`: the dictionary for where this dictation goes, repo terms first (mergeDictationTerms). */ + /** `terms`: the dictionary for where this dictation goes, repo terms first (dictationTermsFor). */ startDictation( modelId: string, sink: SttEventSink, diff --git a/src/renderer/src/components/dictation/dictation-context.ts b/src/renderer/src/components/dictation/dictation-context.ts index f2cbe4088fe..08f9db5290a 100644 --- a/src/renderer/src/components/dictation/dictation-context.ts +++ b/src/renderer/src/components/dictation/dictation-context.ts @@ -11,5 +11,9 @@ export function captureDictationContext(): DictationContext { return {} } const repo = selectRepoByIdForActiveWorkspace(state, getRepoIdFromWorktreeId(worktreeId)) - return { worktreeId, ...(repo?.dictationTerms ? { repoTerms: repo.dictationTerms } : {}) } + return { + worktreeId, + ...(repo?.dictationTerms ? { repoTerms: repo.dictationTerms } : {}), + ...(repo ? { repoName: repo.displayName } : {}) + } } diff --git a/src/renderer/src/components/settings/voice/VoicePane.tsx b/src/renderer/src/components/settings/voice/VoicePane.tsx index b29ea9ff4a7..e95683cda94 100644 --- a/src/renderer/src/components/settings/voice/VoicePane.tsx +++ b/src/renderer/src/components/settings/voice/VoicePane.tsx @@ -12,6 +12,7 @@ import { OpenAiCompatibleProviderDialog } from './OpenAiCompatibleProviderDialog import { OpenAiTranscriptionSettingsRow } from './OpenAiTranscriptionSettingsRow' import { handleVoiceDictationToggle } from './voice-dictation-toggle' import { VoiceDictationLanguageSetting } from './VoiceDictationLanguageSetting' +import { VoiceVocabularySetting } from './VoiceVocabularySetting' import { VOICE_MICROPHONE_ROW_ID, VoiceInputSettings } from './VoiceInputSettings' import { VoiceSetupCard } from './VoiceSetupCard' import { cancelSpeechModelDownload, startSpeechModelDownload } from './speech-model-download' @@ -331,6 +332,13 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J + + + +
) => void +} + +const NO_TERMS: readonly DictationTerm[] = [] + +export function VoiceVocabularySetting({ + voiceSettings, + onUpdateVoiceSettings +}: VoiceVocabularySettingProps): React.JSX.Element { + const t = useTranslate() + // The repo you're working in; a folder workspace has none, so only personal terms show. + const repo = useAppStore((s) => selectRepoByIdForActiveWorkspace(s, s.activeRepoId)) + const updateRepo = useAppStore((s) => s.updateRepo) + const [chosenScope, setScope] = useState('personal') + // A repo write is a round trip (RPC for a remote host); the next edit builds on it, not on the stored list. + const [pending, setPending] = useState<{ + repoId: string + hostId: ExecutionHostId + terms: DictationTerm[] + } | null>(null) + const scope = repo ? chosenScope : 'personal' + const personalTerms = voiceSettings.dictationTerms ?? NO_TERMS + const repoTerms = + pending && repo && pending.repoId === repo.id && pending.hostId === getRepoExecutionHostId(repo) + ? pending.terms + : (repo?.dictationTerms ?? NO_TERMS) + const terms = scope === 'repo' ? repoTerms : personalTerms + + const saveRepoTerms = async ( + repoId: string, + hostId: ExecutionHostId, + next: DictationTerm[] + ): Promise => { + const write = { repoId, hostId, terms: next } + setPending(write) + if (!(await updateRepo(repoId, { dictationTerms: next }, { hostId }))) { + toast.error( + t( + 'components.settings.voice.vocabulary.repoSaveFailed', + "Couldn't save the repo's terms. The host running this repo may need an Orca update." + ) + ) + } + setPending((current) => (current === write ? null : current)) + } + + const saveTerms = (next: DictationTerm[]): void => { + if (scope === 'repo' && repo) { + void saveRepoTerms(repo.id, getRepoExecutionHostId(repo), next) + } else { + onUpdateVoiceSettings({ dictationTerms: next }) + } + } + + return ( +
+ +
+

+ {scope === 'repo' && repo + ? t( + 'components.settings.voice.vocabulary.repoScope', + 'Only used while you dictate in {{repo}}.', + { repo: repo.displayName } + ) + : t('components.settings.voice.vocabulary.personalScope', 'Used in every repo.')} +

+ {repo ? ( + + ) : null} +
+ {terms.length > 0 ? ( +
    + {chipsBySpelling(terms).map((chip) => ( + saveTerms(terms.filter((entry) => entry.term !== chip.term))} + /> + ))} +
+ ) : null} + = MAX_DICTATION_TERMS} + onAdd={(entry) => { + // A term already keys its own spelling, so a bare one needs no second entry. + const exists = terms.some( + (other) => + other.term === entry.term && (!entry.heardAs || other.heardAs === entry.heardAs) + ) + if (!exists) { + saveTerms([...terms, entry]) + } + }} + /> + + onUpdateVoiceSettings({ + dictationRepoNameHint: voiceSettings.dictationRepoNameHint === false + }) + } + /> +
+ ) +} + +type VocabularyChipModel = { term: string; aliases: string[] } + +// One chip per spelling: the stored list keeps one entry per "heard as". +function chipsBySpelling(terms: readonly DictationTerm[]): VocabularyChipModel[] { + const chips = new Map() + for (const { term, heardAs } of terms) { + const chip = chips.get(term) ?? { term, aliases: [] } + if (heardAs) { + chip.aliases.push(heardAs) + } + chips.set(term, chip) + } + return [...chips.values()] +} + +function VocabularyChip({ + chip, + onRemove +}: { + chip: VocabularyChipModel + onRemove: () => void +}): React.JSX.Element { + const t = useTranslate() + return ( +
  • + {chip.term} + {chip.aliases.length > 0 ? ( + + {t('components.settings.voice.vocabulary.heardAsChip', '← {{heardAs}}', { + heardAs: chip.aliases.join(', ') + })} + + ) : null} + +
  • + ) +} + +function VocabularyAddRow({ + full, + onAdd +}: { + full: boolean + onAdd: (entry: DictationTerm) => void +}): React.JSX.Element { + const t = useTranslate() + const [term, setTerm] = useState('') + const [heardAs, setHeardAs] = useState('') + const canAdd = !full && term.trim() !== '' + + const add = (): void => { + if (!canAdd) { + return + } + onAdd({ term: term.trim(), ...(heardAs.trim() ? { heardAs: heardAs.trim() } : {}) }) + setTerm('') + setHeardAs('') + } + const addOnEnter = (event: React.KeyboardEvent): void => { + if (event.key === 'Enter') { + event.preventDefault() + add() + } + } + + return ( +
    + setTerm(event.target.value)} + onKeyDown={addOnEnter} + /> + setHeardAs(event.target.value)} + onKeyDown={addOnEnter} + /> + +
    + ) +} diff --git a/src/renderer/src/components/settings/voice/voice-pane-search.ts b/src/renderer/src/components/settings/voice/voice-pane-search.ts index ff52e8b254c..55abb31d426 100644 --- a/src/renderer/src/components/settings/voice/voice-pane-search.ts +++ b/src/renderer/src/components/settings/voice/voice-pane-search.ts @@ -120,6 +120,24 @@ export const getVoicePaneSearchEntries = createLocalizedCatalog(() => [ ...translateSearchKeyword('auto.components.settings.voice.pane.search.10d45a9fce', 'stt') ] }, + { + title: translate('auto.components.settings.voice.pane.search.vocabularyTitle', 'Vocabulary'), + description: translate( + 'auto.components.settings.voice.pane.search.vocabularyDescription', + 'Names and code terms dictation spells your way, for you or for one repo.' + ), + keywords: [ + ...translateSearchKeyword('auto.components.settings.voice.pane.search.7640ed9848', 'voice'), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.089d31a45b', + 'dictation' + ), + ...translateSearchKeyword( + 'auto.components.settings.voice.pane.search.vocabularyKeyword', + 'dictionary' + ) + ] + }, { title: translate( 'auto.components.settings.voice.pane.search.fillerWordsTitle', diff --git a/src/renderer/src/i18n/locales/de.json b/src/renderer/src/i18n/locales/de.json index 454f252e06d..47e9692fe35 100644 --- a/src/renderer/src/i18n/locales/de.json +++ b/src/renderer/src/i18n/locales/de.json @@ -11299,7 +11299,10 @@ "modelMemoryKeyword": "speicher", "fillerWordsTitle": "Füllwörter entfernen", "fillerWordsDescription": "Entfernt „äh“, „ähm“ und ähnliche Laute aus diktiertem Text.", - "fillerWordsKeyword": "füllwörter" + "fillerWordsKeyword": "füllwörter", + "vocabularyTitle": "Wortschatz", + "vocabularyDescription": "Namen und Code-Begriffe, die das Diktat auf deine Weise schreibt, für dich oder ein Repo.", + "vocabularyKeyword": "wörterbuch" } } }, @@ -20746,6 +20749,24 @@ "local": "Auf diesem Computer", "cloudDescription": "Cloud-Modelle senden deine Audiodaten an den Anbieter. Modelle auf diesem Computer behalten sie hier.", "sendsToCompatible": "Sendet deine Audiodaten an den Dienst, den du eingerichtet hast." + }, + "vocabulary": { + "title": "Wortschatz", + "description": "Namen und Code-Begriffe, die du auf deine Weise geschrieben haben willst. Gib an, wie das Modell sie falsch hört, wenn es sie immer wieder verwechselt.", + "repoScope": "Nur beim Diktieren in {{repo}}.", + "personalScope": "In jedem Repo.", + "scope": "Begriffe für", + "personal": "Persönlich · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} entfernen", + "term": "Wort oder Wendung", + "heardAs": "Gehört als (optional)", + "heardAsLabel": "Gehört als", + "add": "Hinzufügen", + "repoNameHint": "Repo-Namen als Hinweis verwenden", + "repoNameHintDescription": "Hilft, den Namen des Repos zu erkennen.", + "repoSaveFailed": "Die Begriffe des Repos konnten nicht gespeichert werden. Auf dem Host dieses Repos ist eventuell ein Orca-Update nötig." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index fd3e66b2e85..c0bd1dd1335 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -11635,7 +11635,10 @@ "modelMemoryKeyword": "memory", "fillerWordsTitle": "Remove Filler Words", "fillerWordsDescription": "Drop “um”, “uh” and similar sounds from dictated text.", - "fillerWordsKeyword": "filler words" + "fillerWordsKeyword": "filler words", + "vocabularyTitle": "Vocabulary", + "vocabularyDescription": "Names and code terms dictation spells your way, for you or for one repo.", + "vocabularyKeyword": "dictionary" } } }, @@ -21445,6 +21448,24 @@ "where": "Where models run", "local": "On this computer", "cloudDescription": "Cloud models send your audio to the provider. Models on this computer keep it here." + }, + "vocabulary": { + "title": "Vocabulary", + "description": "Names and code terms to spell your way. Add what the model mishears it as, if it keeps getting it wrong.", + "repoScope": "Only used while you dictate in {{repo}}.", + "personalScope": "Used in every repo.", + "scope": "Terms for", + "personal": "Personal · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Remove {{term}}", + "term": "Word or phrase", + "heardAs": "Heard as (optional)", + "heardAsLabel": "Heard as", + "add": "Add", + "repoNameHint": "Use the repo name as a hint", + "repoNameHintDescription": "Helps recognize the repo's name.", + "repoSaveFailed": "Couldn't save the repo's terms. The host running this repo may need an Orca update." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 76d24032fdf..fdbe57e1fcb 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -10882,7 +10882,10 @@ "modelMemoryKeyword": "memoria", "fillerWordsTitle": "Quitar muletillas", "fillerWordsDescription": "Quita «ehm», «em» y sonidos parecidos del texto dictado.", - "fillerWordsKeyword": "muletillas" + "fillerWordsKeyword": "muletillas", + "vocabularyTitle": "Vocabulario", + "vocabularyDescription": "Nombres y términos de código que el dictado escribe a tu manera, para ti o para un repo.", + "vocabularyKeyword": "diccionario" } } }, @@ -20534,6 +20537,24 @@ "local": "En este equipo", "cloudDescription": "Los modelos en la nube envían tu audio al proveedor. Los de este equipo lo mantienen aquí.", "sendsToCompatible": "Envía tu audio al servicio que configuraste." + }, + "vocabulary": { + "title": "Vocabulario", + "description": "Nombres y términos de código que se escriben a tu manera. Añade cómo lo oye mal el modelo si se equivoca una y otra vez.", + "repoScope": "Solo se usa al dictar en {{repo}}.", + "personalScope": "Se usa en todos los repos.", + "scope": "Términos para", + "personal": "Personal · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Quitar {{term}}", + "term": "Palabra o frase", + "heardAs": "Se oye como (opcional)", + "heardAsLabel": "Se oye como", + "add": "Añadir", + "repoNameHint": "Usar el nombre del repo como pista", + "repoNameHintDescription": "Ayuda a reconocer el nombre del repo.", + "repoSaveFailed": "No se pudieron guardar los términos del repo. Puede que el host de este repo necesite actualizar Orca." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 5e54997ebbb..28f2e380f9a 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -11167,7 +11167,10 @@ "modelMemoryKeyword": "mémoire", "fillerWordsTitle": "Supprimer les hésitations", "fillerWordsDescription": "Supprime « euh », « hum » et les sons similaires du texte dicté.", - "fillerWordsKeyword": "hésitations" + "fillerWordsKeyword": "hésitations", + "vocabularyTitle": "Vocabulaire", + "vocabularyDescription": "Noms et termes de code que la dictée écrit à votre façon, pour vous ou pour un dépôt.", + "vocabularyKeyword": "dictionnaire" } } }, @@ -20851,6 +20854,24 @@ "local": "Sur cet ordinateur", "cloudDescription": "Les modèles cloud envoient votre audio au fournisseur. Ceux de cet ordinateur le gardent ici.", "sendsToCompatible": "Envoie votre audio au service que vous avez configuré." + }, + "vocabulary": { + "title": "Vocabulaire", + "description": "Noms et termes de code à écrire à votre façon. Ajoutez ce que le modèle entend à la place s’il se trompe toujours.", + "repoScope": "Utilisé seulement quand vous dictez dans {{repo}}.", + "personalScope": "Utilisé dans tous les dépôts.", + "scope": "Termes pour", + "personal": "Personnel · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Retirer {{term}}", + "term": "Mot ou expression", + "heardAs": "Entendu comme (facultatif)", + "heardAsLabel": "Entendu comme", + "add": "Ajouter", + "repoNameHint": "Utiliser le nom du dépôt comme indice", + "repoNameHintDescription": "Aide à reconnaître le nom du dépôt.", + "repoSaveFailed": "Impossible d’enregistrer les termes du dépôt. L’hôte de ce dépôt doit peut-être mettre Orca à jour." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 6f309bdfab3..4d90f59f72d 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -10626,7 +10626,10 @@ "modelMemoryKeyword": "メモリ", "fillerWordsTitle": "フィラーを削除", "fillerWordsDescription": "「えーと」「うーん」などのつなぎ言葉を音声入力のテキストから取り除きます。", - "fillerWordsKeyword": "フィラー" + "fillerWordsKeyword": "フィラー", + "vocabularyTitle": "語彙", + "vocabularyDescription": "音声入力が自分の書き方で表記する名前やコード用語。個人用またはリポジトリ用。", + "vocabularyKeyword": "辞書" } } }, @@ -20027,6 +20030,24 @@ "local": "このコンピューター", "cloudDescription": "クラウドモデルは音声をプロバイダーに送信します。このコンピューター上のモデルなら音声は外に出ません。", "sendsToCompatible": "音声を設定したサービスに送信します。" + }, + "vocabulary": { + "title": "語彙", + "description": "名前やコード用語を自分の書き方で入力します。モデルが繰り返し聞き間違える場合は、聞き取られる言い方も追加してください。", + "repoScope": "{{repo}} で音声入力するときだけ使います。", + "personalScope": "すべてのリポジトリで使います。", + "scope": "用語の対象", + "personal": "個人 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} を削除", + "term": "単語またはフレーズ", + "heardAs": "聞き取られ方(任意)", + "heardAsLabel": "聞き取られ方", + "add": "追加", + "repoNameHint": "リポジトリ名をヒントとして使う", + "repoNameHintDescription": "リポジトリ名を認識しやすくします。", + "repoSaveFailed": "リポジトリの用語を保存できませんでした。このリポジトリのホストで Orca の更新が必要かもしれません。" } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 068db9b1fbd..db599a834a2 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -10589,7 +10589,10 @@ "modelMemoryKeyword": "메모리", "fillerWordsTitle": "군말 제거", "fillerWordsDescription": "받아쓴 텍스트에서 “음”, “흠” 같은 소리를 지웁니다.", - "fillerWordsKeyword": "군말" + "fillerWordsKeyword": "군말", + "vocabularyTitle": "어휘", + "vocabularyDescription": "받아쓰기가 원하는 대로 표기하는 이름과 코드 용어, 개인용 또는 저장소용.", + "vocabularyKeyword": "사전" } } }, @@ -20027,6 +20030,24 @@ "local": "이 컴퓨터", "cloudDescription": "클라우드 모델은 오디오를 제공업체로 보냅니다. 이 컴퓨터의 모델은 오디오를 밖으로 보내지 않습니다.", "sendsToCompatible": "오디오를 설정한 서비스로 보냅니다." + }, + "vocabulary": { + "title": "어휘", + "description": "원하는 대로 표기할 이름과 코드 용어입니다. 모델이 계속 잘못 들으면 잘못 들리는 말도 추가하세요.", + "repoScope": "{{repo}}에서 받아쓸 때만 사용합니다.", + "personalScope": "모든 저장소에서 사용합니다.", + "scope": "용어 대상", + "personal": "개인 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} 삭제", + "term": "단어 또는 구문", + "heardAs": "들리는 말(선택)", + "heardAsLabel": "들리는 말", + "add": "추가", + "repoNameHint": "저장소 이름을 힌트로 사용", + "repoNameHintDescription": "저장소 이름을 더 잘 인식하도록 돕습니다.", + "repoSaveFailed": "저장소 용어를 저장하지 못했습니다. 이 저장소의 호스트에서 Orca 업데이트가 필요할 수 있습니다." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/pt.json b/src/renderer/src/i18n/locales/pt.json index 12065876a1f..1aad807a1d8 100644 --- a/src/renderer/src/i18n/locales/pt.json +++ b/src/renderer/src/i18n/locales/pt.json @@ -6597,7 +6597,10 @@ "modelMemoryKeyword": "memória", "fillerWordsTitle": "Remover hesitações", "fillerWordsDescription": "Remove “hã”, “hum” e sons parecidos do texto ditado.", - "fillerWordsKeyword": "hesitações" + "fillerWordsKeyword": "hesitações", + "vocabularyTitle": "Vocabulário", + "vocabularyDescription": "Nomes e termos de código que o ditado escreve do seu jeito, para você ou para um repositório.", + "vocabularyKeyword": "dicionário" } } }, @@ -21716,6 +21719,24 @@ "local": "Neste computador", "cloudDescription": "Modelos na nuvem enviam seu áudio ao provedor. Os deste computador o mantêm aqui.", "sendsToCompatible": "Envia seu áudio para o serviço que você configurou." + }, + "vocabulary": { + "title": "Vocabulário", + "description": "Nomes e termos de código escritos do seu jeito. Adicione como o modelo os ouve errado, se ele continuar errando.", + "repoScope": "Usado só ao ditar em {{repo}}.", + "personalScope": "Usado em todos os repositórios.", + "scope": "Termos para", + "personal": "Pessoal · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Remover {{term}}", + "term": "Palavra ou frase", + "heardAs": "Ouvido como (opcional)", + "heardAsLabel": "Ouvido como", + "add": "Adicionar", + "repoNameHint": "Usar o nome do repositório como dica", + "repoNameHintDescription": "Ajuda a reconhecer o nome do repositório.", + "repoSaveFailed": "Não foi possível salvar os termos do repositório. O host deste repositório pode precisar de uma atualização do Orca." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/ru.json b/src/renderer/src/i18n/locales/ru.json index 6cc23c88138..58e12c44554 100644 --- a/src/renderer/src/i18n/locales/ru.json +++ b/src/renderer/src/i18n/locales/ru.json @@ -11651,7 +11651,10 @@ "modelMemoryKeyword": "память", "fillerWordsTitle": "Удалять слова-паразиты", "fillerWordsDescription": "Убирает «э», «эм» и похожие звуки из надиктованного текста.", - "fillerWordsKeyword": "паразиты" + "fillerWordsKeyword": "паразиты", + "vocabularyTitle": "Словарь", + "vocabularyDescription": "Имена и термины из кода, которые диктовка пишет по-вашему, для вас или для репозитория.", + "vocabularyKeyword": "словарь" } } }, @@ -21433,6 +21436,24 @@ "local": "На этом компьютере", "cloudDescription": "Облачные модели отправляют звук провайдеру. Модели на этом компьютере оставляют его здесь.", "sendsToCompatible": "Отправляет звук в настроенный вами сервис." + }, + "vocabulary": { + "title": "Словарь", + "description": "Имена и термины из кода, которые нужно писать по-вашему. Если модель всё время ошибается, добавьте, как она их слышит.", + "repoScope": "Используется только при диктовке в {{repo}}.", + "personalScope": "Используется во всех репозиториях.", + "scope": "Термины для", + "personal": "Личные · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "Удалить {{term}}", + "term": "Слово или фраза", + "heardAs": "Слышится как (необязательно)", + "heardAsLabel": "Слышится как", + "add": "Добавить", + "repoNameHint": "Использовать имя репозитория как подсказку", + "repoNameHintDescription": "Помогает распознавать имя репозитория.", + "repoSaveFailed": "Не удалось сохранить термины репозитория. Возможно, на хосте этого репозитория нужно обновить Orca." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/tr.json b/src/renderer/src/i18n/locales/tr.json index 002982551d7..0893ffd23e7 100644 --- a/src/renderer/src/i18n/locales/tr.json +++ b/src/renderer/src/i18n/locales/tr.json @@ -11283,7 +11283,10 @@ "modelMemoryKeyword": "bellek", "fillerWordsTitle": "Dolgu seslerini kaldır", "fillerWordsDescription": "Dikte edilen metinden “hmm” gibi duraksama seslerini çıkarır.", - "fillerWordsKeyword": "dolgu" + "fillerWordsKeyword": "dolgu", + "vocabularyTitle": "Sözcük dağarcığı", + "vocabularyDescription": "Diktenin senin yazımınla yazdığı adlar ve kod terimleri, senin için veya bir depo için.", + "vocabularyKeyword": "sözlük" } } }, @@ -20728,6 +20731,24 @@ "local": "Bu bilgisayarda", "cloudDescription": "Bulut modelleri sesinizi sağlayıcıya gönderir. Bu bilgisayardaki modeller sesi burada tutar.", "sendsToCompatible": "Sesinizi kurduğunuz hizmete gönderir." + }, + "vocabulary": { + "title": "Sözcük dağarcığı", + "description": "Kendi yazımınla yazılacak adlar ve kod terimleri. Model sürekli yanlış duyuyorsa nasıl duyduğunu da ekle.", + "repoScope": "Yalnızca {{repo}} içinde dikte ederken kullanılır.", + "personalScope": "Her depoda kullanılır.", + "scope": "Terimler", + "personal": "Kişisel · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "{{term}} kaldır", + "term": "Sözcük veya ifade", + "heardAs": "Duyulan biçim (isteğe bağlı)", + "heardAsLabel": "Duyulan biçim", + "add": "Ekle", + "repoNameHint": "Depo adını ipucu olarak kullan", + "repoNameHintDescription": "Depo adının tanınmasına yardımcı olur.", + "repoSaveFailed": "Deponun terimleri kaydedilemedi. Bu deponun ana makinesinde Orca güncellemesi gerekebilir." } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/zh-Hant.json b/src/renderer/src/i18n/locales/zh-Hant.json index 11bd936a4f6..b04ad95e54d 100644 --- a/src/renderer/src/i18n/locales/zh-Hant.json +++ b/src/renderer/src/i18n/locales/zh-Hant.json @@ -11120,7 +11120,10 @@ "modelMemoryKeyword": "記憶體", "fillerWordsTitle": "移除語氣詞", "fillerWordsDescription": "從聽寫文字中移除「嗯」「呃」之類的語氣詞。", - "fillerWordsKeyword": "語氣詞" + "fillerWordsKeyword": "語氣詞", + "vocabularyTitle": "詞彙", + "vocabularyDescription": "聽寫依你的寫法拼寫的名稱和程式碼術語,可用於個人或某個儲存庫。", + "vocabularyKeyword": "詞典" } } }, @@ -20429,6 +20432,24 @@ "local": "這台電腦", "cloudDescription": "雲端模型會把你的音訊傳送給服務商。本機模型會把音訊留在這台電腦上。", "sendsToCompatible": "會把你的音訊傳送到你設定的服務。" + }, + "vocabulary": { + "title": "詞彙", + "description": "依你的寫法拼寫的名稱和程式碼術語。如果模型總是聽錯,可以加上它聽成的說法。", + "repoScope": "僅在 {{repo}} 中聽寫時使用。", + "personalScope": "在所有儲存庫中使用。", + "scope": "術語範圍", + "personal": "個人 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "移除 {{term}}", + "term": "詞語或片語", + "heardAs": "聽成(選填)", + "heardAsLabel": "聽成", + "add": "新增", + "repoNameHint": "將儲存庫名稱用作提示", + "repoNameHintDescription": "幫助辨識儲存庫名稱。", + "repoSaveFailed": "無法儲存儲存庫的術語。執行此儲存庫的主機可能需要更新 Orca。" } }, "repositoryHooks": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 31c5adba076..8985678ae5f 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -10589,7 +10589,10 @@ "modelMemoryKeyword": "内存", "fillerWordsTitle": "移除语气词", "fillerWordsDescription": "从听写文本中去掉“嗯”“呃”之类的语气词。", - "fillerWordsKeyword": "语气词" + "fillerWordsKeyword": "语气词", + "vocabularyTitle": "词汇", + "vocabularyDescription": "听写按你的写法拼写的名称和代码术语,可用于个人或某个仓库。", + "vocabularyKeyword": "词典" } } }, @@ -20026,6 +20029,24 @@ "local": "这台电脑", "cloudDescription": "云端模型会把你的音频发送给服务商。本机模型会把音频留在这台电脑上。", "sendsToCompatible": "会把你的音频发送到你设置的服务。" + }, + "vocabulary": { + "title": "词汇", + "description": "按你的写法拼写的名称和代码术语。如果模型总是听错,可以加上它听成的说法。", + "repoScope": "仅在 {{repo}} 中听写时使用。", + "personalScope": "在所有仓库中使用。", + "scope": "术语范围", + "personal": "个人 · {{total}}", + "repo": "{{repo}} · {{total}}", + "heardAsChip": "← {{heardAs}}", + "remove": "移除 {{term}}", + "term": "词语或短语", + "heardAs": "听成(可选)", + "heardAsLabel": "听成", + "add": "添加", + "repoNameHint": "将仓库名用作提示", + "repoNameHintDescription": "帮助识别仓库名称。", + "repoSaveFailed": "无法保存仓库的术语。运行此仓库的主机可能需要更新 Orca。" } }, "repositoryHooks": { diff --git a/src/renderer/src/store/repos/repo-update.ts b/src/renderer/src/store/repos/repo-update.ts index 3a7604b3831..995331a18a7 100644 --- a/src/renderer/src/store/repos/repo-update.ts +++ b/src/renderer/src/store/repos/repo-update.ts @@ -158,6 +158,14 @@ export function createRepoUpdateActions( ) { return false } + // A host from before dictionaries drops the field and still answers with its row. + if ( + sanitizedUpdates.dictationTerms !== undefined && + JSON.stringify(updatedRepo?.dictationTerms ?? []) !== + JSON.stringify(sanitizedUpdates.dictationTerms) + ) { + return false + } set((s) => { const nextRepos = s.repos.map((r) => { const matchesOwner = ownerHasExplicitHost diff --git a/src/shared/dictation-dictionary.test.ts b/src/shared/dictation-dictionary.test.ts index 1e81f897337..f7b159cd011 100644 --- a/src/shared/dictation-dictionary.test.ts +++ b/src/shared/dictation-dictionary.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { createDictationDictionary, mergeDictationTerms } from './dictation-dictionary' +import { + createDictationDictionary, + dictationTermsFor, + dictationVocabulary +} from './dictation-dictionary' import { createDictationTextPipeline } from './dictation-text-pipeline' const spell = (terms: Parameters[0], text: string): string => @@ -55,8 +59,21 @@ describe('createDictationDictionary', () => { ) }) + it('lets the repo name join its spoken words but never recase a single word', () => { + const oneWord = dictationTermsFor({}, { displayName: 'orca' }) + const words = dictationTermsFor({}, { displayName: 'dd-scratch-repo' }) + + expect(createDictationDictionary(oneWord)).toBeNull() + expect(dictationVocabulary(oneWord)).toEqual(['orca']) + expect(spell(words, 'Open dd scratch repo now.')).toBe('Open dd-scratch-repo now.') + expect(dictationTermsFor({ dictationRepoNameHint: false }, { displayName: 'orca' })).toEqual([]) + }) + it("prefers a repo's spelling over a personal one", () => { - const terms = mergeDictationTerms([{ term: 'RPC' }], [{ term: 'rpc' }]) + const terms = dictationTermsFor( + { dictationTerms: [{ term: 'rpc' }] }, + { dictationTerms: [{ term: 'RPC' }] } + ) expect(spell(terms, 'Call the rpc.')).toBe('Call the RPC.') }) }) diff --git a/src/shared/dictation-dictionary.ts b/src/shared/dictation-dictionary.ts index a66d3f5785b..ad448055809 100644 --- a/src/shared/dictation-dictionary.ts +++ b/src/shared/dictation-dictionary.ts @@ -1,5 +1,8 @@ -/** A word or phrase dictation should spell `term`'s way, optionally also replacing what the model hears instead. */ -export type DictationTerm = { term: string; heardAs?: string } +/** + * A word or phrase dictation should spell `term`'s way, optionally also replacing what the model hears instead. + * `hint` marks the repo name: it biases recognition and joins its words, but never recases a single word. + */ +export type DictationTerm = { term: string; heardAs?: string; hint?: true } export const MAX_DICTATION_TERMS = 500 export const MAX_DICTATION_TERM_LENGTH = 100 @@ -20,32 +23,49 @@ function wordKey(text: string): string { return (text.normalize('NFKC').toLowerCase().match(WORD_RE) ?? []).join('') } -/** Well-formed entries only, trimmed and capped; undefined when the value isn't a list. */ +/** Well-formed, distinct entries only, trimmed and capped; undefined when the value isn't a list. */ export function normalizeDictationTerms(value: unknown): DictationTerm[] | undefined { if (!Array.isArray(value)) { return undefined } const terms: DictationTerm[] = [] + const seen = new Set() for (const entry of value) { const term: unknown = entry?.term const heardAs: unknown = entry?.heardAs if (typeof term !== 'string' || !term.trim()) { continue } - const alias = typeof heardAs === 'string' ? heardAs.trim() : '' + const spelling = term.trim().slice(0, MAX_DICTATION_TERM_LENGTH) + const alias = + typeof heardAs === 'string' ? heardAs.trim().slice(0, MAX_DICTATION_TERM_LENGTH) : '' + // The same spelling and alias twice is one stored entry. + const id = `${spelling}\u0000${alias}` + if (seen.has(id)) { + continue + } + seen.add(id) terms.push({ - term: term.trim().slice(0, MAX_DICTATION_TERM_LENGTH), - ...(alias ? { heardAs: alias.slice(0, MAX_DICTATION_TERM_LENGTH) } : {}) + term: spelling, + ...(alias ? { heardAs: alias } : {}) }) } return terms.slice(0, MAX_DICTATION_TERMS) } -/** Repo terms first, so a repo's spelling wins over a personal one for the same words. */ -export function mergeDictationTerms(repoTerms: unknown, personalTerms: unknown): DictationTerm[] { +/** + * The dictionary for a dictation into `repo` (none for a folder workspace): its terms first, so a repo's + * spelling wins over a personal one, then the personal terms, then its name as a hint unless that's off. + */ +export function dictationTermsFor( + voice: { dictationTerms?: unknown; dictationRepoNameHint?: boolean } | undefined, + repo: { dictationTerms?: unknown; displayName?: string } | undefined +): DictationTerm[] { + const name = repo?.displayName?.trim().slice(0, MAX_DICTATION_TERM_LENGTH) return [ - ...(normalizeDictationTerms(repoTerms) ?? []), - ...(normalizeDictationTerms(personalTerms) ?? []) + ...(normalizeDictationTerms(repo?.dictationTerms) ?? []), + ...(normalizeDictationTerms(voice?.dictationTerms) ?? []), + ...(name && voice?.dictationRepoNameHint !== false ? [{ term: name, hint: true as const }] : []) ] } @@ -75,8 +95,11 @@ export function createDictationDictionary( add(wordKey(heardAs), term) } } - for (const { term } of terms ?? []) { - add(wordKey(term), term) + for (const { term, hint } of terms ?? []) { + // A one-word repo name like "orca" would recase every "Orca"; only a name of several words joins. + if (!hint || (term.match(WORD_RE)?.length ?? 0) > 1) { + add(wordKey(term), term) + } } if (byKey.size === 0 && unspacedAliases.length === 0) { return null diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index bedd641f99f..dafcdfeba76 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -132,10 +132,13 @@ export type VoiceSettings = { speechGpu?: SpeechGpuMode /** Personal dictionary, used in every repo; each repo adds its own Repo.dictationTerms. */ dictationTerms?: DictationTerm[] + /** Adds the current repo's name as a hint (DictationTerm.hint). Absent = on. */ + dictationRepoNameHint?: boolean } /** Where a dictation goes: an engine in the desktop's own process takes repoTerms, a host resolves worktreeId itself. */ export type DictationContext = { worktreeId?: string repoTerms?: DictationTerm[] + repoName?: string } From 854928969e2d5a7251cfba06bea708d401350a75 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:19:11 +0000 Subject: [PATCH 3/3] feat(mobile): add Shift, Ctrl and Alt latches and opt-in agent keys to the terminal key row (#38776) * feat(mobile): smart rail keys for the terminal agent's state, with Shift/Ctrl/Alt latches * fix(mobile): cancel key presses only when the key row really moves * fix(mobile): keep Smart rail on through Reset Defaults and finish two-write keys * fix(mobile): only a finger scrolling the key row cancels its presses * fix(mobile): hold the smart group until the tap's press lands, and drop clicks on keys that moved * refactor(mobile): simplify the smart rail touch hook and layout read * chore(mobile): allowlist the Shift, Ctrl and Alt key names * feat(mobile): Codex async question keys on the smart rail * fix(mobile): drop the Codex Always key, whose key differs by approval Codex binds "don't ask again" to p for a command-prefix rule and a for session and file approvals (0.136 through 0.166). The permission hook doesn't say which one is on screen. --- ...obile-localization-coverage-allowlist.json | 21 ++ .../src/components/CustomKeyModal.styles.ts | 3 + mobile/src/components/CustomKeyModal.tsx | 10 +- mobile/src/components/SmartRailRow.tsx | 24 +++ .../components/TerminalShortcutSettings.tsx | 22 +- mobile/src/i18n/locales/en.json | 23 ++ .../src/session/MobileSessionCommandDock.tsx | 10 +- mobile/src/session/MobileSessionSheets.tsx | 3 + .../session/use-mobile-session-lifecycle.ts | 21 +- .../use-mobile-session-screen-state.ts | 3 + .../session/MobileSessionAccessoryKeys.tsx | 163 +++++++++++--- .../session/MobileSessionLatchKeys.tsx | 58 +++++ .../session/MobileSessionSmartKeys.tsx | 112 ++++++++++ .../session/mobile-terminal-tab-agent.ts | 21 ++ .../terminal/session/terminal-dock-styles.ts | 54 +++++ .../session/use-terminal-dock-keys.ts | 6 +- .../session/use-terminal-key-row-touch.ts | 141 ++++++++++++ .../src/terminal/terminal-accessory-layout.ts | 69 ++++-- .../terminal/input/agent-smart-keys.test.ts | 71 +++++++ src/shared/terminal/input/agent-smart-keys.ts | 200 ++++++++++++++++++ 20 files changed, 967 insertions(+), 68 deletions(-) create mode 100644 mobile/src/components/SmartRailRow.tsx create mode 100644 mobile/src/terminal/session/MobileSessionLatchKeys.tsx create mode 100644 mobile/src/terminal/session/MobileSessionSmartKeys.tsx create mode 100644 mobile/src/terminal/session/use-terminal-key-row-touch.ts create mode 100644 src/shared/terminal/input/agent-smart-keys.test.ts create mode 100644 src/shared/terminal/input/agent-smart-keys.ts diff --git a/config/mobile-localization-coverage-allowlist.json b/config/mobile-localization-coverage-allowlist.json index 50d8243f0f7..265dd13cf28 100644 --- a/config/mobile-localization-coverage-allowlist.json +++ b/config/mobile-localization-coverage-allowlist.json @@ -307,6 +307,27 @@ "dynamic": false, "count": 1 }, + { + "filePath": "mobile/src/terminal/session/MobileSessionLatchKeys.tsx", + "kind": "object-property:label", + "text": "Shift", + "dynamic": false, + "count": 1 + }, + { + "filePath": "mobile/src/terminal/session/MobileSessionLatchKeys.tsx", + "kind": "object-property:label", + "text": "Ctrl", + "dynamic": false, + "count": 1 + }, + { + "filePath": "mobile/src/terminal/session/MobileSessionLatchKeys.tsx", + "kind": "object-property:label", + "text": "Alt", + "dynamic": false, + "count": 1 + }, { "filePath": "mobile/src/tasks/mobile-smart-source-modes.ts", "kind": "object-property:label", diff --git a/mobile/src/components/CustomKeyModal.styles.ts b/mobile/src/components/CustomKeyModal.styles.ts index a95b5abe3be..2921025e632 100644 --- a/mobile/src/components/CustomKeyModal.styles.ts +++ b/mobile/src/components/CustomKeyModal.styles.ts @@ -41,6 +41,9 @@ export const customKeyModalStyles = themedStyleSheet(() => ({ borderRadius: radii.surface, overflow: 'hidden' }, + smartRailGroup: { + marginTop: spacing.md + }, shortcutForm: { paddingTop: spacing.sm }, diff --git a/mobile/src/components/CustomKeyModal.tsx b/mobile/src/components/CustomKeyModal.tsx index b6dfac3cdc6..42afc17fd3c 100644 --- a/mobile/src/components/CustomKeyModal.tsx +++ b/mobile/src/components/CustomKeyModal.tsx @@ -19,6 +19,7 @@ import { useMobileTranslation } from '../i18n/use-mobile-translation' import { MobileSwitch } from './MobileSwitch' import { ListGroup, ListRow } from './list/ListGroup' import { FieldTextInput } from './composer-input/FieldTextInput' +import { SmartRailRow } from './SmartRailRow' type Step = 'choose-type' | 'shortcut-combo' | 'special-keys' | 'text-macro' @@ -62,9 +63,11 @@ type Props = { onClose: () => void onKeysChanged: (keys: CustomKey[]) => void onManageShortcuts?: () => void + smartRail?: { enabled: boolean; onChange: (enabled: boolean) => void } } -export function CustomKeyModal({ visible, onClose, onKeysChanged, onManageShortcuts }: Props) { +export function CustomKeyModal(props: Props) { + const { visible, onClose, onKeysChanged, onManageShortcuts, smartRail } = props useMobileColorScheme() const t = useMobileTranslation() const [step, setStep] = useState('choose-type') @@ -235,6 +238,11 @@ export function CustomKeyModal({ visible, onClose, onKeysChanged, onManageShortc ) : null} )} + {step === 'choose-type' && smartRail ? ( + + + + ) : null} {step === 'shortcut-combo' && ( diff --git a/mobile/src/components/SmartRailRow.tsx b/mobile/src/components/SmartRailRow.tsx new file mode 100644 index 00000000000..8789fcd0f42 --- /dev/null +++ b/mobile/src/components/SmartRailRow.tsx @@ -0,0 +1,24 @@ +import { ListRow } from './list/ListGroup' +import { MobileSwitch } from './MobileSwitch' +import { useMobileTranslation } from '../i18n/use-mobile-translation' + +/** The Smart rail setting: keys for what the terminal's agent is doing, at the front of the key row. */ +export function SmartRailRow({ + enabled, + onChange +}: { + enabled: boolean + onChange: (enabled: boolean) => void +}): React.JSX.Element { + const t = useMobileTranslation() + const title = t('components.smartRailRow.title', 'Smart rail') + return ( + + } + /> + ) +} diff --git a/mobile/src/components/TerminalShortcutSettings.tsx b/mobile/src/components/TerminalShortcutSettings.tsx index db5bfe5b64e..37e14d0620f 100644 --- a/mobile/src/components/TerminalShortcutSettings.tsx +++ b/mobile/src/components/TerminalShortcutSettings.tsx @@ -27,6 +27,7 @@ import { setTerminalAccessoryBuiltInVisible } from '../terminal/terminal-accessory-layout' import { MobileSwitch } from './MobileSwitch' +import { SmartRailRow } from './SmartRailRow' // Absolutely positioned DragReorderList rows require one fixed height per section. const REORDER_ROW_HEIGHT = 56 @@ -148,10 +149,13 @@ export function TerminalShortcutSettings({ [persistLayout] ) + // Resetting the keys leaves the Smart rail setting alone. const resetBuiltInKeys = useCallback(() => { - const next = defaultLayout() - setShortcutLayout(next) - persistLayout(next) + setShortcutLayout((current) => { + const next = { ...defaultLayout(), smartRail: current.smartRail } + persistLayout(next) + return next + }) }, [defaultLayout, persistLayout]) const reorderCustomKeys = useCallback( @@ -182,6 +186,18 @@ export function TerminalShortcutSettings({ return ( <> + + + setShortcutLayout((current) => { + const next = { ...current, smartRail: enabled } + persistLayout(next) + return next + }) + } + /> + {hasPad ? ( { - // A ctrl/alt/shift held on the Duo rail turns the next typed letter into a combo, while the rail shows it. - if ( - !keysInRail || - !dockKeys.interceptTypedCharacter(text, event.nativeEvent.text, liveInputRef) - ) { + // A held Shift, Ctrl or Alt on the key row or the Duo rail turns the next typed letter into a combo. + if (!dockKeys.interceptTypedCharacter(text, event.nativeEvent.text, liveInputRef)) { handleLiveInputChange(event) } }} diff --git a/mobile/src/session/MobileSessionSheets.tsx b/mobile/src/session/MobileSessionSheets.tsx index 027af33cf82..fe6f590a3d2 100644 --- a/mobile/src/session/MobileSessionSheets.tsx +++ b/mobile/src/session/MobileSessionSheets.tsx @@ -77,6 +77,8 @@ export function MobileSessionSheets({ controller }: { controller: MobileSessionC confirmDiscardMarkdown, handleDeleteCustomKey, handleManageShortcuts, + smartRailEnabled, + handleSmartRailChange, handleClearTerminal, handleCreateTerminal, launchQuickCommand, @@ -363,6 +365,7 @@ export function MobileSessionSheets({ controller }: { controller: MobileSessionC onClose={() => setShowCustomKeyModal(false)} onKeysChanged={setCustomKeys} onManageShortcuts={handleManageShortcuts} + smartRail={{ enabled: smartRailEnabled, onChange: handleSmartRailChange }} /> { + // A read that started before the toggle must not flip it back. + layoutReadRevisionRef.current += 1 + setSmartRailEnabled(enabled) + saveTerminalSmartRail(enabled, padAvailable).catch((error: unknown) => { + console.warn('[terminal-keys] the store would not take the Smart rail setting', error) + setSmartRailEnabled(!enabled) + }) + }, + [padAvailable] + ) + useFocusEffect( useCallback(() => { const releaseKeys = refreshCustomKeys() @@ -195,6 +213,7 @@ export function useMobileSessionLifecycle(scope: MobileSessionTabReconciliationM }, [connState, scheduleDelayedAction, subscribeToTerminal, terminalResume, unsubscribeTerminal]) return { accessoryPreferencesReady: customKeysLoaded && accessoryLayoutLoaded, + handleSmartRailChange, pendingForegroundRecoveryRef } } diff --git a/mobile/src/session/use-mobile-session-screen-state.ts b/mobile/src/session/use-mobile-session-screen-state.ts index eb0f7d28b74..bde653c4045 100644 --- a/mobile/src/session/use-mobile-session-screen-state.ts +++ b/mobile/src/session/use-mobile-session-screen-state.ts @@ -82,6 +82,7 @@ export function useMobileSessionScreenState(scope: MobileSessionFoundationModel) const [renameTarget, setRenameTarget] = useState(null) const [customKeys, setCustomKeys] = useState([]) const [directionPadEnabled, setDirectionPadEnabled] = useState(true) + const [smartRailEnabled, setSmartRailEnabled] = useState(false) const [visibleBuiltInIds, setVisibleBuiltInIds] = useState( getDefaultTerminalAccessoryBuiltInIds ) @@ -182,6 +183,8 @@ export function useMobileSessionScreenState(scope: MobileSessionFoundationModel) setCustomKeys, directionPadEnabled, setDirectionPadEnabled, + smartRailEnabled, + setSmartRailEnabled, visibleBuiltInIds, setVisibleBuiltInIds, showCustomKeyModal, diff --git a/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx b/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx index f88d4d336c7..737327fab48 100644 --- a/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx +++ b/mobile/src/terminal/session/MobileSessionAccessoryKeys.tsx @@ -21,6 +21,17 @@ import { styles as sessionStyles } from '../../session/mobile-session-styles' import { useKeyboardPersistingTaps } from '../../platform/keyboard-persisting-taps' import type { MobileSessionController } from '../../session/use-mobile-session-controller' import { useMobileSessionKeyboardLift } from './use-mobile-session-keyboard-lift' +import { + AGENT_SMART_KEY_STEP_MS, + type AgentSmartKey +} from '../../../../src/shared/terminal/input/agent-smart-keys' +import type { TerminalAccessoryKey } from '../terminal-accessory-keys' +import { resolveMobileTerminalTabSmartKeys } from './mobile-terminal-tab-agent' +import { MobileSessionSmartKeys } from './MobileSessionSmartKeys' +import { MobileSessionLatchKeys } from './MobileSessionLatchKeys' +import { isAccessibilityClick, useTerminalKeyRowTouch } from './use-terminal-key-row-touch' + +type PreparedPress = ReturnType // The terminal's key row above the live input; every key goes straight to the terminal. export function MobileSessionAccessoryKeys({ @@ -46,10 +57,26 @@ export function MobileSessionAccessoryKeys({ stopAccessoryRepeat, cancelAccessoryRepeat, prepareAccessoryPress, - handlePaste + handlePaste, + dockKeys, + activeSessionTab, + terminals, + smartRailEnabled } = controller const keyboardLift = useMobileSessionKeyboardLift(controller.dockBottomInset) - const accessoryPresses = useRef(new Map>()) + const accessoryPresses = useRef(new Map()) + const scrollRef = useRef(null) + const latestSmartKeys = useMemo( + () => + smartRailEnabled ? resolveMobileTerminalTabSmartKeys(activeSessionTab, terminals) : null, + [smartRailEnabled, activeSessionTab, terminals] + ) + const touch = useTerminalKeyRowTouch({ + scrollRef, + latest: latestSmartKeys, + cancelAccessoryRepeat + }) + const smartKeys = touch.shown const keepsKeyboard = useKeyboardPersistingTaps('always') // An unmounting Pressable never fires onPressOut, so a held key would outlive its row. useEffect(() => cancelAccessoryRepeat, [cancelAccessoryRepeat]) @@ -70,8 +97,57 @@ export function MobileSessionAccessoryKeys({ ] const sendKeyLabel = (label: string) => t('session.accessoryKeys.sendKey', 'Send {{label}}', { label }) + // Held Shift, Ctrl or Alt turn a key into its combo and are used up when it sends; keys that take none send as they are. + const prepareBuiltInPress = (key: TerminalAccessoryKey): PreparedPress => { + const chord = dockKeys.chordInput(key.id) + const press = prepareAccessoryPress(chord ?? createTerminalLiveAccessoryInput(key)) + const takeChord = () => { + if (chord) { + dockKeys.takeModifiers() + } + } + return { + tap: () => { + takeChord() + return press.tap() + }, + hold: (firstRepeatDelayMs) => { + takeChord() + press.hold(firstRepeatDelayMs) + } + } + } + // Later writes belong to the same press, so a scroll after release mustn't drop them; a tab switch does. + const sendSmartKey = (key: AgentSmartKey) => { + const handle = controller.activeHandleRef.current + void (async () => { + for (const [index, bytes] of key.writes.entries()) { + if (index > 0) { + await new Promise((resolve) => setTimeout(resolve, AGENT_SMART_KEY_STEP_MS)) + } + if (controller.activeHandleRef.current !== handle) { + return + } + if (!(await prepareAccessoryPress({ bytes }).tap())) { + return + } + } + })() + } + const latchIndex = builtInKeys.findIndex((key) => key.id === 'tab') + 1 + const latchKeys = ( + touch.arm('latch')} + onPress={(modifier, event) => + touch.fire('latch', event, () => dockKeys.tapModifier(modifier)) + } + /> + ) return ( - + {/* Why: fixed keyboard escape hatch; outside the ScrollView so it can't scroll away (#5106). */} {keyboardLift > 0 && ( + {smartKeys ? ( + touch.arm(`smart:${key.id}`)} + onPress={(key, event) => touch.fire(`smart:${key.id}`, event, () => sendSmartKey(key))} + /> + ) : null} { - if (activeHandle) { - void toggleDisplayMode(activeHandle) - } - }} + onPressIn={() => touch.arm('displayMode')} + onPress={(event) => + touch.fire('displayMode', event, () => { + if (activeHandle) { + void toggleDisplayMode(activeHandle) + } + }) + } accessibilityRole="button" accessibilityLabel={ phoneDisplay @@ -133,7 +222,8 @@ export function MobileSessionAccessoryKeys({ style={keyStyle} disabled={!canSend} hitSlop={TERMINAL_KEY_HIT_SLOP} - onPress={() => void handlePaste()} + onPressIn={() => touch.arm('paste')} + onPress={(event) => touch.fire('paste', event, () => void handlePaste())} accessibilityRole="button" accessibilityLabel={t( 'session.accessoryKeys.pasteFromClipboard', @@ -143,20 +233,22 @@ export function MobileSessionAccessoryKeys({ {t('session.accessoryKeys.paste', 'Paste')} )} - {builtInKeys.map((key) => ( + {latchIndex === 0 ? latchKeys : null} + {builtInKeys.flatMap((key, index) => [ { + touch.arm(key.id) if (!key.repeatable) { return } - const press = prepareAccessoryPress(createTerminalLiveAccessoryInput(key)) - accessoryPresses.current.set(key.id, press) + const prepared = prepareBuiltInPress(key) + accessoryPresses.current.set(key.id, prepared) if (event.type === 'keydown') { - press.hold(TERMINAL_ACCESSORY_REPEAT_DELAY_MS) + prepared.hold(TERMINAL_ACCESSORY_REPEAT_DELAY_MS) } }} onPressOut={() => { @@ -165,23 +257,23 @@ export function MobileSessionAccessoryKeys({ } }} onPress={(event) => { - if (key.repeatable) { - // Accessibility clicks have no touch-down to prepare a sender. - const accessibilityClick = - event.type === 'click' && - 'detail' in event.nativeEvent && - event.nativeEvent.detail === 0 - const press = - (!accessibilityClick && accessoryPresses.current.get(key.id)) || - prepareAccessoryPress(createTerminalLiveAccessoryInput(key)) - accessoryPresses.current.delete(key.id) - void press.tap() - return - } - void handleAccessoryKey(createTerminalLiveAccessoryInput(key)) + // Accessibility clicks have no touch-down to prepare a sender. + const prepared = + (key.repeatable && + !isAccessibilityClick(event) && + accessoryPresses.current.get(key.id)) || + prepareBuiltInPress(key) + accessoryPresses.current.delete(key.id) + touch.fire(key.id, event, () => void prepared.tap()) }} onLongPress={ - key.repeatable ? () => accessoryPresses.current.get(key.id)?.hold() : undefined + key.repeatable + ? () => { + if (touch.stillArmed(key.id)) { + accessoryPresses.current.get(key.id)?.hold() + } + } + : undefined } delayLongPress={TERMINAL_ACCESSORY_REPEAT_DELAY_MS} // A web