From 6f8403a20c8bac4e63e7b4200e1a907bc11fa7af Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:56:59 +0000 Subject: [PATCH] feat(dictation): optional AI cleanup of dictated text, with a correction word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Off by default. When on, a dictation's text goes through the user's agent CLI (the AI text actions' runner, Haiku by default for Claude) before it lands: the pill says "Cleaning up…" and the text is inserted once. Failure or a 10 s timeout inserts it as heard with a warning; the pill's X skips cleanup. A consent dialog asks before text first goes to an agent. The correction word ("scratch that") survives filler removal for cleanup to act on. The microphone test gets "Try cleanup", and phone dictation the computer transcribes is cleaned on the host. --- mobile/src/i18n/locales/en.json | 5 +- .../dictation-cleanup-failure-copy.ts | 25 ++ .../text-generation-failure-copy.ts | 4 +- .../conversation-name-request-sessions.ts | 18 -- .../session-scanner-candidate-parse.ts | 4 +- .../parsing/text-task-request-sessions.ts | 22 ++ .../register-core-handlers.ts | 2 +- src/main/ipc/speech.ts | 46 +++- .../opencode/text-tasks/opencode-text-task.ts | 4 +- .../wsl/opencode-wsl-text-task-entry.ts | 3 +- .../runtime-service-command-surface.ts | 3 +- .../runtime-mobile-dictation-controller.ts | 33 ++- src/main/speech/dictation-cleanup-service.ts | 105 ++++++++ .../agent-generation-request.ts | 12 +- .../dictation-cleanup-generation-request.ts | 42 ++++ .../source-control-agent-failure.ts | 34 ++- .../source-control-local-process.ts | 3 +- .../source-control-remote-generation.ts | 3 +- .../source-control-text-generation-types.ts | 1 + src/preload/api/speech-api.ts | 8 + src/preload/api/speech-bridge.ts | 9 + .../dictation/DictationCleanupNoticePill.tsx | 57 +++++ .../dictation/DictationController.tsx | 89 +++---- .../dictation/DictationIndicator.fixture.tsx | 38 ++- .../dictation/DictationIndicator.tsx | 33 ++- .../dictation/dictation-stopped-sessions.ts | 15 ++ .../dictation-transcript-delivery.ts | 39 ++- .../dictation/use-dictation-cleanup.ts | 87 +++++++ .../dictation/use-dictation-retry.ts | 64 +++++ .../settings-capability-section-renderers.tsx | 6 +- .../source-control-action-recipe-options.ts | 12 +- .../SourceControlTextActionAgentPicker.tsx | 26 +- .../source-control-text-action-picker-save.ts | 5 +- .../voice/VoiceCleanupConsentDialog.tsx | 109 ++++++++ .../voice/VoiceCleanupInstructionsSetting.tsx | 86 +++++++ .../settings/voice/VoiceCleanupSettings.tsx | 235 ++++++++++++++++++ .../settings/voice/VoiceCleanupTry.tsx | 77 ++++++ .../voice/VoiceMicrophoneTestPanel.tsx | 4 + .../settings/voice/VoicePane.test.tsx | 16 +- .../components/settings/voice/VoicePane.tsx | 16 +- .../settings/voice/voice-pane-search.ts | 22 ++ src/renderer/src/i18n/locales/en.json | 72 +++++- .../dictation-cleanup-error-copy.ts | 25 ++ .../text-generation-error-message-catalog.ts | 4 +- src/shared/claude-background-run-isolation.ts | 17 +- src/shared/dictation-cleanup.ts | 114 +++++++++ src/shared/dictation-filler-words.test.ts | 6 + src/shared/dictation-filler-words.ts | 28 ++- src/shared/dictation-text-pipeline.ts | 10 +- .../languages/protected-copy-keys.json | 13 + .../generation/commit-message-agent-output.ts | 9 + .../generation/commit-message-agent-spec.ts | 1 + .../commit-message-agent-specs-primary.ts | 4 +- .../ai/generation/commit-message-plan.ts | 3 + .../source-control-ai-action-variables.ts | 1 + .../ai/recipes/source-control-ai-actions.ts | 12 +- .../source-control-ai-action-model-choice.ts | 11 +- .../source-control-ai-action-model-edits.ts | 3 +- .../source-control-ai-model-selection.ts | 12 +- .../ai/source-control-ai-operation-policy.ts | 47 ++-- .../source-control-ai-operation-resolution.ts | 46 ++-- src/shared/speech-types.ts | 8 + src/shared/text-generation-failure-copy.ts | 5 + 63 files changed, 1665 insertions(+), 208 deletions(-) create mode 100644 mobile/src/source-control/dictation-cleanup-failure-copy.ts delete mode 100644 src/main/ai-vault/parsing/conversation-name-request-sessions.ts create mode 100644 src/main/ai-vault/parsing/text-task-request-sessions.ts create mode 100644 src/main/speech/dictation-cleanup-service.ts create mode 100644 src/main/text-generation/dictation-cleanup-generation-request.ts create mode 100644 src/renderer/src/components/dictation/DictationCleanupNoticePill.tsx create mode 100644 src/renderer/src/components/dictation/use-dictation-cleanup.ts create mode 100644 src/renderer/src/components/dictation/use-dictation-retry.ts create mode 100644 src/renderer/src/components/settings/voice/VoiceCleanupConsentDialog.tsx create mode 100644 src/renderer/src/components/settings/voice/VoiceCleanupInstructionsSetting.tsx create mode 100644 src/renderer/src/components/settings/voice/VoiceCleanupSettings.tsx create mode 100644 src/renderer/src/components/settings/voice/VoiceCleanupTry.tsx create mode 100644 src/renderer/src/lib/text-generation/dictation-cleanup-error-copy.ts create mode 100644 src/shared/dictation-cleanup.ts diff --git a/mobile/src/i18n/locales/en.json b/mobile/src/i18n/locales/en.json index 46dc3c90822..7fbf79c5313 100644 --- a/mobile/src/i18n/locales/en.json +++ b/mobile/src/i18n/locales/en.json @@ -3933,7 +3933,10 @@ "planAgentUnsupported": "Agent \"{{agent}}\" does not support AI commit messages.", "planModelUnavailable": "Model \"{{model}}\" is not available for {{label}}.", "planModelWithoutThinking": "Model \"{{model}}\" does not support a thinking effort level.", - "planThinkingLevelInvalid": "Thinking level \"{{level}}\" is not valid for {{model}}." + "planThinkingLevelInvalid": "Thinking level \"{{level}}\" is not valid for {{model}}.", + "commandTemplateEmptyDictationCleanup": "Command template is empty for dictation cleanup.", + "dictationCleanupAgentUnsupported": "Choose a supported agent for AI cleanup in Settings -> Voice -> Cleanup. Supported agents: {{agents}}, or Custom command.", + "agentCannotGenerateDictationCleanup": "Agent \"{{agent}}\" does not support dictation cleanup. Supported agents: {{agents}}, or Custom command." }, "conflictActions": { "resolveFile": "{{action}}: {{path}}", diff --git a/mobile/src/source-control/dictation-cleanup-failure-copy.ts b/mobile/src/source-control/dictation-cleanup-failure-copy.ts new file mode 100644 index 00000000000..b343d74f474 --- /dev/null +++ b/mobile/src/source-control/dictation-cleanup-failure-copy.ts @@ -0,0 +1,25 @@ +import type { TextGenerationFailureCode } from '../../../src/shared/text-generation-failure-copy' +import { translate } from '../i18n/mobile-i18n' + +export const DICTATION_CLEANUP_FAILURE_COPY = { + commandTemplateEmptyDictationCleanup: (values) => + translate( + 'sourceControl.textGenerationFailure.commandTemplateEmptyDictationCleanup', + 'Command template is empty for dictation cleanup.', + values + ), + dictationCleanupAgentUnsupported: (values) => + translate( + 'sourceControl.textGenerationFailure.dictationCleanupAgentUnsupported', + 'Choose a supported agent for AI cleanup in Settings -> Voice -> Cleanup. Supported agents: {{agents}}, or Custom command.', + values + ), + agentCannotGenerateDictationCleanup: (values) => + translate( + 'sourceControl.textGenerationFailure.agentCannotGenerateDictationCleanup', + 'Agent "{{agent}}" does not support dictation cleanup. Supported agents: {{agents}}, or Custom command.', + values + ) +} satisfies Partial< + Record) => string> +> diff --git a/mobile/src/source-control/text-generation-failure-copy.ts b/mobile/src/source-control/text-generation-failure-copy.ts index be32494f6e6..c50f7eada9e 100644 --- a/mobile/src/source-control/text-generation-failure-copy.ts +++ b/mobile/src/source-control/text-generation-failure-copy.ts @@ -5,6 +5,7 @@ import { import { translate } from '../i18n/mobile-i18n' import { AGENT_OUTPUT_FAILURE_COPY } from './agent-output-failure-copy' import { TEXT_GENERATION_PLAN_FAILURE_COPY } from './text-generation-plan-failure-copy' +import { DICTATION_CLEANUP_FAILURE_COPY } from './dictation-cleanup-failure-copy' type CopyValues = Record @@ -256,7 +257,8 @@ const COPY: Record string> = values ), ...TEXT_GENERATION_PLAN_FAILURE_COPY, - ...AGENT_OUTPUT_FAILURE_COPY + ...AGENT_OUTPUT_FAILURE_COPY, + ...DICTATION_CLEANUP_FAILURE_COPY } function isTextGenerationFailureCode(code: unknown): code is TextGenerationFailureCode { diff --git a/src/main/ai-vault/parsing/conversation-name-request-sessions.ts b/src/main/ai-vault/parsing/conversation-name-request-sessions.ts deleted file mode 100644 index da85830e89f..00000000000 --- a/src/main/ai-vault/parsing/conversation-name-request-sessions.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { AiVaultSession } from '../../../shared/ai-vault/contracts/ai-vault-types' -import { CONVERSATION_NAME_PROMPT_OPENING } from '../../../shared/source-control/ai/generation/conversation-name-generation' - -const MAX_NAME_REQUEST_MESSAGES = 6 - -/** CLI naming runs share chat folders; keep sessions with user follow-ups visible (#29973). */ -export function isConversationNameRequestSession(session: AiVaultSession): boolean { - if (session.previewMessagesTruncated || session.messageCount > MAX_NAME_REQUEST_MESSAGES) { - return false - } - const [request, ...followUps] = session.previewMessages.filter( - (message) => message.role === 'user' - ) - return ( - followUps.length === 0 && - request?.text.trimStart().startsWith(CONVERSATION_NAME_PROMPT_OPENING) === true - ) -} diff --git a/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts b/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts index d377d13b964..e0adfc2acc3 100644 --- a/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts +++ b/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts @@ -1,5 +1,5 @@ import { yieldToEventLoop } from '../../../shared/event-loop-yield' -import { isConversationNameRequestSession } from './conversation-name-request-sessions' +import { isTextTaskRequestSession } from './text-task-request-sessions' import { candidateFileTime } from '../scanners/antigravity/antigravity-transcript-candidates' import { aiVaultAgentLabel, @@ -107,7 +107,7 @@ export async function parseSessionCandidates(args: { } if ( result.session && - !isConversationNameRequestSession(result.session) && + !isTextTaskRequestSession(result.session) && (!args.admitSession || args.admitSession(result.session)) ) { if (args.listed === false) { diff --git a/src/main/ai-vault/parsing/text-task-request-sessions.ts b/src/main/ai-vault/parsing/text-task-request-sessions.ts new file mode 100644 index 00000000000..601dee915b1 --- /dev/null +++ b/src/main/ai-vault/parsing/text-task-request-sessions.ts @@ -0,0 +1,22 @@ +import type { AiVaultSession } from '../../../shared/ai-vault/contracts/ai-vault-types' +import { DICTATION_CLEANUP_PROMPT_OPENING } from '../../../shared/dictation-cleanup' +import { CONVERSATION_NAME_PROMPT_OPENING } from '../../../shared/source-control/ai/generation/conversation-name-generation' + +const MAX_TEXT_TASK_REQUEST_MESSAGES = 6 + +/** One-shot CLI runs (chat naming, dictation cleanup) aren't conversations; sessions with user follow-ups stay visible (#29973). */ +export function isTextTaskRequestSession(session: AiVaultSession): boolean { + if (session.previewMessagesTruncated || session.messageCount > MAX_TEXT_TASK_REQUEST_MESSAGES) { + return false + } + const [request, ...followUps] = session.previewMessages.filter( + (message) => message.role === 'user' + ) + const text = request?.text.trimStart() + return ( + followUps.length === 0 && + text !== undefined && + (text.startsWith(CONVERSATION_NAME_PROMPT_OPENING) || + text.startsWith(DICTATION_CLEANUP_PROMPT_OPENING)) + ) +} diff --git a/src/main/ipc/register-core-handlers/register-core-handlers.ts b/src/main/ipc/register-core-handlers/register-core-handlers.ts index f100d92b59f..728c8c79cb9 100644 --- a/src/main/ipc/register-core-handlers/register-core-handlers.ts +++ b/src/main/ipc/register-core-handlers/register-core-handlers.ts @@ -296,5 +296,5 @@ export function registerCoreHandlers( } }) registerUpdaterHandlers(store) - registerSpeechHandlers(store) + registerSpeechHandlers(store, commitMessageAgentEnv) } diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 98f7f52f9b0..102a758c7a4 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -7,6 +7,7 @@ import { } from 'electron' import { abortWhenRendererGone } from './renderer-lifetime-abort' import { dictationTermsFor } from '../../shared/dictation-dictionary' +import { dictationTextSettingsFor } from '../../shared/dictation-cleanup' import { SPEECH_MODEL_CATALOG } from '../speech/model-catalog' import { deleteLocalSpeechModel } from '../speech/speech-model-deletion' import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service' @@ -25,8 +26,49 @@ import { } from '../speech/openai-compatible-speech-provider-store' import type { DictationContext, OpenAiCompatibleSpeechProvider } from '../../shared/speech-types' import type { Store } from '../persistence/loading-store/store' +import type { DictationCleanupResult } from '../../shared/dictation-cleanup' +import { cancelDictationCleanup, cleanUpDictatedText } from '../speech/dictation-cleanup-service' +import type { CommitMessageAgentEnvironmentResolvers } from '../text-generation/commit-message-agent-environment' + +// Ten minutes of fast speech is under 20,000 characters; anything far past it isn't dictation. +const MAX_CLEANUP_TEXT_LENGTH = 100_000 + +export function registerSpeechHandlers( + store: Store, + agentEnvResolvers?: CommitMessageAgentEnvironmentResolvers +): void { + ipcMain.handle( + 'speech:cleanUpDictation', + ( + _event, + args: { text: string; context?: DictationContext; requestId: string } + ): Promise => { + if (typeof args?.text !== 'string' || args.text.length > MAX_CLEANUP_TEXT_LENGTH) { + return Promise.resolve({ + status: 'failed', + reason: 'Dictated text is too long to clean up.' + }) + } + return cleanUpDictatedText( + { + text: args.text, + terms: dictationTermsFor(store.getSettings().voice, { + dictationTerms: args.context?.repoTerms, + displayName: args.context?.repoName + }), + requestId: typeof args.requestId === 'string' ? args.requestId : undefined + }, + { getSettings: () => store.getSettings(), getAgentEnvResolvers: () => agentEnvResolvers } + ) + } + ) + + ipcMain.handle('speech:cancelDictationCleanup', (_event, requestId: unknown) => { + if (typeof requestId === 'string') { + cancelDictationCleanup(requestId) + } + }) -export function registerSpeechHandlers(store: Store): void { ipcMain.handle('speech:getCatalog', () => { return SPEECH_MODEL_CATALOG }) @@ -225,7 +267,7 @@ export function registerSpeechHandlers(store: Store): void { displayName: context?.repoName }), owner, - store.getSettings().voice + dictationTextSettingsFor(store.getSettings()) ) // Coalesced starts install no sink; only an accepted start supersedes the close hook. if (sessionReady) { diff --git a/src/main/opencode/text-tasks/opencode-text-task.ts b/src/main/opencode/text-tasks/opencode-text-task.ts index e56f2c6ae78..d56a8ebe0b4 100644 --- a/src/main/opencode/text-tasks/opencode-text-task.ts +++ b/src/main/opencode/text-tasks/opencode-text-task.ts @@ -1,6 +1,6 @@ import { mergeCommandEnvironment } from '../../../shared/command-environment' import { textGenerationFailure } from '../../../shared/text-generation-failure-copy' -import { cleanGeneratedCommitMessage } from '../../../shared/source-control/ai/generation/commit-message-agent-output' +import { cleanAgentAnswer } from '../../text-generation/source-control-agent-failure' import { wslDistroForCommand } from '../../../shared/wsl-paths' import type { CommitMessagePlan } from '../../../shared/source-control/ai/generation/commit-message-plan' import { OpenCodeTextTaskError, runOwnedOpenCodeTextTask } from './opencode-owned-text-task' @@ -82,7 +82,7 @@ export async function runOpenCodeTextTask(input: { stop: killSourceControlAgentProcess, shouldExecute: input.shouldExecute }) - const cleaned = cleanGeneratedCommitMessage(rawOutput) + const cleaned = cleanAgentAnswer(rawOutput, input.operation) if (!cleaned) { throw new OpenCodeTextTaskError('session') } diff --git a/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts b/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts index 4f0b0513031..7938efaadde 100644 --- a/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts +++ b/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts @@ -25,7 +25,8 @@ const request = z.object({ 'pull-request-fields', 'branch-name', 'jira-issue-summary', - 'conversation-name' + 'conversation-name', + 'dictation-cleanup' ]) .optional(), timeoutMs: z.number().int().positive() diff --git a/src/main/runtime/construction/runtime-service-command-surface.ts b/src/main/runtime/construction/runtime-service-command-surface.ts index 5c35c36b2da..8cd562414d0 100644 --- a/src/main/runtime/construction/runtime-service-command-surface.ts +++ b/src/main/runtime/construction/runtime-service-command-surface.ts @@ -201,7 +201,8 @@ export function installRuntimeServiceCommandSurface( clearOpenAiSpeechKey: speech.clearOpenAiKey.bind(speech), startMobileDictation: dictation.start.bind(dictation), feedMobileDictation: dictation.feed.bind(dictation), - finishMobileDictation: dictation.finish.bind(dictation), + finishMobileDictation: (params) => + dictation.finish(params, accounts.getCommitMessageAgentEnvironment()), cancelMobileDictation: dictation.cancel.bind(dictation), getAllBrowserDrivers: browsers.getAll.bind(browsers), reclaimBrowserForDesktop: browsers.reclaimForDesktop.bind(browsers), diff --git a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts index 2bbf14ed4ed..12e31c2d0b4 100644 --- a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts @@ -13,6 +13,10 @@ import { } from './mobile-dictation-chunk-order' import type { RuntimeStore } from '../construction/runtime-store-contract' import { hostDictationTerms } from './runtime-dictation-terms' +import { dictationTextSettingsFor } from '../../../shared/dictation-cleanup' +import type { DictationTerm } from '../../../shared/dictation-dictionary' +import { cleanUpDictatedText } from '../../speech/dictation-cleanup-service' +import type { CommitMessageAgentEnvironmentResolvers } from '../../text-generation/commit-message-agent-environment' import { DictationTranscript, type DictationTranscriptSnapshot @@ -22,6 +26,8 @@ type MobileDictationSession = { id: string owner: string clientId: string + /** The dictionary it was transcribed with, which cleanup spells by too. */ + terms: DictationTerm[] state: 'starting' | 'active' | 'closing' transcript: DictationTranscript errors: string[] @@ -74,10 +80,12 @@ export class RuntimeMobileDictationController { } const owner = `mobile:${clientId}:${params.dictationId}` + const terms = hostDictationTerms(store, params.worktreeId) const session: MobileDictationSession = { id: params.dictationId, owner, clientId, + terms, state: 'starting', transcript: new DictationTranscript(), errors: [], @@ -91,9 +99,9 @@ export class RuntimeMobileDictationController { await getSpeechSttService(store).startDictation( modelId, (event) => acceptEvent(session, event), - hostDictationTerms(store, params.worktreeId), + terms, owner, - voice + dictationTextSettingsFor({ ...store.getSettings(), voice }) ) if (this.sessions.get(clientId) !== session) { throw new Error('dictation_canceled') @@ -149,10 +157,11 @@ export class RuntimeMobileDictationController { return { ...reply, ...session.transcript.snapshot() } } - async finish(params: { - dictationId: string - clientId?: string - }): Promise<{ dictationId: string; text: string }> { + /** `agentEnv`: what AI cleanup runs the user's agent with, when it's on. */ + async finish( + params: { dictationId: string; clientId?: string }, + agentEnv?: CommitMessageAgentEnvironmentResolvers + ): Promise<{ dictationId: string; text: string }> { const session = this.requireOwnedSession(params) for (const chunk of session.chunkOrder.drain()) { this.feedChunk(session, chunk) @@ -161,7 +170,17 @@ export class RuntimeMobileDictationController { if (session.errors.length > 0) { throw new Error(session.errors[0]) } - return { dictationId: params.dictationId, text: session.transcript.snapshot().text } + const heard = session.transcript.snapshot().text + // The phone waits out cleanup inside its finish deadline; on any failure it gets the words as heard. + const store = this.requireStore() + const cleaned = await cleanUpDictatedText( + { text: heard, terms: session.terms }, + { getSettings: () => store.getSettings(), getAgentEnvResolvers: () => agentEnv } + ) + return { + dictationId: params.dictationId, + text: cleaned.status === 'cleaned' ? cleaned.text : heard + } } async cancel(params: { diff --git a/src/main/speech/dictation-cleanup-service.ts b/src/main/speech/dictation-cleanup-service.ts new file mode 100644 index 00000000000..6ae234774e9 --- /dev/null +++ b/src/main/speech/dictation-cleanup-service.ts @@ -0,0 +1,105 @@ +import { mkdir } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + DICTATION_CLEANUP_TIMEOUT_MS, + isDictationCleanupOn, + type DictationCleanupResult +} from '../../shared/dictation-cleanup' +import type { DictationTerm } from '../../shared/dictation-dictionary' +import type { VoiceSettings } from '../../shared/speech-types' +import { getCommitMessageModel } from '../../shared/source-control/ai/generation/commit-message-agent-spec' +import { + prepareLocalCommitMessageAgentEnv, + type CommitMessageAgentEnvironmentResolvers +} from '../text-generation/commit-message-agent-environment' +import { resolveTextGenerationParams } from '../text-generation/commit-message-text-generation' +import { generateDictationCleanup } from '../text-generation/dictation-cleanup-generation-request' +import { cancelLocalGeneration } from '../text-generation/source-control-generation-lanes' + +export type DictationCleanupDeps = { + getSettings: () => Parameters[0] & { voice?: VoiceSettings } + getAgentEnvResolvers: () => CommitMessageAgentEnvironmentResolvers | undefined +} + +const skipsByRequest = new Map void>() + +export function cancelDictationCleanup(requestId: string): void { + skipsByRequest.get(requestId)?.() +} + +// One folder for every run, so the agent reads no project and keeps one history bucket, not one per dictation. +const cleanupFolder = (): string => join(tmpdir(), 'orca-dictation-cleanup') + +/** Runs the user's agent on dictated text, on this computer, in an empty folder of its own. */ +export async function cleanUpDictatedText( + input: { text: string; terms: readonly DictationTerm[]; requestId?: string }, + deps: DictationCleanupDeps +): Promise { + const settings = deps.getSettings() + if (!input.text.trim() || !isDictationCleanupOn(settings)) { + return { status: 'off' } + } + const startedAt = Date.now() + const stop = Promise.withResolvers<'timed-out' | 'canceled'>() + const timer = setTimeout(() => stop.resolve('timed-out'), DICTATION_CLEANUP_TIMEOUT_MS) + if (input.requestId) { + skipsByRequest.set(input.requestId, () => stop.resolve('canceled')) + } + let stopped = false + const cwd = cleanupFolder() + try { + const resolved = resolveTextGenerationParams(settings, undefined, 'dictationCleanup') + if (!resolved.ok) { + return { status: 'failed', reason: resolved.error } + } + const { params } = resolved + const prepared = await Promise.race([ + prepareLocalCommitMessageAgentEnv(params.agentId, deps.getAgentEnvResolvers()), + stop.promise + ]) + if (typeof prepared === 'string') { + return { status: prepared } + } + if (!prepared.ok) { + return { status: 'failed', reason: prepared.error } + } + await mkdir(cwd, { recursive: true }) + const generation = generateDictationCleanup({ + text: input.text, + voice: settings.voice, + terms: input.terms, + params, + target: { kind: 'local', cwd, ...(prepared.env ? { env: prepared.env } : {}) } + }) + const outcome = await Promise.race([generation, stop.promise]) + if (typeof outcome === 'string') { + stopped = true + return { status: outcome } + } + if (!outcome.success) { + return outcome.canceled ? { status: 'canceled' } : { status: 'failed', reason: outcome.error } + } + return { + status: 'cleaned', + text: outcome.text, + agentLabel: outcome.agentLabel ?? params.agentId, + model: + (params.agentId !== 'custom' && + getCommitMessageModel(params.agentId, params.model)?.label) || + params.model, + elapsedMs: Date.now() - startedAt + } + } catch (error) { + return { status: 'failed', reason: error instanceof Error ? error.message : String(error) } + } finally { + clearTimeout(timer) + if (input.requestId) { + skipsByRequest.delete(input.requestId) + } + // Stops the agent still running after a timeout or a skip. + if (stopped) { + cancelLocalGeneration('dictation-cleanup', cwd) + } + } +} diff --git a/src/main/text-generation/agent-generation-request.ts b/src/main/text-generation/agent-generation-request.ts index cb4eadb106e..b7ad8d5e2a9 100644 --- a/src/main/text-generation/agent-generation-request.ts +++ b/src/main/text-generation/agent-generation-request.ts @@ -1,10 +1,7 @@ import { conversationNameTaskPrompt } from '../../shared/source-control/ai/generation/conversation-name-generation' import { textGenerationFailureCodeFields } from '../../shared/text-generation-failure-copy' import { runClaudeGenerationAttempts } from '../../shared/claude-background-run-isolation' -import { - cleanGeneratedCommitMessage, - stripPrefilledReasoningPreamble -} from '../../shared/source-control/ai/generation/commit-message-agent-output' +import { stripPrefilledReasoningPreamble } from '../../shared/source-control/ai/generation/commit-message-agent-output' import type { CommandTemplateBackslash } from '../../shared/source-control/ai/generation/commit-message-prompt' import { isCustomAgentId } from '../../shared/source-control/ai/generation/commit-message-agent-spec' import { @@ -16,7 +13,7 @@ import { openCodeVariantRetryPlan } from '../../shared/opencode-generation-comma import { copilotUnavailableModelRetryPlan } from './copilot-unavailable-model-retry' import { runLocalPlanForAgent } from './source-control-local-generation' import { runOpenCodeTextTask } from '../opencode/text-tasks/opencode-text-task' -import { emptyResultFailure } from './source-control-agent-failure' +import { cleanAgentAnswer, emptyResultFailure } from './source-control-agent-failure' import { runRemoteSourceControlPlan } from './source-control-remote-generation' import type { CommitMessageGenerationTarget, @@ -44,7 +41,8 @@ export async function runGenerationRequest(input: { }): Promise { const params = { ...input.params, - backslash: commandBackslashMode(input.target) + backslash: commandBackslashMode(input.target), + toolFree: input.operation === 'dictation-cleanup' } const prompt = input.operation === 'conversation-name' @@ -126,7 +124,7 @@ export async function runGenerationRequest(input: { if (answer === result.rawOutput) { return result } - const rawOutput = cleanGeneratedCommitMessage(answer) + const rawOutput = cleanAgentAnswer(answer, input.operation) return rawOutput ? { ...result, rawOutput } : { success: false, ...emptyResultFailure(input.operation, plan.label) } diff --git a/src/main/text-generation/dictation-cleanup-generation-request.ts b/src/main/text-generation/dictation-cleanup-generation-request.ts new file mode 100644 index 00000000000..7612a5f548e --- /dev/null +++ b/src/main/text-generation/dictation-cleanup-generation-request.ts @@ -0,0 +1,42 @@ +import { + buildDictationCleanupPrompt, + sanitizeDictationCleanupOutput +} from '../../shared/dictation-cleanup' +import type { DictationTerm } from '../../shared/dictation-dictionary' +import { renderSourceControlActionCommandTemplate } from '../../shared/source-control/ai/recipes/source-control-ai-actions' +import type { ResolvedSourceControlAiGenerationParams } from '../../shared/source-control/ai/source-control-ai-operation-resolution' +import type { VoiceSettings } from '../../shared/speech-types' +import { runGenerationRequest } from './agent-generation-request' +import { spawnSourceControlAgent } from './source-control-agent-launch' +import type { CommitMessageGenerationTarget } from './source-control-text-generation-types' + +export type GenerateDictationCleanupResult = + | { success: true; text: string; agentLabel?: string } + | { success: false; error: string; canceled?: boolean } + +export async function generateDictationCleanup(input: { + text: string + voice: VoiceSettings | undefined + terms: readonly DictationTerm[] + params: ResolvedSourceControlAiGenerationParams + target: CommitMessageGenerationTarget +}): Promise { + const basePrompt = buildDictationCleanupPrompt(input) + const result = await runGenerationRequest({ + params: input.params, + prompt: renderSourceControlActionCommandTemplate( + input.params.commandInputTemplate ?? '{basePrompt}', + { basePrompt } + ), + target: input.target, + operation: 'dictation-cleanup', + spawnAgent: spawnSourceControlAgent + }) + if (!result.success) { + return { success: false, error: result.error, canceled: result.canceled } + } + const text = sanitizeDictationCleanupOutput(result.rawOutput) + return text + ? { success: true, text, agentLabel: result.agentLabel } + : { success: false, error: `${result.agentLabel ?? input.params.agentId} returned no text.` } +} diff --git a/src/main/text-generation/source-control-agent-failure.ts b/src/main/text-generation/source-control-agent-failure.ts index 6a0aa9397dd..6fff7048b24 100644 --- a/src/main/text-generation/source-control-agent-failure.ts +++ b/src/main/text-generation/source-control-agent-failure.ts @@ -1,5 +1,5 @@ import { - cleanGeneratedCommitMessage, + cleanGeneratedText, excerptAgentFailureOutput, sanitizeAgentFailureDetail } from '../../shared/source-control/ai/generation/commit-message-agent-output' @@ -25,6 +25,18 @@ import { type TextGenerationFailureFields } from '../../shared/text-generation-failure-copy' +// Their stdout is a name or the user's own dictated words, not a diagnostic worth keeping in the error. +const OPERATIONS_WITHOUT_STDOUT_DETAIL: ReadonlySet = new Set([ + 'branch-name', + 'conversation-name', + 'dictation-cleanup' +]) + +/** The agent's answer without CLI framing; dictated text keeps the lines and markers Git text drops. */ +export function cleanAgentAnswer(raw: string, operation: TextGenerationOperation): string { + return cleanGeneratedText(raw, operation === 'dictation-cleanup' ? 'prose' : 'git') +} + type AgentCliFailureOptions = { includeLocalMacDnsHint?: boolean includeStdoutDetail?: boolean @@ -82,7 +94,8 @@ const EMPTY_RESULT_FAILURES = { const UNCODED_EMPTY_RESULT_NAMES = { 'branch-name': 'branch name', - 'conversation-name': 'chat name' + 'conversation-name': 'chat name', + 'dictation-cleanup': 'cleanup' } as const export function emptyResultFailure( @@ -90,7 +103,11 @@ export function emptyResultFailure( label: string, detail?: string | null ): TextGenerationFailureFields { - if (operation === 'branch-name' || operation === 'conversation-name') { + if ( + operation === 'branch-name' || + operation === 'conversation-name' || + operation === 'dictation-cleanup' + ) { // Background rename failures are stored as English rename errors, outside the coded result path. const name = UNCODED_EMPTY_RESULT_NAMES[operation] return { @@ -116,6 +133,11 @@ export function finalizeFromAgentOutput(args: { outputFormat?: CommitMessagePlan['outputFormat'] }): InternalTextGenerationResult { const { code, stdout, stderr, label } = args + const options = { + ...args, + includeStdoutDetail: + args.includeStdoutDetail ?? !OPERATIONS_WITHOUT_STDOUT_DETAIL.has(args.operation) + } const parsed = args.outputFormat === 'opencode-json' ? parseOpenCodeGenerationOutput(stdout) @@ -134,14 +156,14 @@ export function finalizeFromAgentOutput(args: { return { success: false, ...(args.outputFormat === 'kimi-stream-json' - ? formatKimiGenerationFailure(findKimiGenerationFailure(stdout), stderr, code, args) + ? formatKimiGenerationFailure(findKimiGenerationFailure(stdout), stderr, code, options) : agentCliFailure( label, stdout, // Orca's own parse failures (coded) say nothing about why the agent exited. parsed && !parsed.ok && parsed.errorCode === undefined ? parsed.error : stderr, code, - args + options )), failureOutput: captureFailureOutput(label, code, stdout, stderr) } @@ -158,7 +180,7 @@ export function finalizeFromAgentOutput(args: { failureOutput: captureFailureOutput(label, code, stdout, stderr) } } - const cleaned = cleanGeneratedCommitMessage(parsed?.ok ? parsed.text : stdout) + const cleaned = cleanAgentAnswer(parsed?.ok ? parsed.text : stdout, args.operation) if (cleaned) { return { success: true, rawOutput: cleaned, agentLabel: label } } diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index fe75e880b9d..db91e525792 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -238,8 +238,7 @@ export function runLocalSourceControlPlan( stderr, label: plan.label, outputFormat: plan.outputFormat, - operation, - includeStdoutDetail: operation !== 'branch-name' && operation !== 'conversation-name' + operation }) ) } diff --git a/src/main/text-generation/source-control-remote-generation.ts b/src/main/text-generation/source-control-remote-generation.ts index 6f734710f11..554bda8172d 100644 --- a/src/main/text-generation/source-control-remote-generation.ts +++ b/src/main/text-generation/source-control-remote-generation.ts @@ -70,7 +70,6 @@ export async function runRemoteSourceControlPlan(input: { label: plan.label, outputFormat: plan.outputFormat, operation, - includeLocalMacDnsHint: false, - includeStdoutDetail: operation !== 'branch-name' && operation !== 'conversation-name' + includeLocalMacDnsHint: false }) } diff --git a/src/main/text-generation/source-control-text-generation-types.ts b/src/main/text-generation/source-control-text-generation-types.ts index bf51fcd19b9..b42e40e0512 100644 --- a/src/main/text-generation/source-control-text-generation-types.ts +++ b/src/main/text-generation/source-control-text-generation-types.ts @@ -50,6 +50,7 @@ export type TextGenerationOperation = | 'branch-name' | 'jira-issue-summary' | 'conversation-name' + | 'dictation-cleanup' export type CommitMessageGenerationTarget = | { kind: 'local'; cwd: string; env?: NodeJS.ProcessEnv; wslDistro?: string } diff --git a/src/preload/api/speech-api.ts b/src/preload/api/speech-api.ts index c7415932ba7..3120e6ec82e 100644 --- a/src/preload/api/speech-api.ts +++ b/src/preload/api/speech-api.ts @@ -1,4 +1,5 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' +import type { DictationCleanupResult } from '../../shared/dictation-cleanup' import type { DictationContext, OpenAiCompatibleSpeechProvider, @@ -38,6 +39,13 @@ export type SpeechApi = { stopDictation: (sessionId?: string) => Promise /** Drops the session's audio and any text it hasn't returned yet, without transcribing the rest. */ cancelDictation: (sessionId: string) => Promise + /** Absent in the paired web client: its host cleans up inside `speech.dictation.finish`. */ + cleanUpDictation?: ( + text: string, + context: DictationContext | undefined, + requestId: string + ) => Promise + cancelDictationCleanup?: (requestId: string) => Promise onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void onSpeechPending: (callback: (data: SpeechPendingEvent) => void) => () => void diff --git a/src/preload/api/speech-bridge.ts b/src/preload/api/speech-bridge.ts index 66acdfc0bac..ee3eb951e4a 100644 --- a/src/preload/api/speech-bridge.ts +++ b/src/preload/api/speech-bridge.ts @@ -1,4 +1,5 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' +import type { DictationCleanupResult } from '../../shared/dictation-cleanup' import { ipcRenderer } from 'electron' import type { DictationContext, @@ -99,6 +100,14 @@ export const speechApi = { audioPorts.delete(sessionId) return ipcRenderer.invoke('speech:cancelDictation', sessionId) }, + cleanUpDictation: ( + text: string, + context: DictationContext | undefined, + requestId: string + ): Promise => + ipcRenderer.invoke('speech:cleanUpDictation', { text, context, requestId }), + cancelDictationCleanup: (requestId: string): Promise => + ipcRenderer.invoke('speech:cancelDictationCleanup', requestId), onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void): (() => void) => { const listener = (_event: Electron.IpcRendererEvent, data: SpeechTranscriptEvent): void => diff --git a/src/renderer/src/components/dictation/DictationCleanupNoticePill.tsx b/src/renderer/src/components/dictation/DictationCleanupNoticePill.tsx new file mode 100644 index 00000000000..f257d13bb29 --- /dev/null +++ b/src/renderer/src/components/dictation/DictationCleanupNoticePill.tsx @@ -0,0 +1,57 @@ +import { CircleAlert } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { useTranslate } from '@/i18n/use-translate' +import { DictationPillCloseButton } from './DictationPillFailure' +import { DictationPillShell } from './DictationPillShell' +import { openVoiceSettings } from './dictation-toasts' +import type { DictationCleanupNotice } from './use-dictation-cleanup' + +/** After the words landed as heard because cleanup failed or took too long; closes itself. */ +export function DictationCleanupNoticePill({ + notice, + onClose +}: { + notice: DictationCleanupNotice + onClose: () => void +}) { + const t = useTranslate() + const message = + notice.kind === 'timed-out' + ? t( + 'components.dictation.cleanupNotice.timedOut', + 'Cleanup took too long. Inserted as heard.' + ) + : t('components.dictation.cleanupNotice.failed', 'Cleanup failed. Inserted as heard.') + return ( + + + {message} + + + + + {message} + + {notice.kind === 'failed' ? ( + + ) : null} + + ) +} diff --git a/src/renderer/src/components/dictation/DictationController.tsx b/src/renderer/src/components/dictation/DictationController.tsx index b10035c6acd..0a683b72483 100644 --- a/src/renderer/src/components/dictation/DictationController.tsx +++ b/src/renderer/src/components/dictation/DictationController.tsx @@ -8,13 +8,13 @@ import { dictationPillAnchorFor, type DictationPillAnchor } from './dictation-pi import { canRetryDictation, createDictationKeptRecording, - transcribeKeptRecording, type DictationKeptRecording } from './dictation-kept-recording' +import { useDictationRetry } from './use-dictation-retry' import { useDictationControls } from './use-dictation-controls' import { useDictationQueuedSend } from './use-dictation-queued-send' import { createDictationTranscriptDelivery } from './dictation-transcript-delivery' -import { recordStoppedSession, waitForStoppedSession } from './dictation-stopped-sessions' +import { createDictationStoppedSessions } from './dictation-stopped-sessions' import { showDictationDisabledToast, showDictationNoModelToast, @@ -33,6 +33,7 @@ import { useDictationSilenceStop } from './use-dictation-silence-stop' import { stopAfterSilenceSeconds } from '../../../../shared/dictation-silence-stop' import { useDictationSpeechEvents } from './use-dictation-speech-events' import { useDictationSpeechModelSetup } from './use-dictation-speech-model-setup' +import { shouldHoldForCleanup, useDictationCleanup } from './use-dictation-cleanup' import { NO_DICTATION_LIVE_TEXT } from '@/store/slices/dictation/dictation' import { canStartDictation, @@ -63,8 +64,7 @@ export function DictationController() { const sessionRef = useRef(initialDictationSession) // Inserts the current session's finals where dictation started. const deliverRef = useRef<((text: string) => void) | null>(null) - const stoppedSessionIdsRef = useRef(new Set()) - const stoppedResolversRef = useRef(new Map void>()) + const [stoppedSessions] = useState(createDictationStoppedSessions) // Push-to-talk restarts per utterance; report a missing preferred microphone once per preference. const micFallbackNotifiedForRef = useRef(null) const [captureLive, setCaptureLive] = useState(false) @@ -80,6 +80,8 @@ export function DictationController() { useDictationSpeechModelSetup(settings?.voice, isIdle, setPillAnchor) const queuedSend = useDictationQueuedSend(sessionRef, keptRef) const { clear: clearQueuedSend, release: releaseQueuedSend } = queuedSend + const cleanup = useDictationCleanup(keptRef) + const { land: landCleanup, interrupt: interruptCleanup } = cleanup const dispatch = useCallback( (event: DictationSessionEvent): DictationSession => { @@ -110,10 +112,6 @@ export function DictationController() { [setDictationState, setLiveText, clearQueuedSend] ) - const drainStoppedSession = useCallback((sessionId: string) => { - void waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) - }, []) - const finishDictationSession = useCallback( async ({ run, sessionId }: DictationSession, beforeEnd?: () => void) => { if (sessionId === null) { @@ -130,7 +128,7 @@ export function DictationController() { // Swallow stop errors — the worker may already be torn down. } // Main stop completion can precede renderer final-text IPC; wait for this session’s stopped event before accepting the next run. - await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) + await stoppedSessions.wait(sessionId) if (sessionRef.current.run !== run) { return } @@ -138,17 +136,26 @@ export function DictationController() { showNoSpeechDetectedToast() } beforeEnd?.() + await landCleanup() await releaseQueuedSend(run) if (sessionRef.current.run === run) { dispatch({ type: 'ended' }) } }, - [dispatch, finishCapture, getCapturedChunkCount, releaseQueuedSend] + [ + dispatch, + finishCapture, + getCapturedChunkCount, + stoppedSessions, + releaseQueuedSend, + landCleanup + ] ) /** The error pill when the recording can be retried; otherwise the toast, as nothing is left to retry. */ const showFailure = useCallback( (kind: DictationFailureKind | null, showToast: () => void) => { + interruptCleanup() dispatch({ type: 'fail' }) if (kind && canRetryDictation(keptRef.current)) { setFailure(kind) @@ -157,11 +164,15 @@ export function DictationController() { showToast() dispatch({ type: 'ended' }) }, - [dispatch] + [dispatch, interruptCleanup] ) // Words that already landed stay; the engine drops the rest instead of transcribing it. const cancelDictation = useCallback(() => { + // ✕ while cleaning up skips cleanup; the words then land as heard. + if (interruptCleanup()) { + return + } invalidatePendingStart() if (sessionRef.current.phase === 'error') { dispatch({ type: 'ended' }) @@ -173,8 +184,8 @@ export function DictationController() { } stopCapture() void window.api.speech.cancelDictation(previous.sessionId).catch(() => undefined) - drainStoppedSession(previous.sessionId) - }, [dispatch, stopCapture, drainStoppedSession, invalidatePendingStart]) + void stoppedSessions.wait(previous.sessionId) + }, [dispatch, stopCapture, stoppedSessions, invalidatePendingStart, interruptCleanup]) const stopDictation = useCallback(async () => { invalidatePendingStart() @@ -217,7 +228,7 @@ export function DictationController() { } const isCurrent = (): boolean => sessionRef.current.run === run const insertionTarget = captureInsertionTarget() - const deliver = createDictationTranscriptDelivery() + const deliver = createDictationTranscriptDelivery(shouldHoldForCleanup()) deliverRef.current = (text) => deliver(text, insertionTarget) const context = captureDictationContext() const kept = createDictationKeptRecording(insertionTarget, deliver, context) @@ -235,7 +246,7 @@ export function DictationController() { } if (engineStarted) { await window.api.speech.cancelDictation(sessionId).catch(() => undefined) - drainStoppedSession(sessionId) + void stoppedSessions.wait(sessionId) } } @@ -323,38 +334,21 @@ export function DictationController() { stopCapture, stopDictation, finishDictationSession, - drainStoppedSession, + stoppedSessions, recordFeatureInteraction, showFailure, readyToRecord ]) - // Sends the failed session's recording through the speech host again, into the same place. - const retryDictation = useCallback(async () => { - const kept = keptRef.current - const modelId = settings?.voice?.sttModel - if (sessionRef.current.phase !== 'error' || !modelId || !kept || !canRetryDictation(kept)) { - return - } - const { run, sessionId } = dispatch({ type: 'retry' }) - if (sessionId === null) { - return - } - await transcribeKeptRecording({ - recording: kept, - modelId, - sessionId, - isCurrent: () => sessionRef.current.run === run, - routeFinals: (sink) => { - deliverRef.current = sink - }, - finish: (beforeEnd) => finishDictationSession(sessionRef.current, beforeEnd), - fail: (message) => - showFailure(dictationStartFailureKind(message) ?? 'transcribe', () => - showDictationStartErrorToast(message) - ) - }) - }, [settings, dispatch, finishDictationSession, showFailure]) + const retryDictation = useDictationRetry({ + modelId: settings?.voice?.sttModel, + sessionRef, + keptRef, + deliverRef, + dispatch, + finish: finishDictationSession, + showFailure + }) const toggleDictation = useCallback(() => { // A second press while the model check runs takes the first one back. @@ -377,10 +371,7 @@ export function DictationController() { retry: () => void retryDictation(), enterQueues: (event) => queuedSend.queueTerminalEnter(event.target), // The error pill leaves Esc to Settings and dialogs; only its ✕ discards the recording. - escCancels: () => - hasPendingStart() || - isDictationUnderway(sessionRef.current.phase) || - sessionRef.current.phase === 'stopping' + escCancels: () => hasPendingStart() || !canStartDictation(sessionRef.current.phase) }) const { isHolding } = useHoldDictationGesture({ @@ -408,15 +399,14 @@ export function DictationController() { dispatch({ type: 'final-text' }) deliverRef.current?.(text) }, - onStopped: (sessionId) => - recordStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef), + onStopped: (sessionId) => stoppedSessions.record(sessionId), onError: (sessionId, error) => { const { run } = dispatch({ type: 'abandon' }) stopCapture() discardBufferedAudio() void (async () => { await window.api.speech.stopDictation(sessionId).catch(() => undefined) - await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) + await stoppedSessions.wait(sessionId) if (sessionRef.current.run !== run) { return } @@ -432,6 +422,7 @@ export function DictationController() { anchor={pillAnchor} failure={failure} queued={queuedSend.queuedKind} + cleanup={cleanup} setup={setupTarget} onSetupClose={closeSetup} /> diff --git a/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx b/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx index 294c70cb1f8..cf982f2cdf1 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx @@ -6,17 +6,22 @@ import type { DictationFailureKind } from './dictation-toasts' import type { SpeechModelUpdateTarget } from './speech-model-update-target' import type { SpeechModelManifest, SpeechModelState } from '../../../../shared/speech-types' import type { DictationQueuedSendKind } from './use-dictation-queued-send' +import type { DictationCleanupNotice } from './use-dictation-cleanup' function PaneWithPill({ captureLive = true, failure = null, queued = null, - setup = null + setup = null, + cleaning = false, + notice = null }: { captureLive?: boolean failure?: DictationFailureKind | null queued?: DictationQueuedSendKind | null setup?: SpeechModelUpdateTarget | null + cleaning?: boolean + notice?: DictationCleanupNotice['kind'] | null }) { const [pane, setPane] = useState(null) const anchor = useMemo( @@ -31,6 +36,11 @@ function PaneWithPill({ anchor={anchor} failure={failure} queued={queued} + cleanup={{ + cleaning, + notice: notice ? { kind: notice, anchor } : null, + dismissNotice: () => {} + }} setup={setup} onSetupClose={() => {}} /> @@ -131,6 +141,32 @@ export const modelFailed = defineFixture({ render: () => }) +export const cleaningUp = defineFixture({ + width: 520, + store: { + dictationState: 'stopping', + liveText: { + committed: 'move the retry into the worker at four scratch that at five', + tentative: '', + speechPending: false + } + }, + setup: resetDictationMeter, + render: () => +}) + +export const cleanupFailed = defineFixture({ + width: 520, + store: { dictationState: 'idle' }, + render: () => +}) + +export const cleanupTimedOut = defineFixture({ + width: 520, + store: { dictationState: 'idle' }, + render: () => +}) + export const queuedChatSend = defineFixture({ width: 520, store: { dictationState: 'stopping' }, diff --git a/src/renderer/src/components/dictation/DictationIndicator.tsx b/src/renderer/src/components/dictation/DictationIndicator.tsx index 28121e94e28..6c4737fb227 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.tsx @@ -22,6 +22,8 @@ import { loadSpeechCatalog } from './speech-model-update-check' import type { SpeechModelUpdateTarget } from './speech-model-update-target' import type { DictationFailureKind } from './dictation-toasts' import type { DictationQueuedSendKind } from './use-dictation-queued-send' +import type { DictationCleanupNotice } from './use-dictation-cleanup' +import { DictationCleanupNoticePill } from './DictationCleanupNoticePill' // The finish button turns into a spinner only when finishing is slow enough to notice. const FINISH_SPINNER_DELAY_MS = 200 @@ -36,6 +38,12 @@ type DictationIndicatorProps = { failure: DictationFailureKind | null /** A send waiting for this dictation's words to land. */ queued: DictationQueuedSendKind | null + cleanup: { + /** AI cleanup is running on the finished dictation's words. */ + cleaning: boolean + notice: DictationCleanupNotice | null + dismissNotice: () => void + } } type DictationSetupProps = { @@ -53,9 +61,11 @@ export function DictationIndicator({ if (isDictationPillShown(dictationState)) { return } - return setup ? ( - - ) : null + if (setup) { + return + } + const { notice, dismissNotice } = props.cleanup + return notice ? : null } function formatElapsed(seconds: number): string { @@ -150,7 +160,7 @@ function DictationPendingDots() { ) } -function DictationPill({ captureLive, anchor, failure, queued }: DictationIndicatorProps) { +function DictationPill({ captureLive, anchor, failure, queued, cleanup }: DictationIndicatorProps) { const t = useTranslate() const dictationState = useAppStore((state) => state.dictationState) const liveText = useAppStore((state) => state.liveText) @@ -179,13 +189,18 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica 'Presses Enter when the text lands' ) : null - const status = finishing - ? t('auto.components.dictation.DictationIndicator.finishing', 'Finishing…') - : t('auto.components.dictation.DictationIndicator.listening', 'Listening…') + const { cleaning } = cleanup + const status = cleaning + ? t('components.dictation.DictationIndicator.cleaningUp', 'Cleaning up…') + : finishing + ? t('auto.components.dictation.DictationIndicator.finishing', 'Finishing…') + : t('auto.components.dictation.DictationIndicator.listening', 'Listening…') const announced = nearCloudLimit ? `${status} ${t('auto.components.dictation.DictationIndicator.oneMinuteLeft', 'One minute left.')}` : (queuedStatus ?? status) - const cancelLabel = t('auto.components.dictation.DictationIndicator.cancel', 'Cancel') + const cancelLabel = cleaning + ? t('components.dictation.DictationIndicator.skipCleanup', 'Skip cleanup and insert as heard') + : t('auto.components.dictation.DictationIndicator.cancel', 'Cancel') const finishLabel = t('auto.components.dictation.DictationIndicator.finish', 'Finish') return ( @@ -212,7 +227,7 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica )} {queuedStatus} - ) : committed || tentative || speechPending ? ( + ) : !cleaning && (committed || tentative || speechPending) ? ( // Right-aligned so the newest words stay in view; older ones fade off the left edge. void + /** Resolves once the session's stopped event arrives, or after a second. */ + wait: (sessionId: string) => Promise +} + +export function createDictationStoppedSessions(): DictationStoppedSessions { + const stoppedSessionIds = { current: new Set() } + const stoppedResolvers = { current: new Map void>() } + return { + record: (sessionId) => recordStoppedSession(sessionId, stoppedSessionIds, stoppedResolvers), + wait: (sessionId) => waitForStoppedSession(sessionId, stoppedSessionIds, stoppedResolvers) + } +} diff --git a/src/renderer/src/components/dictation/dictation-transcript-delivery.ts b/src/renderer/src/components/dictation/dictation-transcript-delivery.ts index 3cfeadbae95..bcbbdec6aa6 100644 --- a/src/renderer/src/components/dictation/dictation-transcript-delivery.ts +++ b/src/renderer/src/components/dictation/dictation-transcript-delivery.ts @@ -12,15 +12,24 @@ export type DictationTranscriptDelivery = (( landedText: () => string /** Waits for every final so far to land; false if any went to the clipboard instead. */ settled: () => Promise + /** What AI cleanup is holding back, or null when nothing is held. */ + held: () => string | null + /** Stops holding: `text` (the cleaned words), else the held words as heard, lands once where they were going. */ + releaseHeld: (text?: string) => void + /** Holds the finals that follow, as a retry of a failed session does while cleanup is on. */ + hold: () => void } -/** One session's finals, inserted in order; text that can't land goes to the clipboard so it isn't lost. */ -export function createDictationTranscriptDelivery(): DictationTranscriptDelivery { +/** One session's finals, inserted in order; text that can't land goes to the clipboard so it isn't lost. With `hold`, finals wait for AI cleanup and land together. */ +export function createDictationTranscriptDelivery(hold = false): DictationTranscriptDelivery { let lastLanded = '' let allLanded = '' let missed = '' let queue = Promise.resolve() - const deliver = (text: string, target: DictationInsertionTarget | null): void => { + let holding = hold + let held = '' + let heldTarget: DictationInsertionTarget | null = null + const insert = (text: string, target: DictationInsertionTarget | null): void => { queue = queue.then(async () => { const landed = target ? await insertText(text, target, lastLanded).catch(() => false) : false if (landed) { @@ -32,9 +41,31 @@ export function createDictationTranscriptDelivery(): DictationTranscriptDelivery await copyMissedDictation(missed, target !== null) }) } + const deliver = (text: string, target: DictationInsertionTarget | null): void => { + if (!holding) { + insert(text, target) + return + } + held = appendDictationText(held, text) + heldTarget = target + } return Object.assign(deliver, { landedText: () => allLanded, - settled: () => queue.then(() => missed === '') + settled: () => queue.then(() => missed === ''), + held: () => (holding && held ? held : null), + hold: (): void => { + holding = true + held = '' + }, + releaseHeld: (text = held): void => { + if (!holding) { + return + } + holding = false + if (text) { + insert(text, heldTarget) + } + } }) } diff --git a/src/renderer/src/components/dictation/use-dictation-cleanup.ts b/src/renderer/src/components/dictation/use-dictation-cleanup.ts new file mode 100644 index 00000000000..d61a54e4a8f --- /dev/null +++ b/src/renderer/src/components/dictation/use-dictation-cleanup.ts @@ -0,0 +1,87 @@ +import { useCallback, useEffect, useRef, useState, type RefObject } from 'react' +import { useAppStore } from '@/store' +import { + isDictationCleanupOn, + type DictationCleanupResult +} from '../../../../shared/dictation-cleanup' +import { createNonSecureContextUuid } from '../../../../shared/non-secure-context-uuid' +import { dictationPillAnchorFor, type DictationPillAnchor } from './dictation-pill-anchor' +import type { DictationKeptRecording } from './dictation-kept-recording' + +const NOTICE_MS = 4000 + +export type DictationCleanupNotice = { + kind: 'failed' | 'timed-out' + anchor: DictationPillAnchor | null +} + +/** Whether this desktop cleans dictated text itself; a paired web client's host does it in `speech.dictation.finish`. */ +export function shouldHoldForCleanup(): boolean { + const settings = useAppStore.getState().settings + return Boolean(window.api.speech.cleanUpDictation && settings && isDictationCleanupOn(settings)) +} + +/** Runs AI cleanup on the session's held words at finish; ✕ skips it, and the words then land as heard. */ +export function useDictationCleanup(keptRef: RefObject) { + const [cleaning, setCleaning] = useState(false) + const [notice, setNotice] = useState(null) + const skipRef = useRef<(() => void) | null>(null) + + useEffect(() => { + if (!notice) { + return + } + const id = window.setTimeout(() => setNotice(null), NOTICE_MS) + return () => window.clearTimeout(id) + }, [notice]) + + /** Lands the held words once: cleaned, or as heard when cleanup fails, takes too long or is skipped. */ + const land = useCallback(async (): Promise => { + const kept = keptRef.current + const heard = kept?.deliver.held() ?? null + const cleanUp = window.api.speech.cleanUpDictation + if (!kept || heard === null || !cleanUp) { + kept?.deliver.releaseHeld() + return + } + const requestId = createNonSecureContextUuid() + const skipped = Promise.withResolvers() + skipRef.current = () => { + skipped.resolve({ status: 'canceled' }) + void window.api.speech.cancelDictationCleanup?.(requestId).catch(() => undefined) + } + setNotice(null) + setCleaning(true) + const result = await Promise.race([ + cleanUp(heard, kept.context, requestId).catch((error: unknown): DictationCleanupResult => ({ + status: 'failed', + reason: String(error) + })), + skipped.promise + ]) + skipRef.current = null + setCleaning(false) + kept.deliver.releaseHeld(result.status === 'cleaned' ? result.text : undefined) + if (result.status === 'failed') { + console.warn('[dictation] AI cleanup failed:', result.reason) + } + if (result.status === 'failed' || result.status === 'timed-out') { + setNotice({ kind: result.status, anchor: dictationPillAnchorFor(kept.target) }) + } + }, [keptRef]) + + /** Skips a cleanup in progress (true), else lands any held words as heard: the session ends unfinished. */ + const interrupt = useCallback((): boolean => { + const skip = skipRef.current + if (skip) { + skip() + return true + } + keptRef.current?.deliver.releaseHeld() + return false + }, [keptRef]) + + const dismissNotice = useCallback(() => setNotice(null), []) + + return { cleaning, notice, land, interrupt, dismissNotice } +} diff --git a/src/renderer/src/components/dictation/use-dictation-retry.ts b/src/renderer/src/components/dictation/use-dictation-retry.ts new file mode 100644 index 00000000000..e2a73d9122d --- /dev/null +++ b/src/renderer/src/components/dictation/use-dictation-retry.ts @@ -0,0 +1,64 @@ +import { useCallback, type RefObject } from 'react' +import type { + DictationSession, + DictationSessionEvent +} from '../../../../shared/dictation-session-machine' +import { + canRetryDictation, + transcribeKeptRecording, + type DictationKeptRecording +} from './dictation-kept-recording' +import { + dictationStartFailureKind, + showDictationStartErrorToast, + type DictationFailureKind +} from './dictation-toasts' +import { shouldHoldForCleanup } from './use-dictation-cleanup' + +/** Sends the failed session's recording through the speech host again, into the same place. */ +export function useDictationRetry({ + modelId, + sessionRef, + keptRef, + deliverRef, + dispatch, + finish, + showFailure +}: { + modelId: string | undefined + sessionRef: RefObject + keptRef: RefObject + deliverRef: RefObject<((text: string) => void) | null> + dispatch: (event: DictationSessionEvent) => DictationSession + finish: (session: DictationSession, beforeEnd: () => void) => Promise + showFailure: (kind: DictationFailureKind | null, showToast: () => void) => void +}): () => Promise { + return useCallback(async () => { + const kept = keptRef.current + if (sessionRef.current.phase !== 'error' || !modelId || !kept || !canRetryDictation(kept)) { + return + } + const { run, sessionId } = dispatch({ type: 'retry' }) + if (sessionId === null) { + return + } + // The failure landed what was held as heard; what the retry adds waits for cleanup again. + if (shouldHoldForCleanup()) { + kept.deliver.hold() + } + await transcribeKeptRecording({ + recording: kept, + modelId, + sessionId, + isCurrent: () => sessionRef.current.run === run, + routeFinals: (sink) => { + deliverRef.current = sink + }, + finish: (beforeEnd) => finish(sessionRef.current, beforeEnd), + fail: (message) => + showFailure(dictationStartFailureKind(message) ?? 'transcribe', () => + showDictationStartErrorToast(message) + ) + }) + }, [modelId, sessionRef, keptRef, deliverRef, dispatch, finish, showFailure]) +} diff --git a/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx b/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx index f4776568666..69ca9429d21 100644 --- a/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx +++ b/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx @@ -183,7 +183,11 @@ export function renderDesktopCapabilitySettingsSections( searchEntries={navigation.getSectionSearchEntries('voice')} > {view.isSectionMounted('voice') ? ( - + ) : null} diff --git a/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts b/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts index 1d6d04b032c..95559c36dbc 100644 --- a/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts +++ b/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts @@ -27,6 +27,10 @@ export const getActionDescriptions = createLocalizedCatalog((): Record(['amp', 'muse']) + const FALLBACK_AGENT_ARGS_PLACEHOLDER = '--model sonnet' const AGENT_ARGS_PLACEHOLDER_OVERRIDES: Partial> = { @@ -108,7 +115,10 @@ export function getAgentCatalogForAction( return getAgentCatalog() } return getAgentCatalog().filter( - (agent) => TEXT_GENERATION_AGENT_ID_SET.has(agent.id) || agent.id === selectedAgent + (agent) => + (TEXT_GENERATION_AGENT_ID_SET.has(agent.id) && + !(actionId === 'dictationCleanup' && NAMING_ONLY_TEXT_AGENT_IDS.has(agent.id))) || + agent.id === selectedAgent ) } diff --git a/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx b/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx index 469f6f0d646..b6ae73cffc1 100644 --- a/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx +++ b/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx @@ -35,7 +35,10 @@ import { registeredRepoTextModelHostKeys } from './source-control-text-action-mo import { useTextActionModelDiscovery } from './use-text-action-model-discovery' import { textActionPickerEntryView } from './source-control-text-action-picker-entries' -import { AI_TEXT_ACTION_IDS } from '../../../../../../shared/source-control/ai/recipes/source-control-ai-actions' +import { + AI_TEXT_ACTION_IDS, + isGlobalAiTextAction +} from '../../../../../../shared/source-control/ai/recipes/source-control-ai-actions' import type { SourceControlAiSettingsPatch } from '../../../../../../shared/source-control/ai/settings/source-control-ai-settings-types' import { agentPickerDraft, @@ -124,7 +127,7 @@ const TextActionPicker = memo( } const inheritedSource = { ...source, modelOverridesByOperation: inheritedChoices } const repoChoices = { ...repoAi?.modelOverridesByOperation } - if (operation !== 'conversationName') { + if (!isGlobalAiTextAction(operation)) { delete repoChoices[operation] } const inheritedRepo = { ...repoAi, modelOverridesByOperation: repoChoices } @@ -165,18 +168,17 @@ const TextActionPicker = memo( [operation]: { ...source.actions?.[operation], agentId: agent.id } } } - const forcedRepo = - operation !== 'conversationName' - ? { - sourceControlAi: { - ...(ownRepo ?? repoAi), - actionOverrides: { - ...repoAi?.actionOverrides, - [operation]: { ...repoAi?.actionOverrides?.[operation], agentId: agent.id } - } + const forcedRepo = !isGlobalAiTextAction(operation) + ? { + sourceControlAi: { + ...(ownRepo ?? repoAi), + actionOverrides: { + ...repoAi?.actionOverrides, + [operation]: { ...repoAi?.actionOverrides?.[operation], agentId: agent.id } } } - : null + } + : null const inherited = resolveSourceControlAiActionModelDefault( { settings: { diff --git a/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts b/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts index 69808f28810..76e07a683de 100644 --- a/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts +++ b/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts @@ -1,3 +1,4 @@ +import { isGlobalAiTextAction } from '../../../../../../shared/source-control/ai/recipes/source-control-ai-actions' import { agentPickerEffectiveChoices, type AgentPickerEntry, @@ -61,7 +62,7 @@ export function saveSourceControlTextPickerDraft({ } } let currentRepo: Pick | null = - operation !== 'conversationName' && repoId + !isGlobalAiTextAction(operation) && repoId ? { sourceControlAi: { ...repoAi, @@ -103,7 +104,7 @@ export function saveSourceControlTextPickerDraft({ currentSource = saved.sourceControlAi nextChoice = currentSource.modelOverridesByOperation?.[operation] nextArgs = currentSource.actions?.[operation]?.agentArgs ?? nextArgs - } else if (operation !== 'conversationName') { + } else if (!isGlobalAiTextAction(operation)) { currentRepo = { sourceControlAi: saved.update.sourceControlAi ?? undefined } nextChoice = currentRepo.sourceControlAi?.modelOverridesByOperation?.[operation] nextArgs = currentRepo.sourceControlAi?.actionOverrides?.[operation]?.agentArgs ?? nextArgs diff --git a/src/renderer/src/components/settings/voice/VoiceCleanupConsentDialog.tsx b/src/renderer/src/components/settings/voice/VoiceCleanupConsentDialog.tsx new file mode 100644 index 00000000000..f940b550772 --- /dev/null +++ b/src/renderer/src/components/settings/voice/VoiceCleanupConsentDialog.tsx @@ -0,0 +1,109 @@ +import { useTranslate, type Translate } from '@/i18n/use-translate' +import { Button } from '../../ui/button' +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle +} from '../../ui/dialog' + +// Only agents whose service is fixed; the others go wherever the user set them up to send. +const AGENT_SERVICES: Record = { + claude: { company: 'Anthropic', plan: 'Claude' }, + codex: { company: 'OpenAI', plan: 'ChatGPT' } +} + +function destination(agentId: string, agentLabel: string, t: Translate): string { + const service = AGENT_SERVICES[agentId] + return service + ? t( + 'components.settings.voice.cleanup.consent.goesToKnown', + '{{company}}, through {{agent}} on this computer. Counts toward your {{plan}} plan.', + { ...service, agent: agentLabel } + ) + : t( + 'components.settings.voice.cleanup.consent.goesToAgent', + 'Wherever {{agent}} on this computer is set up to send prompts.', + { agent: agentLabel } + ) +} + +/** Asked before dictated text first goes to an agent, and again for each new agent. */ +export function VoiceCleanupConsentDialog({ + request, + onCancel, + onConfirm +}: { + /** The agent text would go to; null keeps the dialog closed. */ + request: { agentId: string; agentLabel: string } | null + onCancel: () => void + onConfirm: () => void +}): React.JSX.Element { + const t = useTranslate() + const agentLabel = request?.agentLabel ?? '' + const rows = request + ? [ + { + label: t('components.settings.voice.cleanup.consent.sentLabel', 'Sent'), + value: t( + 'components.settings.voice.cleanup.consent.sent', + 'The dictated text, your vocabulary and your instructions.' + ) + }, + { + label: t('components.settings.voice.cleanup.consent.notSentLabel', 'Not sent'), + value: t( + 'components.settings.voice.cleanup.consent.notSent', + 'Audio, files, or what’s on screen.' + ) + }, + { + label: t('components.settings.voice.cleanup.consent.goesToLabel', 'Goes to'), + value: destination(request.agentId, agentLabel, t) + }, + { + label: t('components.settings.voice.cleanup.consent.waitLabel', 'Wait'), + value: t( + 'components.settings.voice.cleanup.consent.wait', + 'Text lands a few seconds later.' + ) + } + ] + : [] + return ( + !open && onCancel()}> + + + + {t('components.settings.voice.cleanup.consent.title', 'Turn on AI cleanup?')} + + + {t( + 'components.settings.voice.cleanup.consent.description', + 'After each dictation, Orca sends the text to {{agent}} to tidy it before it lands.', + { agent: agentLabel } + )} + + +
+ {rows.map((row) => ( +
+
{row.label}
+
{row.value}
+
+ ))} +
+ + + + +
+
+ ) +} diff --git a/src/renderer/src/components/settings/voice/VoiceCleanupInstructionsSetting.tsx b/src/renderer/src/components/settings/voice/VoiceCleanupInstructionsSetting.tsx new file mode 100644 index 00000000000..339cb26b183 --- /dev/null +++ b/src/renderer/src/components/settings/voice/VoiceCleanupInstructionsSetting.tsx @@ -0,0 +1,86 @@ +import { useId, useState } from 'react' +import { BUILT_IN_DICTATION_CLEANUP_INSTRUCTIONS } from '../../../../../shared/dictation-cleanup' +import { useTranslate } from '@/i18n/use-translate' +import { Button } from '../../ui/button' +import { Label } from '../../ui/label' +import { Textarea } from '../../ui/textarea' +import { SettingsRow } from '../form-controls/SettingsFormControls' + +/** Collapsed to a summary; Edit opens the built-in or saved text to change in place. */ +export function VoiceCleanupInstructionsSetting({ + instructions, + onSave +}: { + /** Absent: the built-in instructions. */ + instructions: string | undefined + onSave: (instructions: string | undefined) => void +}): React.JSX.Element { + const t = useTranslate() + const fieldId = useId() + const [draft, setDraft] = useState(null) + const custom = instructions?.trim() ? instructions : undefined + const label = t('components.settings.voice.cleanup.instructions.label', 'Instructions') + if (draft === null) { + return ( + setDraft(custom ?? BUILT_IN_DICTATION_CLEANUP_INSTRUCTIONS)} + > + {t('components.settings.voice.cleanup.instructions.edit', 'Edit')} + + } + /> + ) + } + const save = (): void => { + const text = draft.trim() + onSave(text && text !== BUILT_IN_DICTATION_CLEANUP_INSTRUCTIONS ? text : undefined) + setDraft(null) + } + return ( +
+
+ +

+ {t( + 'components.settings.voice.cleanup.instructions.emptyHint', + 'Leave empty for the built-in ones.' + )} +

+
+