diff --git a/mobile/src/i18n/locales/en.json b/mobile/src/i18n/locales/en.json index 46dc3c90822..7fbf79c5313 100644 --- a/mobile/src/i18n/locales/en.json +++ b/mobile/src/i18n/locales/en.json @@ -3933,7 +3933,10 @@ "planAgentUnsupported": "Agent \"{{agent}}\" does not support AI commit messages.", "planModelUnavailable": "Model \"{{model}}\" is not available for {{label}}.", "planModelWithoutThinking": "Model \"{{model}}\" does not support a thinking effort level.", - "planThinkingLevelInvalid": "Thinking level \"{{level}}\" is not valid for {{model}}." + "planThinkingLevelInvalid": "Thinking level \"{{level}}\" is not valid for {{model}}.", + "commandTemplateEmptyDictationCleanup": "Command template is empty for dictation cleanup.", + "dictationCleanupAgentUnsupported": "Choose a supported agent for AI cleanup in Settings -> Voice -> Cleanup. Supported agents: {{agents}}, or Custom command.", + "agentCannotGenerateDictationCleanup": "Agent \"{{agent}}\" does not support dictation cleanup. Supported agents: {{agents}}, or Custom command." }, "conflictActions": { "resolveFile": "{{action}}: {{path}}", diff --git a/mobile/src/source-control/dictation-cleanup-failure-copy.ts b/mobile/src/source-control/dictation-cleanup-failure-copy.ts new file mode 100644 index 00000000000..b343d74f474 --- /dev/null +++ b/mobile/src/source-control/dictation-cleanup-failure-copy.ts @@ -0,0 +1,25 @@ +import type { TextGenerationFailureCode } from '../../../src/shared/text-generation-failure-copy' +import { translate } from '../i18n/mobile-i18n' + +export const DICTATION_CLEANUP_FAILURE_COPY = { + commandTemplateEmptyDictationCleanup: (values) => + translate( + 'sourceControl.textGenerationFailure.commandTemplateEmptyDictationCleanup', + 'Command template is empty for dictation cleanup.', + values + ), + dictationCleanupAgentUnsupported: (values) => + translate( + 'sourceControl.textGenerationFailure.dictationCleanupAgentUnsupported', + 'Choose a supported agent for AI cleanup in Settings -> Voice -> Cleanup. Supported agents: {{agents}}, or Custom command.', + values + ), + agentCannotGenerateDictationCleanup: (values) => + translate( + 'sourceControl.textGenerationFailure.agentCannotGenerateDictationCleanup', + 'Agent "{{agent}}" does not support dictation cleanup. Supported agents: {{agents}}, or Custom command.', + values + ) +} satisfies Partial< + Record) => string> +> diff --git a/mobile/src/source-control/text-generation-failure-copy.ts b/mobile/src/source-control/text-generation-failure-copy.ts index be32494f6e6..c50f7eada9e 100644 --- a/mobile/src/source-control/text-generation-failure-copy.ts +++ b/mobile/src/source-control/text-generation-failure-copy.ts @@ -5,6 +5,7 @@ import { import { translate } from '../i18n/mobile-i18n' import { AGENT_OUTPUT_FAILURE_COPY } from './agent-output-failure-copy' import { TEXT_GENERATION_PLAN_FAILURE_COPY } from './text-generation-plan-failure-copy' +import { DICTATION_CLEANUP_FAILURE_COPY } from './dictation-cleanup-failure-copy' type CopyValues = Record @@ -256,7 +257,8 @@ const COPY: Record string> = values ), ...TEXT_GENERATION_PLAN_FAILURE_COPY, - ...AGENT_OUTPUT_FAILURE_COPY + ...AGENT_OUTPUT_FAILURE_COPY, + ...DICTATION_CLEANUP_FAILURE_COPY } function isTextGenerationFailureCode(code: unknown): code is TextGenerationFailureCode { diff --git a/src/main/ai-vault/parsing/conversation-name-request-sessions.ts b/src/main/ai-vault/parsing/conversation-name-request-sessions.ts deleted file mode 100644 index da85830e89f..00000000000 --- a/src/main/ai-vault/parsing/conversation-name-request-sessions.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { AiVaultSession } from '../../../shared/ai-vault/contracts/ai-vault-types' -import { CONVERSATION_NAME_PROMPT_OPENING } from '../../../shared/source-control/ai/generation/conversation-name-generation' - -const MAX_NAME_REQUEST_MESSAGES = 6 - -/** CLI naming runs share chat folders; keep sessions with user follow-ups visible (#29973). */ -export function isConversationNameRequestSession(session: AiVaultSession): boolean { - if (session.previewMessagesTruncated || session.messageCount > MAX_NAME_REQUEST_MESSAGES) { - return false - } - const [request, ...followUps] = session.previewMessages.filter( - (message) => message.role === 'user' - ) - return ( - followUps.length === 0 && - request?.text.trimStart().startsWith(CONVERSATION_NAME_PROMPT_OPENING) === true - ) -} diff --git a/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts b/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts index d377d13b964..e0adfc2acc3 100644 --- a/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts +++ b/src/main/ai-vault/parsing/session-scanner-candidate-parse.ts @@ -1,5 +1,5 @@ import { yieldToEventLoop } from '../../../shared/event-loop-yield' -import { isConversationNameRequestSession } from './conversation-name-request-sessions' +import { isTextTaskRequestSession } from './text-task-request-sessions' import { candidateFileTime } from '../scanners/antigravity/antigravity-transcript-candidates' import { aiVaultAgentLabel, @@ -107,7 +107,7 @@ export async function parseSessionCandidates(args: { } if ( result.session && - !isConversationNameRequestSession(result.session) && + !isTextTaskRequestSession(result.session) && (!args.admitSession || args.admitSession(result.session)) ) { if (args.listed === false) { diff --git a/src/main/ai-vault/parsing/text-task-request-sessions.ts b/src/main/ai-vault/parsing/text-task-request-sessions.ts new file mode 100644 index 00000000000..601dee915b1 --- /dev/null +++ b/src/main/ai-vault/parsing/text-task-request-sessions.ts @@ -0,0 +1,22 @@ +import type { AiVaultSession } from '../../../shared/ai-vault/contracts/ai-vault-types' +import { DICTATION_CLEANUP_PROMPT_OPENING } from '../../../shared/dictation-cleanup' +import { CONVERSATION_NAME_PROMPT_OPENING } from '../../../shared/source-control/ai/generation/conversation-name-generation' + +const MAX_TEXT_TASK_REQUEST_MESSAGES = 6 + +/** One-shot CLI runs (chat naming, dictation cleanup) aren't conversations; sessions with user follow-ups stay visible (#29973). */ +export function isTextTaskRequestSession(session: AiVaultSession): boolean { + if (session.previewMessagesTruncated || session.messageCount > MAX_TEXT_TASK_REQUEST_MESSAGES) { + return false + } + const [request, ...followUps] = session.previewMessages.filter( + (message) => message.role === 'user' + ) + const text = request?.text.trimStart() + return ( + followUps.length === 0 && + text !== undefined && + (text.startsWith(CONVERSATION_NAME_PROMPT_OPENING) || + text.startsWith(DICTATION_CLEANUP_PROMPT_OPENING)) + ) +} diff --git a/src/main/ipc/register-core-handlers/register-core-handlers.ts b/src/main/ipc/register-core-handlers/register-core-handlers.ts index f100d92b59f..728c8c79cb9 100644 --- a/src/main/ipc/register-core-handlers/register-core-handlers.ts +++ b/src/main/ipc/register-core-handlers/register-core-handlers.ts @@ -296,5 +296,5 @@ export function registerCoreHandlers( } }) registerUpdaterHandlers(store) - registerSpeechHandlers(store) + registerSpeechHandlers(store, commitMessageAgentEnv) } diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 98f7f52f9b0..102a758c7a4 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -7,6 +7,7 @@ import { } from 'electron' import { abortWhenRendererGone } from './renderer-lifetime-abort' import { dictationTermsFor } from '../../shared/dictation-dictionary' +import { dictationTextSettingsFor } from '../../shared/dictation-cleanup' import { SPEECH_MODEL_CATALOG } from '../speech/model-catalog' import { deleteLocalSpeechModel } from '../speech/speech-model-deletion' import { getSpeechModelManager, getSpeechSttService } from '../speech/speech-runtime-service' @@ -25,8 +26,49 @@ import { } from '../speech/openai-compatible-speech-provider-store' import type { DictationContext, OpenAiCompatibleSpeechProvider } from '../../shared/speech-types' import type { Store } from '../persistence/loading-store/store' +import type { DictationCleanupResult } from '../../shared/dictation-cleanup' +import { cancelDictationCleanup, cleanUpDictatedText } from '../speech/dictation-cleanup-service' +import type { CommitMessageAgentEnvironmentResolvers } from '../text-generation/commit-message-agent-environment' + +// Ten minutes of fast speech is under 20,000 characters; anything far past it isn't dictation. +const MAX_CLEANUP_TEXT_LENGTH = 100_000 + +export function registerSpeechHandlers( + store: Store, + agentEnvResolvers?: CommitMessageAgentEnvironmentResolvers +): void { + ipcMain.handle( + 'speech:cleanUpDictation', + ( + _event, + args: { text: string; context?: DictationContext; requestId: string } + ): Promise => { + if (typeof args?.text !== 'string' || args.text.length > MAX_CLEANUP_TEXT_LENGTH) { + return Promise.resolve({ + status: 'failed', + reason: 'Dictated text is too long to clean up.' + }) + } + return cleanUpDictatedText( + { + text: args.text, + terms: dictationTermsFor(store.getSettings().voice, { + dictationTerms: args.context?.repoTerms, + displayName: args.context?.repoName + }), + requestId: typeof args.requestId === 'string' ? args.requestId : undefined + }, + { getSettings: () => store.getSettings(), getAgentEnvResolvers: () => agentEnvResolvers } + ) + } + ) + + ipcMain.handle('speech:cancelDictationCleanup', (_event, requestId: unknown) => { + if (typeof requestId === 'string') { + cancelDictationCleanup(requestId) + } + }) -export function registerSpeechHandlers(store: Store): void { ipcMain.handle('speech:getCatalog', () => { return SPEECH_MODEL_CATALOG }) @@ -225,7 +267,7 @@ export function registerSpeechHandlers(store: Store): void { displayName: context?.repoName }), owner, - store.getSettings().voice + dictationTextSettingsFor(store.getSettings()) ) // Coalesced starts install no sink; only an accepted start supersedes the close hook. if (sessionReady) { diff --git a/src/main/opencode/text-tasks/opencode-text-task.ts b/src/main/opencode/text-tasks/opencode-text-task.ts index e56f2c6ae78..d56a8ebe0b4 100644 --- a/src/main/opencode/text-tasks/opencode-text-task.ts +++ b/src/main/opencode/text-tasks/opencode-text-task.ts @@ -1,6 +1,6 @@ import { mergeCommandEnvironment } from '../../../shared/command-environment' import { textGenerationFailure } from '../../../shared/text-generation-failure-copy' -import { cleanGeneratedCommitMessage } from '../../../shared/source-control/ai/generation/commit-message-agent-output' +import { cleanAgentAnswer } from '../../text-generation/source-control-agent-failure' import { wslDistroForCommand } from '../../../shared/wsl-paths' import type { CommitMessagePlan } from '../../../shared/source-control/ai/generation/commit-message-plan' import { OpenCodeTextTaskError, runOwnedOpenCodeTextTask } from './opencode-owned-text-task' @@ -82,7 +82,7 @@ export async function runOpenCodeTextTask(input: { stop: killSourceControlAgentProcess, shouldExecute: input.shouldExecute }) - const cleaned = cleanGeneratedCommitMessage(rawOutput) + const cleaned = cleanAgentAnswer(rawOutput, input.operation) if (!cleaned) { throw new OpenCodeTextTaskError('session') } diff --git a/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts b/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts index 4f0b0513031..7938efaadde 100644 --- a/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts +++ b/src/main/opencode/text-tasks/wsl/opencode-wsl-text-task-entry.ts @@ -25,7 +25,8 @@ const request = z.object({ 'pull-request-fields', 'branch-name', 'jira-issue-summary', - 'conversation-name' + 'conversation-name', + 'dictation-cleanup' ]) .optional(), timeoutMs: z.number().int().positive() diff --git a/src/main/runtime/construction/runtime-service-command-surface.ts b/src/main/runtime/construction/runtime-service-command-surface.ts index 5c35c36b2da..8cd562414d0 100644 --- a/src/main/runtime/construction/runtime-service-command-surface.ts +++ b/src/main/runtime/construction/runtime-service-command-surface.ts @@ -201,7 +201,8 @@ export function installRuntimeServiceCommandSurface( clearOpenAiSpeechKey: speech.clearOpenAiKey.bind(speech), startMobileDictation: dictation.start.bind(dictation), feedMobileDictation: dictation.feed.bind(dictation), - finishMobileDictation: dictation.finish.bind(dictation), + finishMobileDictation: (params) => + dictation.finish(params, accounts.getCommitMessageAgentEnvironment()), cancelMobileDictation: dictation.cancel.bind(dictation), getAllBrowserDrivers: browsers.getAll.bind(browsers), reclaimBrowserForDesktop: browsers.reclaimForDesktop.bind(browsers), diff --git a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts index 2bbf14ed4ed..12e31c2d0b4 100644 --- a/src/main/runtime/speech/runtime-mobile-dictation-controller.ts +++ b/src/main/runtime/speech/runtime-mobile-dictation-controller.ts @@ -13,6 +13,10 @@ import { } from './mobile-dictation-chunk-order' import type { RuntimeStore } from '../construction/runtime-store-contract' import { hostDictationTerms } from './runtime-dictation-terms' +import { dictationTextSettingsFor } from '../../../shared/dictation-cleanup' +import type { DictationTerm } from '../../../shared/dictation-dictionary' +import { cleanUpDictatedText } from '../../speech/dictation-cleanup-service' +import type { CommitMessageAgentEnvironmentResolvers } from '../../text-generation/commit-message-agent-environment' import { DictationTranscript, type DictationTranscriptSnapshot @@ -22,6 +26,8 @@ type MobileDictationSession = { id: string owner: string clientId: string + /** The dictionary it was transcribed with, which cleanup spells by too. */ + terms: DictationTerm[] state: 'starting' | 'active' | 'closing' transcript: DictationTranscript errors: string[] @@ -74,10 +80,12 @@ export class RuntimeMobileDictationController { } const owner = `mobile:${clientId}:${params.dictationId}` + const terms = hostDictationTerms(store, params.worktreeId) const session: MobileDictationSession = { id: params.dictationId, owner, clientId, + terms, state: 'starting', transcript: new DictationTranscript(), errors: [], @@ -91,9 +99,9 @@ export class RuntimeMobileDictationController { await getSpeechSttService(store).startDictation( modelId, (event) => acceptEvent(session, event), - hostDictationTerms(store, params.worktreeId), + terms, owner, - voice + dictationTextSettingsFor({ ...store.getSettings(), voice }) ) if (this.sessions.get(clientId) !== session) { throw new Error('dictation_canceled') @@ -149,10 +157,11 @@ export class RuntimeMobileDictationController { return { ...reply, ...session.transcript.snapshot() } } - async finish(params: { - dictationId: string - clientId?: string - }): Promise<{ dictationId: string; text: string }> { + /** `agentEnv`: what AI cleanup runs the user's agent with, when it's on. */ + async finish( + params: { dictationId: string; clientId?: string }, + agentEnv?: CommitMessageAgentEnvironmentResolvers + ): Promise<{ dictationId: string; text: string }> { const session = this.requireOwnedSession(params) for (const chunk of session.chunkOrder.drain()) { this.feedChunk(session, chunk) @@ -161,7 +170,17 @@ export class RuntimeMobileDictationController { if (session.errors.length > 0) { throw new Error(session.errors[0]) } - return { dictationId: params.dictationId, text: session.transcript.snapshot().text } + const heard = session.transcript.snapshot().text + // The phone waits out cleanup inside its finish deadline; on any failure it gets the words as heard. + const store = this.requireStore() + const cleaned = await cleanUpDictatedText( + { text: heard, terms: session.terms }, + { getSettings: () => store.getSettings(), getAgentEnvResolvers: () => agentEnv } + ) + return { + dictationId: params.dictationId, + text: cleaned.status === 'cleaned' ? cleaned.text : heard + } } async cancel(params: { diff --git a/src/main/speech/dictation-cleanup-service.ts b/src/main/speech/dictation-cleanup-service.ts new file mode 100644 index 00000000000..6ae234774e9 --- /dev/null +++ b/src/main/speech/dictation-cleanup-service.ts @@ -0,0 +1,105 @@ +import { mkdir } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + DICTATION_CLEANUP_TIMEOUT_MS, + isDictationCleanupOn, + type DictationCleanupResult +} from '../../shared/dictation-cleanup' +import type { DictationTerm } from '../../shared/dictation-dictionary' +import type { VoiceSettings } from '../../shared/speech-types' +import { getCommitMessageModel } from '../../shared/source-control/ai/generation/commit-message-agent-spec' +import { + prepareLocalCommitMessageAgentEnv, + type CommitMessageAgentEnvironmentResolvers +} from '../text-generation/commit-message-agent-environment' +import { resolveTextGenerationParams } from '../text-generation/commit-message-text-generation' +import { generateDictationCleanup } from '../text-generation/dictation-cleanup-generation-request' +import { cancelLocalGeneration } from '../text-generation/source-control-generation-lanes' + +export type DictationCleanupDeps = { + getSettings: () => Parameters[0] & { voice?: VoiceSettings } + getAgentEnvResolvers: () => CommitMessageAgentEnvironmentResolvers | undefined +} + +const skipsByRequest = new Map void>() + +export function cancelDictationCleanup(requestId: string): void { + skipsByRequest.get(requestId)?.() +} + +// One folder for every run, so the agent reads no project and keeps one history bucket, not one per dictation. +const cleanupFolder = (): string => join(tmpdir(), 'orca-dictation-cleanup') + +/** Runs the user's agent on dictated text, on this computer, in an empty folder of its own. */ +export async function cleanUpDictatedText( + input: { text: string; terms: readonly DictationTerm[]; requestId?: string }, + deps: DictationCleanupDeps +): Promise { + const settings = deps.getSettings() + if (!input.text.trim() || !isDictationCleanupOn(settings)) { + return { status: 'off' } + } + const startedAt = Date.now() + const stop = Promise.withResolvers<'timed-out' | 'canceled'>() + const timer = setTimeout(() => stop.resolve('timed-out'), DICTATION_CLEANUP_TIMEOUT_MS) + if (input.requestId) { + skipsByRequest.set(input.requestId, () => stop.resolve('canceled')) + } + let stopped = false + const cwd = cleanupFolder() + try { + const resolved = resolveTextGenerationParams(settings, undefined, 'dictationCleanup') + if (!resolved.ok) { + return { status: 'failed', reason: resolved.error } + } + const { params } = resolved + const prepared = await Promise.race([ + prepareLocalCommitMessageAgentEnv(params.agentId, deps.getAgentEnvResolvers()), + stop.promise + ]) + if (typeof prepared === 'string') { + return { status: prepared } + } + if (!prepared.ok) { + return { status: 'failed', reason: prepared.error } + } + await mkdir(cwd, { recursive: true }) + const generation = generateDictationCleanup({ + text: input.text, + voice: settings.voice, + terms: input.terms, + params, + target: { kind: 'local', cwd, ...(prepared.env ? { env: prepared.env } : {}) } + }) + const outcome = await Promise.race([generation, stop.promise]) + if (typeof outcome === 'string') { + stopped = true + return { status: outcome } + } + if (!outcome.success) { + return outcome.canceled ? { status: 'canceled' } : { status: 'failed', reason: outcome.error } + } + return { + status: 'cleaned', + text: outcome.text, + agentLabel: outcome.agentLabel ?? params.agentId, + model: + (params.agentId !== 'custom' && + getCommitMessageModel(params.agentId, params.model)?.label) || + params.model, + elapsedMs: Date.now() - startedAt + } + } catch (error) { + return { status: 'failed', reason: error instanceof Error ? error.message : String(error) } + } finally { + clearTimeout(timer) + if (input.requestId) { + skipsByRequest.delete(input.requestId) + } + // Stops the agent still running after a timeout or a skip. + if (stopped) { + cancelLocalGeneration('dictation-cleanup', cwd) + } + } +} diff --git a/src/main/text-generation/agent-generation-request.ts b/src/main/text-generation/agent-generation-request.ts index cb4eadb106e..b7ad8d5e2a9 100644 --- a/src/main/text-generation/agent-generation-request.ts +++ b/src/main/text-generation/agent-generation-request.ts @@ -1,10 +1,7 @@ import { conversationNameTaskPrompt } from '../../shared/source-control/ai/generation/conversation-name-generation' import { textGenerationFailureCodeFields } from '../../shared/text-generation-failure-copy' import { runClaudeGenerationAttempts } from '../../shared/claude-background-run-isolation' -import { - cleanGeneratedCommitMessage, - stripPrefilledReasoningPreamble -} from '../../shared/source-control/ai/generation/commit-message-agent-output' +import { stripPrefilledReasoningPreamble } from '../../shared/source-control/ai/generation/commit-message-agent-output' import type { CommandTemplateBackslash } from '../../shared/source-control/ai/generation/commit-message-prompt' import { isCustomAgentId } from '../../shared/source-control/ai/generation/commit-message-agent-spec' import { @@ -16,7 +13,7 @@ import { openCodeVariantRetryPlan } from '../../shared/opencode-generation-comma import { copilotUnavailableModelRetryPlan } from './copilot-unavailable-model-retry' import { runLocalPlanForAgent } from './source-control-local-generation' import { runOpenCodeTextTask } from '../opencode/text-tasks/opencode-text-task' -import { emptyResultFailure } from './source-control-agent-failure' +import { cleanAgentAnswer, emptyResultFailure } from './source-control-agent-failure' import { runRemoteSourceControlPlan } from './source-control-remote-generation' import type { CommitMessageGenerationTarget, @@ -44,7 +41,8 @@ export async function runGenerationRequest(input: { }): Promise { const params = { ...input.params, - backslash: commandBackslashMode(input.target) + backslash: commandBackslashMode(input.target), + toolFree: input.operation === 'dictation-cleanup' } const prompt = input.operation === 'conversation-name' @@ -126,7 +124,7 @@ export async function runGenerationRequest(input: { if (answer === result.rawOutput) { return result } - const rawOutput = cleanGeneratedCommitMessage(answer) + const rawOutput = cleanAgentAnswer(answer, input.operation) return rawOutput ? { ...result, rawOutput } : { success: false, ...emptyResultFailure(input.operation, plan.label) } diff --git a/src/main/text-generation/dictation-cleanup-generation-request.ts b/src/main/text-generation/dictation-cleanup-generation-request.ts new file mode 100644 index 00000000000..7612a5f548e --- /dev/null +++ b/src/main/text-generation/dictation-cleanup-generation-request.ts @@ -0,0 +1,42 @@ +import { + buildDictationCleanupPrompt, + sanitizeDictationCleanupOutput +} from '../../shared/dictation-cleanup' +import type { DictationTerm } from '../../shared/dictation-dictionary' +import { renderSourceControlActionCommandTemplate } from '../../shared/source-control/ai/recipes/source-control-ai-actions' +import type { ResolvedSourceControlAiGenerationParams } from '../../shared/source-control/ai/source-control-ai-operation-resolution' +import type { VoiceSettings } from '../../shared/speech-types' +import { runGenerationRequest } from './agent-generation-request' +import { spawnSourceControlAgent } from './source-control-agent-launch' +import type { CommitMessageGenerationTarget } from './source-control-text-generation-types' + +export type GenerateDictationCleanupResult = + | { success: true; text: string; agentLabel?: string } + | { success: false; error: string; canceled?: boolean } + +export async function generateDictationCleanup(input: { + text: string + voice: VoiceSettings | undefined + terms: readonly DictationTerm[] + params: ResolvedSourceControlAiGenerationParams + target: CommitMessageGenerationTarget +}): Promise { + const basePrompt = buildDictationCleanupPrompt(input) + const result = await runGenerationRequest({ + params: input.params, + prompt: renderSourceControlActionCommandTemplate( + input.params.commandInputTemplate ?? '{basePrompt}', + { basePrompt } + ), + target: input.target, + operation: 'dictation-cleanup', + spawnAgent: spawnSourceControlAgent + }) + if (!result.success) { + return { success: false, error: result.error, canceled: result.canceled } + } + const text = sanitizeDictationCleanupOutput(result.rawOutput) + return text + ? { success: true, text, agentLabel: result.agentLabel } + : { success: false, error: `${result.agentLabel ?? input.params.agentId} returned no text.` } +} diff --git a/src/main/text-generation/source-control-agent-failure.ts b/src/main/text-generation/source-control-agent-failure.ts index 6a0aa9397dd..6fff7048b24 100644 --- a/src/main/text-generation/source-control-agent-failure.ts +++ b/src/main/text-generation/source-control-agent-failure.ts @@ -1,5 +1,5 @@ import { - cleanGeneratedCommitMessage, + cleanGeneratedText, excerptAgentFailureOutput, sanitizeAgentFailureDetail } from '../../shared/source-control/ai/generation/commit-message-agent-output' @@ -25,6 +25,18 @@ import { type TextGenerationFailureFields } from '../../shared/text-generation-failure-copy' +// Their stdout is a name or the user's own dictated words, not a diagnostic worth keeping in the error. +const OPERATIONS_WITHOUT_STDOUT_DETAIL: ReadonlySet = new Set([ + 'branch-name', + 'conversation-name', + 'dictation-cleanup' +]) + +/** The agent's answer without CLI framing; dictated text keeps the lines and markers Git text drops. */ +export function cleanAgentAnswer(raw: string, operation: TextGenerationOperation): string { + return cleanGeneratedText(raw, operation === 'dictation-cleanup' ? 'prose' : 'git') +} + type AgentCliFailureOptions = { includeLocalMacDnsHint?: boolean includeStdoutDetail?: boolean @@ -82,7 +94,8 @@ const EMPTY_RESULT_FAILURES = { const UNCODED_EMPTY_RESULT_NAMES = { 'branch-name': 'branch name', - 'conversation-name': 'chat name' + 'conversation-name': 'chat name', + 'dictation-cleanup': 'cleanup' } as const export function emptyResultFailure( @@ -90,7 +103,11 @@ export function emptyResultFailure( label: string, detail?: string | null ): TextGenerationFailureFields { - if (operation === 'branch-name' || operation === 'conversation-name') { + if ( + operation === 'branch-name' || + operation === 'conversation-name' || + operation === 'dictation-cleanup' + ) { // Background rename failures are stored as English rename errors, outside the coded result path. const name = UNCODED_EMPTY_RESULT_NAMES[operation] return { @@ -116,6 +133,11 @@ export function finalizeFromAgentOutput(args: { outputFormat?: CommitMessagePlan['outputFormat'] }): InternalTextGenerationResult { const { code, stdout, stderr, label } = args + const options = { + ...args, + includeStdoutDetail: + args.includeStdoutDetail ?? !OPERATIONS_WITHOUT_STDOUT_DETAIL.has(args.operation) + } const parsed = args.outputFormat === 'opencode-json' ? parseOpenCodeGenerationOutput(stdout) @@ -134,14 +156,14 @@ export function finalizeFromAgentOutput(args: { return { success: false, ...(args.outputFormat === 'kimi-stream-json' - ? formatKimiGenerationFailure(findKimiGenerationFailure(stdout), stderr, code, args) + ? formatKimiGenerationFailure(findKimiGenerationFailure(stdout), stderr, code, options) : agentCliFailure( label, stdout, // Orca's own parse failures (coded) say nothing about why the agent exited. parsed && !parsed.ok && parsed.errorCode === undefined ? parsed.error : stderr, code, - args + options )), failureOutput: captureFailureOutput(label, code, stdout, stderr) } @@ -158,7 +180,7 @@ export function finalizeFromAgentOutput(args: { failureOutput: captureFailureOutput(label, code, stdout, stderr) } } - const cleaned = cleanGeneratedCommitMessage(parsed?.ok ? parsed.text : stdout) + const cleaned = cleanAgentAnswer(parsed?.ok ? parsed.text : stdout, args.operation) if (cleaned) { return { success: true, rawOutput: cleaned, agentLabel: label } } diff --git a/src/main/text-generation/source-control-local-process.ts b/src/main/text-generation/source-control-local-process.ts index fe75e880b9d..db91e525792 100644 --- a/src/main/text-generation/source-control-local-process.ts +++ b/src/main/text-generation/source-control-local-process.ts @@ -238,8 +238,7 @@ export function runLocalSourceControlPlan( stderr, label: plan.label, outputFormat: plan.outputFormat, - operation, - includeStdoutDetail: operation !== 'branch-name' && operation !== 'conversation-name' + operation }) ) } diff --git a/src/main/text-generation/source-control-remote-generation.ts b/src/main/text-generation/source-control-remote-generation.ts index 6f734710f11..554bda8172d 100644 --- a/src/main/text-generation/source-control-remote-generation.ts +++ b/src/main/text-generation/source-control-remote-generation.ts @@ -70,7 +70,6 @@ export async function runRemoteSourceControlPlan(input: { label: plan.label, outputFormat: plan.outputFormat, operation, - includeLocalMacDnsHint: false, - includeStdoutDetail: operation !== 'branch-name' && operation !== 'conversation-name' + includeLocalMacDnsHint: false }) } diff --git a/src/main/text-generation/source-control-text-generation-types.ts b/src/main/text-generation/source-control-text-generation-types.ts index bf51fcd19b9..b42e40e0512 100644 --- a/src/main/text-generation/source-control-text-generation-types.ts +++ b/src/main/text-generation/source-control-text-generation-types.ts @@ -50,6 +50,7 @@ export type TextGenerationOperation = | 'branch-name' | 'jira-issue-summary' | 'conversation-name' + | 'dictation-cleanup' export type CommitMessageGenerationTarget = | { kind: 'local'; cwd: string; env?: NodeJS.ProcessEnv; wslDistro?: string } diff --git a/src/preload/api/speech-api.ts b/src/preload/api/speech-api.ts index c7415932ba7..3120e6ec82e 100644 --- a/src/preload/api/speech-api.ts +++ b/src/preload/api/speech-api.ts @@ -1,4 +1,5 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' +import type { DictationCleanupResult } from '../../shared/dictation-cleanup' import type { DictationContext, OpenAiCompatibleSpeechProvider, @@ -38,6 +39,13 @@ export type SpeechApi = { stopDictation: (sessionId?: string) => Promise /** Drops the session's audio and any text it hasn't returned yet, without transcribing the rest. */ cancelDictation: (sessionId: string) => Promise + /** Absent in the paired web client: its host cleans up inside `speech.dictation.finish`. */ + cleanUpDictation?: ( + text: string, + context: DictationContext | undefined, + requestId: string + ) => Promise + cancelDictationCleanup?: (requestId: string) => Promise onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void onSpeechPending: (callback: (data: SpeechPendingEvent) => void) => () => void diff --git a/src/preload/api/speech-bridge.ts b/src/preload/api/speech-bridge.ts index 66acdfc0bac..ee3eb951e4a 100644 --- a/src/preload/api/speech-bridge.ts +++ b/src/preload/api/speech-bridge.ts @@ -1,4 +1,5 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' +import type { DictationCleanupResult } from '../../shared/dictation-cleanup' import { ipcRenderer } from 'electron' import type { DictationContext, @@ -99,6 +100,14 @@ export const speechApi = { audioPorts.delete(sessionId) return ipcRenderer.invoke('speech:cancelDictation', sessionId) }, + cleanUpDictation: ( + text: string, + context: DictationContext | undefined, + requestId: string + ): Promise => + ipcRenderer.invoke('speech:cleanUpDictation', { text, context, requestId }), + cancelDictationCleanup: (requestId: string): Promise => + ipcRenderer.invoke('speech:cancelDictationCleanup', requestId), onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void): (() => void) => { const listener = (_event: Electron.IpcRendererEvent, data: SpeechTranscriptEvent): void => diff --git a/src/renderer/src/components/dictation/DictationCleanupNoticePill.tsx b/src/renderer/src/components/dictation/DictationCleanupNoticePill.tsx new file mode 100644 index 00000000000..f257d13bb29 --- /dev/null +++ b/src/renderer/src/components/dictation/DictationCleanupNoticePill.tsx @@ -0,0 +1,57 @@ +import { CircleAlert } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { useTranslate } from '@/i18n/use-translate' +import { DictationPillCloseButton } from './DictationPillFailure' +import { DictationPillShell } from './DictationPillShell' +import { openVoiceSettings } from './dictation-toasts' +import type { DictationCleanupNotice } from './use-dictation-cleanup' + +/** After the words landed as heard because cleanup failed or took too long; closes itself. */ +export function DictationCleanupNoticePill({ + notice, + onClose +}: { + notice: DictationCleanupNotice + onClose: () => void +}) { + const t = useTranslate() + const message = + notice.kind === 'timed-out' + ? t( + 'components.dictation.cleanupNotice.timedOut', + 'Cleanup took too long. Inserted as heard.' + ) + : t('components.dictation.cleanupNotice.failed', 'Cleanup failed. Inserted as heard.') + return ( + + + {message} + + + + + {message} + + {notice.kind === 'failed' ? ( + + ) : null} + + ) +} diff --git a/src/renderer/src/components/dictation/DictationController.tsx b/src/renderer/src/components/dictation/DictationController.tsx index b10035c6acd..0a683b72483 100644 --- a/src/renderer/src/components/dictation/DictationController.tsx +++ b/src/renderer/src/components/dictation/DictationController.tsx @@ -8,13 +8,13 @@ import { dictationPillAnchorFor, type DictationPillAnchor } from './dictation-pi import { canRetryDictation, createDictationKeptRecording, - transcribeKeptRecording, type DictationKeptRecording } from './dictation-kept-recording' +import { useDictationRetry } from './use-dictation-retry' import { useDictationControls } from './use-dictation-controls' import { useDictationQueuedSend } from './use-dictation-queued-send' import { createDictationTranscriptDelivery } from './dictation-transcript-delivery' -import { recordStoppedSession, waitForStoppedSession } from './dictation-stopped-sessions' +import { createDictationStoppedSessions } from './dictation-stopped-sessions' import { showDictationDisabledToast, showDictationNoModelToast, @@ -33,6 +33,7 @@ import { useDictationSilenceStop } from './use-dictation-silence-stop' import { stopAfterSilenceSeconds } from '../../../../shared/dictation-silence-stop' import { useDictationSpeechEvents } from './use-dictation-speech-events' import { useDictationSpeechModelSetup } from './use-dictation-speech-model-setup' +import { shouldHoldForCleanup, useDictationCleanup } from './use-dictation-cleanup' import { NO_DICTATION_LIVE_TEXT } from '@/store/slices/dictation/dictation' import { canStartDictation, @@ -63,8 +64,7 @@ export function DictationController() { const sessionRef = useRef(initialDictationSession) // Inserts the current session's finals where dictation started. const deliverRef = useRef<((text: string) => void) | null>(null) - const stoppedSessionIdsRef = useRef(new Set()) - const stoppedResolversRef = useRef(new Map void>()) + const [stoppedSessions] = useState(createDictationStoppedSessions) // Push-to-talk restarts per utterance; report a missing preferred microphone once per preference. const micFallbackNotifiedForRef = useRef(null) const [captureLive, setCaptureLive] = useState(false) @@ -80,6 +80,8 @@ export function DictationController() { useDictationSpeechModelSetup(settings?.voice, isIdle, setPillAnchor) const queuedSend = useDictationQueuedSend(sessionRef, keptRef) const { clear: clearQueuedSend, release: releaseQueuedSend } = queuedSend + const cleanup = useDictationCleanup(keptRef) + const { land: landCleanup, interrupt: interruptCleanup } = cleanup const dispatch = useCallback( (event: DictationSessionEvent): DictationSession => { @@ -110,10 +112,6 @@ export function DictationController() { [setDictationState, setLiveText, clearQueuedSend] ) - const drainStoppedSession = useCallback((sessionId: string) => { - void waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) - }, []) - const finishDictationSession = useCallback( async ({ run, sessionId }: DictationSession, beforeEnd?: () => void) => { if (sessionId === null) { @@ -130,7 +128,7 @@ export function DictationController() { // Swallow stop errors — the worker may already be torn down. } // Main stop completion can precede renderer final-text IPC; wait for this session’s stopped event before accepting the next run. - await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) + await stoppedSessions.wait(sessionId) if (sessionRef.current.run !== run) { return } @@ -138,17 +136,26 @@ export function DictationController() { showNoSpeechDetectedToast() } beforeEnd?.() + await landCleanup() await releaseQueuedSend(run) if (sessionRef.current.run === run) { dispatch({ type: 'ended' }) } }, - [dispatch, finishCapture, getCapturedChunkCount, releaseQueuedSend] + [ + dispatch, + finishCapture, + getCapturedChunkCount, + stoppedSessions, + releaseQueuedSend, + landCleanup + ] ) /** The error pill when the recording can be retried; otherwise the toast, as nothing is left to retry. */ const showFailure = useCallback( (kind: DictationFailureKind | null, showToast: () => void) => { + interruptCleanup() dispatch({ type: 'fail' }) if (kind && canRetryDictation(keptRef.current)) { setFailure(kind) @@ -157,11 +164,15 @@ export function DictationController() { showToast() dispatch({ type: 'ended' }) }, - [dispatch] + [dispatch, interruptCleanup] ) // Words that already landed stay; the engine drops the rest instead of transcribing it. const cancelDictation = useCallback(() => { + // ✕ while cleaning up skips cleanup; the words then land as heard. + if (interruptCleanup()) { + return + } invalidatePendingStart() if (sessionRef.current.phase === 'error') { dispatch({ type: 'ended' }) @@ -173,8 +184,8 @@ export function DictationController() { } stopCapture() void window.api.speech.cancelDictation(previous.sessionId).catch(() => undefined) - drainStoppedSession(previous.sessionId) - }, [dispatch, stopCapture, drainStoppedSession, invalidatePendingStart]) + void stoppedSessions.wait(previous.sessionId) + }, [dispatch, stopCapture, stoppedSessions, invalidatePendingStart, interruptCleanup]) const stopDictation = useCallback(async () => { invalidatePendingStart() @@ -217,7 +228,7 @@ export function DictationController() { } const isCurrent = (): boolean => sessionRef.current.run === run const insertionTarget = captureInsertionTarget() - const deliver = createDictationTranscriptDelivery() + const deliver = createDictationTranscriptDelivery(shouldHoldForCleanup()) deliverRef.current = (text) => deliver(text, insertionTarget) const context = captureDictationContext() const kept = createDictationKeptRecording(insertionTarget, deliver, context) @@ -235,7 +246,7 @@ export function DictationController() { } if (engineStarted) { await window.api.speech.cancelDictation(sessionId).catch(() => undefined) - drainStoppedSession(sessionId) + void stoppedSessions.wait(sessionId) } } @@ -323,38 +334,21 @@ export function DictationController() { stopCapture, stopDictation, finishDictationSession, - drainStoppedSession, + stoppedSessions, recordFeatureInteraction, showFailure, readyToRecord ]) - // Sends the failed session's recording through the speech host again, into the same place. - const retryDictation = useCallback(async () => { - const kept = keptRef.current - const modelId = settings?.voice?.sttModel - if (sessionRef.current.phase !== 'error' || !modelId || !kept || !canRetryDictation(kept)) { - return - } - const { run, sessionId } = dispatch({ type: 'retry' }) - if (sessionId === null) { - return - } - await transcribeKeptRecording({ - recording: kept, - modelId, - sessionId, - isCurrent: () => sessionRef.current.run === run, - routeFinals: (sink) => { - deliverRef.current = sink - }, - finish: (beforeEnd) => finishDictationSession(sessionRef.current, beforeEnd), - fail: (message) => - showFailure(dictationStartFailureKind(message) ?? 'transcribe', () => - showDictationStartErrorToast(message) - ) - }) - }, [settings, dispatch, finishDictationSession, showFailure]) + const retryDictation = useDictationRetry({ + modelId: settings?.voice?.sttModel, + sessionRef, + keptRef, + deliverRef, + dispatch, + finish: finishDictationSession, + showFailure + }) const toggleDictation = useCallback(() => { // A second press while the model check runs takes the first one back. @@ -377,10 +371,7 @@ export function DictationController() { retry: () => void retryDictation(), enterQueues: (event) => queuedSend.queueTerminalEnter(event.target), // The error pill leaves Esc to Settings and dialogs; only its ✕ discards the recording. - escCancels: () => - hasPendingStart() || - isDictationUnderway(sessionRef.current.phase) || - sessionRef.current.phase === 'stopping' + escCancels: () => hasPendingStart() || !canStartDictation(sessionRef.current.phase) }) const { isHolding } = useHoldDictationGesture({ @@ -408,15 +399,14 @@ export function DictationController() { dispatch({ type: 'final-text' }) deliverRef.current?.(text) }, - onStopped: (sessionId) => - recordStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef), + onStopped: (sessionId) => stoppedSessions.record(sessionId), onError: (sessionId, error) => { const { run } = dispatch({ type: 'abandon' }) stopCapture() discardBufferedAudio() void (async () => { await window.api.speech.stopDictation(sessionId).catch(() => undefined) - await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) + await stoppedSessions.wait(sessionId) if (sessionRef.current.run !== run) { return } @@ -432,6 +422,7 @@ export function DictationController() { anchor={pillAnchor} failure={failure} queued={queuedSend.queuedKind} + cleanup={cleanup} setup={setupTarget} onSetupClose={closeSetup} /> diff --git a/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx b/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx index 294c70cb1f8..cf982f2cdf1 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.fixture.tsx @@ -6,17 +6,22 @@ import type { DictationFailureKind } from './dictation-toasts' import type { SpeechModelUpdateTarget } from './speech-model-update-target' import type { SpeechModelManifest, SpeechModelState } from '../../../../shared/speech-types' import type { DictationQueuedSendKind } from './use-dictation-queued-send' +import type { DictationCleanupNotice } from './use-dictation-cleanup' function PaneWithPill({ captureLive = true, failure = null, queued = null, - setup = null + setup = null, + cleaning = false, + notice = null }: { captureLive?: boolean failure?: DictationFailureKind | null queued?: DictationQueuedSendKind | null setup?: SpeechModelUpdateTarget | null + cleaning?: boolean + notice?: DictationCleanupNotice['kind'] | null }) { const [pane, setPane] = useState(null) const anchor = useMemo( @@ -31,6 +36,11 @@ function PaneWithPill({ anchor={anchor} failure={failure} queued={queued} + cleanup={{ + cleaning, + notice: notice ? { kind: notice, anchor } : null, + dismissNotice: () => {} + }} setup={setup} onSetupClose={() => {}} /> @@ -131,6 +141,32 @@ export const modelFailed = defineFixture({ render: () => }) +export const cleaningUp = defineFixture({ + width: 520, + store: { + dictationState: 'stopping', + liveText: { + committed: 'move the retry into the worker at four scratch that at five', + tentative: '', + speechPending: false + } + }, + setup: resetDictationMeter, + render: () => +}) + +export const cleanupFailed = defineFixture({ + width: 520, + store: { dictationState: 'idle' }, + render: () => +}) + +export const cleanupTimedOut = defineFixture({ + width: 520, + store: { dictationState: 'idle' }, + render: () => +}) + export const queuedChatSend = defineFixture({ width: 520, store: { dictationState: 'stopping' }, diff --git a/src/renderer/src/components/dictation/DictationIndicator.tsx b/src/renderer/src/components/dictation/DictationIndicator.tsx index 28121e94e28..6c4737fb227 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.tsx @@ -22,6 +22,8 @@ import { loadSpeechCatalog } from './speech-model-update-check' import type { SpeechModelUpdateTarget } from './speech-model-update-target' import type { DictationFailureKind } from './dictation-toasts' import type { DictationQueuedSendKind } from './use-dictation-queued-send' +import type { DictationCleanupNotice } from './use-dictation-cleanup' +import { DictationCleanupNoticePill } from './DictationCleanupNoticePill' // The finish button turns into a spinner only when finishing is slow enough to notice. const FINISH_SPINNER_DELAY_MS = 200 @@ -36,6 +38,12 @@ type DictationIndicatorProps = { failure: DictationFailureKind | null /** A send waiting for this dictation's words to land. */ queued: DictationQueuedSendKind | null + cleanup: { + /** AI cleanup is running on the finished dictation's words. */ + cleaning: boolean + notice: DictationCleanupNotice | null + dismissNotice: () => void + } } type DictationSetupProps = { @@ -53,9 +61,11 @@ export function DictationIndicator({ if (isDictationPillShown(dictationState)) { return } - return setup ? ( - - ) : null + if (setup) { + return + } + const { notice, dismissNotice } = props.cleanup + return notice ? : null } function formatElapsed(seconds: number): string { @@ -150,7 +160,7 @@ function DictationPendingDots() { ) } -function DictationPill({ captureLive, anchor, failure, queued }: DictationIndicatorProps) { +function DictationPill({ captureLive, anchor, failure, queued, cleanup }: DictationIndicatorProps) { const t = useTranslate() const dictationState = useAppStore((state) => state.dictationState) const liveText = useAppStore((state) => state.liveText) @@ -179,13 +189,18 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica 'Presses Enter when the text lands' ) : null - const status = finishing - ? t('auto.components.dictation.DictationIndicator.finishing', 'Finishing…') - : t('auto.components.dictation.DictationIndicator.listening', 'Listening…') + const { cleaning } = cleanup + const status = cleaning + ? t('components.dictation.DictationIndicator.cleaningUp', 'Cleaning up…') + : finishing + ? t('auto.components.dictation.DictationIndicator.finishing', 'Finishing…') + : t('auto.components.dictation.DictationIndicator.listening', 'Listening…') const announced = nearCloudLimit ? `${status} ${t('auto.components.dictation.DictationIndicator.oneMinuteLeft', 'One minute left.')}` : (queuedStatus ?? status) - const cancelLabel = t('auto.components.dictation.DictationIndicator.cancel', 'Cancel') + const cancelLabel = cleaning + ? t('components.dictation.DictationIndicator.skipCleanup', 'Skip cleanup and insert as heard') + : t('auto.components.dictation.DictationIndicator.cancel', 'Cancel') const finishLabel = t('auto.components.dictation.DictationIndicator.finish', 'Finish') return ( @@ -212,7 +227,7 @@ function DictationPill({ captureLive, anchor, failure, queued }: DictationIndica )} {queuedStatus} - ) : committed || tentative || speechPending ? ( + ) : !cleaning && (committed || tentative || speechPending) ? ( // Right-aligned so the newest words stay in view; older ones fade off the left edge. void + /** Resolves once the session's stopped event arrives, or after a second. */ + wait: (sessionId: string) => Promise +} + +export function createDictationStoppedSessions(): DictationStoppedSessions { + const stoppedSessionIds = { current: new Set() } + const stoppedResolvers = { current: new Map void>() } + return { + record: (sessionId) => recordStoppedSession(sessionId, stoppedSessionIds, stoppedResolvers), + wait: (sessionId) => waitForStoppedSession(sessionId, stoppedSessionIds, stoppedResolvers) + } +} diff --git a/src/renderer/src/components/dictation/dictation-transcript-delivery.ts b/src/renderer/src/components/dictation/dictation-transcript-delivery.ts index 3cfeadbae95..bcbbdec6aa6 100644 --- a/src/renderer/src/components/dictation/dictation-transcript-delivery.ts +++ b/src/renderer/src/components/dictation/dictation-transcript-delivery.ts @@ -12,15 +12,24 @@ export type DictationTranscriptDelivery = (( landedText: () => string /** Waits for every final so far to land; false if any went to the clipboard instead. */ settled: () => Promise + /** What AI cleanup is holding back, or null when nothing is held. */ + held: () => string | null + /** Stops holding: `text` (the cleaned words), else the held words as heard, lands once where they were going. */ + releaseHeld: (text?: string) => void + /** Holds the finals that follow, as a retry of a failed session does while cleanup is on. */ + hold: () => void } -/** One session's finals, inserted in order; text that can't land goes to the clipboard so it isn't lost. */ -export function createDictationTranscriptDelivery(): DictationTranscriptDelivery { +/** One session's finals, inserted in order; text that can't land goes to the clipboard so it isn't lost. With `hold`, finals wait for AI cleanup and land together. */ +export function createDictationTranscriptDelivery(hold = false): DictationTranscriptDelivery { let lastLanded = '' let allLanded = '' let missed = '' let queue = Promise.resolve() - const deliver = (text: string, target: DictationInsertionTarget | null): void => { + let holding = hold + let held = '' + let heldTarget: DictationInsertionTarget | null = null + const insert = (text: string, target: DictationInsertionTarget | null): void => { queue = queue.then(async () => { const landed = target ? await insertText(text, target, lastLanded).catch(() => false) : false if (landed) { @@ -32,9 +41,31 @@ export function createDictationTranscriptDelivery(): DictationTranscriptDelivery await copyMissedDictation(missed, target !== null) }) } + const deliver = (text: string, target: DictationInsertionTarget | null): void => { + if (!holding) { + insert(text, target) + return + } + held = appendDictationText(held, text) + heldTarget = target + } return Object.assign(deliver, { landedText: () => allLanded, - settled: () => queue.then(() => missed === '') + settled: () => queue.then(() => missed === ''), + held: () => (holding && held ? held : null), + hold: (): void => { + holding = true + held = '' + }, + releaseHeld: (text = held): void => { + if (!holding) { + return + } + holding = false + if (text) { + insert(text, heldTarget) + } + } }) } diff --git a/src/renderer/src/components/dictation/use-dictation-cleanup.ts b/src/renderer/src/components/dictation/use-dictation-cleanup.ts new file mode 100644 index 00000000000..d61a54e4a8f --- /dev/null +++ b/src/renderer/src/components/dictation/use-dictation-cleanup.ts @@ -0,0 +1,87 @@ +import { useCallback, useEffect, useRef, useState, type RefObject } from 'react' +import { useAppStore } from '@/store' +import { + isDictationCleanupOn, + type DictationCleanupResult +} from '../../../../shared/dictation-cleanup' +import { createNonSecureContextUuid } from '../../../../shared/non-secure-context-uuid' +import { dictationPillAnchorFor, type DictationPillAnchor } from './dictation-pill-anchor' +import type { DictationKeptRecording } from './dictation-kept-recording' + +const NOTICE_MS = 4000 + +export type DictationCleanupNotice = { + kind: 'failed' | 'timed-out' + anchor: DictationPillAnchor | null +} + +/** Whether this desktop cleans dictated text itself; a paired web client's host does it in `speech.dictation.finish`. */ +export function shouldHoldForCleanup(): boolean { + const settings = useAppStore.getState().settings + return Boolean(window.api.speech.cleanUpDictation && settings && isDictationCleanupOn(settings)) +} + +/** Runs AI cleanup on the session's held words at finish; ✕ skips it, and the words then land as heard. */ +export function useDictationCleanup(keptRef: RefObject) { + const [cleaning, setCleaning] = useState(false) + const [notice, setNotice] = useState(null) + const skipRef = useRef<(() => void) | null>(null) + + useEffect(() => { + if (!notice) { + return + } + const id = window.setTimeout(() => setNotice(null), NOTICE_MS) + return () => window.clearTimeout(id) + }, [notice]) + + /** Lands the held words once: cleaned, or as heard when cleanup fails, takes too long or is skipped. */ + const land = useCallback(async (): Promise => { + const kept = keptRef.current + const heard = kept?.deliver.held() ?? null + const cleanUp = window.api.speech.cleanUpDictation + if (!kept || heard === null || !cleanUp) { + kept?.deliver.releaseHeld() + return + } + const requestId = createNonSecureContextUuid() + const skipped = Promise.withResolvers() + skipRef.current = () => { + skipped.resolve({ status: 'canceled' }) + void window.api.speech.cancelDictationCleanup?.(requestId).catch(() => undefined) + } + setNotice(null) + setCleaning(true) + const result = await Promise.race([ + cleanUp(heard, kept.context, requestId).catch((error: unknown): DictationCleanupResult => ({ + status: 'failed', + reason: String(error) + })), + skipped.promise + ]) + skipRef.current = null + setCleaning(false) + kept.deliver.releaseHeld(result.status === 'cleaned' ? result.text : undefined) + if (result.status === 'failed') { + console.warn('[dictation] AI cleanup failed:', result.reason) + } + if (result.status === 'failed' || result.status === 'timed-out') { + setNotice({ kind: result.status, anchor: dictationPillAnchorFor(kept.target) }) + } + }, [keptRef]) + + /** Skips a cleanup in progress (true), else lands any held words as heard: the session ends unfinished. */ + const interrupt = useCallback((): boolean => { + const skip = skipRef.current + if (skip) { + skip() + return true + } + keptRef.current?.deliver.releaseHeld() + return false + }, [keptRef]) + + const dismissNotice = useCallback(() => setNotice(null), []) + + return { cleaning, notice, land, interrupt, dismissNotice } +} diff --git a/src/renderer/src/components/dictation/use-dictation-retry.ts b/src/renderer/src/components/dictation/use-dictation-retry.ts new file mode 100644 index 00000000000..e2a73d9122d --- /dev/null +++ b/src/renderer/src/components/dictation/use-dictation-retry.ts @@ -0,0 +1,64 @@ +import { useCallback, type RefObject } from 'react' +import type { + DictationSession, + DictationSessionEvent +} from '../../../../shared/dictation-session-machine' +import { + canRetryDictation, + transcribeKeptRecording, + type DictationKeptRecording +} from './dictation-kept-recording' +import { + dictationStartFailureKind, + showDictationStartErrorToast, + type DictationFailureKind +} from './dictation-toasts' +import { shouldHoldForCleanup } from './use-dictation-cleanup' + +/** Sends the failed session's recording through the speech host again, into the same place. */ +export function useDictationRetry({ + modelId, + sessionRef, + keptRef, + deliverRef, + dispatch, + finish, + showFailure +}: { + modelId: string | undefined + sessionRef: RefObject + keptRef: RefObject + deliverRef: RefObject<((text: string) => void) | null> + dispatch: (event: DictationSessionEvent) => DictationSession + finish: (session: DictationSession, beforeEnd: () => void) => Promise + showFailure: (kind: DictationFailureKind | null, showToast: () => void) => void +}): () => Promise { + return useCallback(async () => { + const kept = keptRef.current + if (sessionRef.current.phase !== 'error' || !modelId || !kept || !canRetryDictation(kept)) { + return + } + const { run, sessionId } = dispatch({ type: 'retry' }) + if (sessionId === null) { + return + } + // The failure landed what was held as heard; what the retry adds waits for cleanup again. + if (shouldHoldForCleanup()) { + kept.deliver.hold() + } + await transcribeKeptRecording({ + recording: kept, + modelId, + sessionId, + isCurrent: () => sessionRef.current.run === run, + routeFinals: (sink) => { + deliverRef.current = sink + }, + finish: (beforeEnd) => finish(sessionRef.current, beforeEnd), + fail: (message) => + showFailure(dictationStartFailureKind(message) ?? 'transcribe', () => + showDictationStartErrorToast(message) + ) + }) + }, [modelId, sessionRef, keptRef, deliverRef, dispatch, finish, showFailure]) +} diff --git a/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx b/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx index f4776568666..69ca9429d21 100644 --- a/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx +++ b/src/renderer/src/components/settings/page/settings-capability-section-renderers.tsx @@ -183,7 +183,11 @@ export function renderDesktopCapabilitySettingsSections( searchEntries={navigation.getSectionSearchEntries('voice')} > {view.isSectionMounted('voice') ? ( - + ) : null} diff --git a/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts b/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts index 1d6d04b032c..95559c36dbc 100644 --- a/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts +++ b/src/renderer/src/components/settings/source-control/source-control-action-recipe-options.ts @@ -27,6 +27,10 @@ export const getActionDescriptions = createLocalizedCatalog((): Record(['amp', 'muse']) + const FALLBACK_AGENT_ARGS_PLACEHOLDER = '--model sonnet' const AGENT_ARGS_PLACEHOLDER_OVERRIDES: Partial> = { @@ -108,7 +115,10 @@ export function getAgentCatalogForAction( return getAgentCatalog() } return getAgentCatalog().filter( - (agent) => TEXT_GENERATION_AGENT_ID_SET.has(agent.id) || agent.id === selectedAgent + (agent) => + (TEXT_GENERATION_AGENT_ID_SET.has(agent.id) && + !(actionId === 'dictationCleanup' && NAMING_ONLY_TEXT_AGENT_IDS.has(agent.id))) || + agent.id === selectedAgent ) } diff --git a/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx b/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx index 469f6f0d646..b6ae73cffc1 100644 --- a/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx +++ b/src/renderer/src/components/settings/source-control/text-actions/SourceControlTextActionAgentPicker.tsx @@ -35,7 +35,10 @@ import { registeredRepoTextModelHostKeys } from './source-control-text-action-mo import { useTextActionModelDiscovery } from './use-text-action-model-discovery' import { textActionPickerEntryView } from './source-control-text-action-picker-entries' -import { AI_TEXT_ACTION_IDS } from '../../../../../../shared/source-control/ai/recipes/source-control-ai-actions' +import { + AI_TEXT_ACTION_IDS, + isGlobalAiTextAction +} from '../../../../../../shared/source-control/ai/recipes/source-control-ai-actions' import type { SourceControlAiSettingsPatch } from '../../../../../../shared/source-control/ai/settings/source-control-ai-settings-types' import { agentPickerDraft, @@ -124,7 +127,7 @@ const TextActionPicker = memo( } const inheritedSource = { ...source, modelOverridesByOperation: inheritedChoices } const repoChoices = { ...repoAi?.modelOverridesByOperation } - if (operation !== 'conversationName') { + if (!isGlobalAiTextAction(operation)) { delete repoChoices[operation] } const inheritedRepo = { ...repoAi, modelOverridesByOperation: repoChoices } @@ -165,18 +168,17 @@ const TextActionPicker = memo( [operation]: { ...source.actions?.[operation], agentId: agent.id } } } - const forcedRepo = - operation !== 'conversationName' - ? { - sourceControlAi: { - ...(ownRepo ?? repoAi), - actionOverrides: { - ...repoAi?.actionOverrides, - [operation]: { ...repoAi?.actionOverrides?.[operation], agentId: agent.id } - } + const forcedRepo = !isGlobalAiTextAction(operation) + ? { + sourceControlAi: { + ...(ownRepo ?? repoAi), + actionOverrides: { + ...repoAi?.actionOverrides, + [operation]: { ...repoAi?.actionOverrides?.[operation], agentId: agent.id } } } - : null + } + : null const inherited = resolveSourceControlAiActionModelDefault( { settings: { diff --git a/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts b/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts index 69808f28810..76e07a683de 100644 --- a/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts +++ b/src/renderer/src/components/settings/source-control/text-actions/source-control-text-action-picker-save.ts @@ -1,3 +1,4 @@ +import { isGlobalAiTextAction } from '../../../../../../shared/source-control/ai/recipes/source-control-ai-actions' import { agentPickerEffectiveChoices, type AgentPickerEntry, @@ -61,7 +62,7 @@ export function saveSourceControlTextPickerDraft({ } } let currentRepo: Pick | null = - operation !== 'conversationName' && repoId + !isGlobalAiTextAction(operation) && repoId ? { sourceControlAi: { ...repoAi, @@ -103,7 +104,7 @@ export function saveSourceControlTextPickerDraft({ currentSource = saved.sourceControlAi nextChoice = currentSource.modelOverridesByOperation?.[operation] nextArgs = currentSource.actions?.[operation]?.agentArgs ?? nextArgs - } else if (operation !== 'conversationName') { + } else if (!isGlobalAiTextAction(operation)) { currentRepo = { sourceControlAi: saved.update.sourceControlAi ?? undefined } nextChoice = currentRepo.sourceControlAi?.modelOverridesByOperation?.[operation] nextArgs = currentRepo.sourceControlAi?.actionOverrides?.[operation]?.agentArgs ?? nextArgs diff --git a/src/renderer/src/components/settings/voice/VoiceCleanupConsentDialog.tsx b/src/renderer/src/components/settings/voice/VoiceCleanupConsentDialog.tsx new file mode 100644 index 00000000000..f940b550772 --- /dev/null +++ b/src/renderer/src/components/settings/voice/VoiceCleanupConsentDialog.tsx @@ -0,0 +1,109 @@ +import { useTranslate, type Translate } from '@/i18n/use-translate' +import { Button } from '../../ui/button' +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle +} from '../../ui/dialog' + +// Only agents whose service is fixed; the others go wherever the user set them up to send. +const AGENT_SERVICES: Record = { + claude: { company: 'Anthropic', plan: 'Claude' }, + codex: { company: 'OpenAI', plan: 'ChatGPT' } +} + +function destination(agentId: string, agentLabel: string, t: Translate): string { + const service = AGENT_SERVICES[agentId] + return service + ? t( + 'components.settings.voice.cleanup.consent.goesToKnown', + '{{company}}, through {{agent}} on this computer. Counts toward your {{plan}} plan.', + { ...service, agent: agentLabel } + ) + : t( + 'components.settings.voice.cleanup.consent.goesToAgent', + 'Wherever {{agent}} on this computer is set up to send prompts.', + { agent: agentLabel } + ) +} + +/** Asked before dictated text first goes to an agent, and again for each new agent. */ +export function VoiceCleanupConsentDialog({ + request, + onCancel, + onConfirm +}: { + /** The agent text would go to; null keeps the dialog closed. */ + request: { agentId: string; agentLabel: string } | null + onCancel: () => void + onConfirm: () => void +}): React.JSX.Element { + const t = useTranslate() + const agentLabel = request?.agentLabel ?? '' + const rows = request + ? [ + { + label: t('components.settings.voice.cleanup.consent.sentLabel', 'Sent'), + value: t( + 'components.settings.voice.cleanup.consent.sent', + 'The dictated text, your vocabulary and your instructions.' + ) + }, + { + label: t('components.settings.voice.cleanup.consent.notSentLabel', 'Not sent'), + value: t( + 'components.settings.voice.cleanup.consent.notSent', + 'Audio, files, or what’s on screen.' + ) + }, + { + label: t('components.settings.voice.cleanup.consent.goesToLabel', 'Goes to'), + value: destination(request.agentId, agentLabel, t) + }, + { + label: t('components.settings.voice.cleanup.consent.waitLabel', 'Wait'), + value: t( + 'components.settings.voice.cleanup.consent.wait', + 'Text lands a few seconds later.' + ) + } + ] + : [] + return ( + !open && onCancel()}> + + + + {t('components.settings.voice.cleanup.consent.title', 'Turn on AI cleanup?')} + + + {t( + 'components.settings.voice.cleanup.consent.description', + 'After each dictation, Orca sends the text to {{agent}} to tidy it before it lands.', + { agent: agentLabel } + )} + + +
+ {rows.map((row) => ( +
+
{row.label}
+
{row.value}
+
+ ))} +
+ + + + +
+
+ ) +} diff --git a/src/renderer/src/components/settings/voice/VoiceCleanupInstructionsSetting.tsx b/src/renderer/src/components/settings/voice/VoiceCleanupInstructionsSetting.tsx new file mode 100644 index 00000000000..339cb26b183 --- /dev/null +++ b/src/renderer/src/components/settings/voice/VoiceCleanupInstructionsSetting.tsx @@ -0,0 +1,86 @@ +import { useId, useState } from 'react' +import { BUILT_IN_DICTATION_CLEANUP_INSTRUCTIONS } from '../../../../../shared/dictation-cleanup' +import { useTranslate } from '@/i18n/use-translate' +import { Button } from '../../ui/button' +import { Label } from '../../ui/label' +import { Textarea } from '../../ui/textarea' +import { SettingsRow } from '../form-controls/SettingsFormControls' + +/** Collapsed to a summary; Edit opens the built-in or saved text to change in place. */ +export function VoiceCleanupInstructionsSetting({ + instructions, + onSave +}: { + /** Absent: the built-in instructions. */ + instructions: string | undefined + onSave: (instructions: string | undefined) => void +}): React.JSX.Element { + const t = useTranslate() + const fieldId = useId() + const [draft, setDraft] = useState(null) + const custom = instructions?.trim() ? instructions : undefined + const label = t('components.settings.voice.cleanup.instructions.label', 'Instructions') + if (draft === null) { + return ( + setDraft(custom ?? BUILT_IN_DICTATION_CLEANUP_INSTRUCTIONS)} + > + {t('components.settings.voice.cleanup.instructions.edit', 'Edit')} + + } + /> + ) + } + const save = (): void => { + const text = draft.trim() + onSave(text && text !== BUILT_IN_DICTATION_CLEANUP_INSTRUCTIONS ? text : undefined) + setDraft(null) + } + return ( +
+
+ +

+ {t( + 'components.settings.voice.cleanup.instructions.emptyHint', + 'Leave empty for the built-in ones.' + )} +

+
+