diff --git a/mobile/src/dictation/MobileDictationLiveControls.tsx b/mobile/src/dictation/MobileDictationLiveControls.tsx index 2e298d16c80..0c5ece35fd9 100644 --- a/mobile/src/dictation/MobileDictationLiveControls.tsx +++ b/mobile/src/dictation/MobileDictationLiveControls.tsx @@ -1,14 +1,15 @@ import { useEffect, useMemo, useState, type Ref } from 'react' import { ActivityIndicator, Pressable, Text, View } from 'react-native' -import { Laptop, Mic, Smartphone, Square, X } from 'lucide-react-native' +import { CircleAlert, Laptop, Mic, RotateCcw, Smartphone, Square, X } from 'lucide-react-native' import type { MobileDictationMeter } from '../hooks/mobile-dictation-meter' import type { MobileDictationEngineShown } from '../hooks/mobile-dictation-session-state' import { useMobileTranslation } from '../i18n/use-mobile-translation' import { COMPOSER_CONTROL_HIT_SLOP, + COMPOSER_CONTROL_SIZE, composerInputStyles } from '../components/composer-input/composer-input-styles' -import { colors, spacing, typography } from '../theme/mobile-theme' +import { colors, radii, spacing, typography } from '../theme/mobile-theme' import { themedStyleSheet } from '../theme/themed-styles' import { useMobileColorScheme } from '../theme/use-mobile-color-scheme' import { @@ -27,11 +28,14 @@ const FINISHING_NOTE_DELAY_MS = 300 export function MobileDictationCancelButton({ onCancel, targetRef, - compact = false + compact = false, + discards = false }: { onCancel: () => void targetRef?: Ref compact?: boolean + // After a failure, ✕ throws the kept recording away. + discards?: boolean }): React.JSX.Element { useMobileColorScheme() useMobileTranslation() @@ -39,7 +43,9 @@ export function MobileDictationCancelButton({ [ composerInputStyles.controlButton, compact && styles.compactCancel, @@ -86,6 +92,77 @@ export function MobileDictationMicIcon({ ) } +// The failure that kept its recording: a reason and "Recording kept", on two lines in chat and one in the terminal. +export function MobileDictationFailureNote({ + reason, + compact = false +}: { + reason: string + compact?: boolean +}): React.JSX.Element { + useMobileColorScheme() + useMobileTranslation() + const kept = dictationControlsCopy.recordingKept() + return ( + + + {compact ? ( + + {`${reason} · ${kept}`} + + ) : ( + + + {reason} + + + {kept} + + + )} + + ) +} + +// Retry sits where Stop was: a labelled pill in chat, the round control in the terminal. +export function MobileDictationRetryButton({ + onRetry, + labelled +}: { + onRetry: () => void + labelled: boolean +}): React.JSX.Element { + useMobileColorScheme() + useMobileTranslation() + return ( + [ + labelled ? styles.retryPill : composerInputStyles.controlButton, + pressed && composerInputStyles.pressed + ]} + hitSlop={COMPOSER_CONTROL_HIT_SLOP} + onPress={onRetry} + > + + {labelled ? ( + {dictationControlsCopy.retryShort()} + ) : null} + + ) +} + function EngineIcon({ kind, color }: { kind: MobileDictationEngineShown['kind']; color: string }) { return kind === 'phone' ? ( @@ -190,5 +267,38 @@ const styles = themedStyleSheet(() => ({ flexShrink: 1, color: colors.textSecondary, fontSize: typography.metaSize + 1 + }, + failureLines: { + flexShrink: 1, + minWidth: 0 + }, + failureReason: { + color: colors.textPrimary, + fontSize: typography.metaSize + 1, + fontWeight: '500' + }, + failureKept: { + color: colors.textSecondary, + fontSize: typography.metaSize + }, + failureCompact: { + flexShrink: 1, + color: colors.textPrimary, + fontSize: typography.metaSize + }, + retryPill: { + height: COMPOSER_CONTROL_SIZE, + borderRadius: radii.pill, + alignItems: 'center', + justifyContent: 'center', + flexDirection: 'row', + gap: 6, + paddingHorizontal: spacing.md, + backgroundColor: colors.surfaceBright + }, + retryLabel: { + color: colors.bgBase, + fontSize: typography.bodySize, + fontWeight: '600' } })) diff --git a/mobile/src/dictation/mobile-dictation-controls-copy.ts b/mobile/src/dictation/mobile-dictation-controls-copy.ts index 3947035da5c..8b3b7126219 100644 --- a/mobile/src/dictation/mobile-dictation-controls-copy.ts +++ b/mobile/src/dictation/mobile-dictation-controls-copy.ts @@ -1,5 +1,11 @@ import { translate } from '../i18n/mobile-i18n' -import type { MobileDictationEngineShown } from '../hooks/mobile-dictation-session-state' +import type { + MobileDictationEngineShown, + MobileDictationKeptFailure +} from '../hooks/mobile-dictation-session-state' +import { MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE } from '../hooks/mobile-dictation-pending-audio-budget' +import { phoneDictationFailedMessage } from '../hooks/mobile-dictation-device-engine' +import { dictationFailureMessage, dictationSessionLostMessage } from './mobile-dictation-setup' // Active: starting or recording, so the mic is a Stop. export type MobileDictationMicState = { starting: boolean; active: boolean; processing: boolean } @@ -11,6 +17,10 @@ export const dictationControlsCopy = { stop: () => translate('dictation.controls.stop', 'Stop and insert'), finishing: () => translate('dictation.controls.finishing', 'Finishing dictation'), cancel: () => translate('dictation.controls.cancel', 'Cancel dictation'), + discard: () => translate('dictation.controls.discard', 'Discard recording'), + retry: () => translate('dictation.controls.retry', 'Retry dictation'), + retryShort: () => translate('dictation.controls.retryShort', 'Retry'), + recordingKept: () => translate('dictation.failure.recordingKept', 'Recording kept'), mic: ({ starting, active, processing }: MobileDictationMicState) => starting ? dictationControlsCopy.starting() @@ -79,3 +89,17 @@ export function dictationMeterLabel( } return translate('dictation.controls.meter', 'Recording, {{duration}}', { duration }) } + +// The few words a kept failure shows in the composer row; the full message is too long for it. +export function dictationFailureReason({ message, offline }: MobileDictationKeptFailure): string { + if (offline || dictationFailureMessage(message) === dictationSessionLostMessage()) { + return translate('dictation.failure.connectionLost', 'Connection lost') + } + if (message === MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE) { + return translate('dictation.failure.connectionSlow', 'Connection too slow') + } + if (message === phoneDictationFailedMessage()) { + return translate('dictation.failure.phoneStopped', 'Stopped on this phone') + } + return translate('dictation.failure.stopped', 'Dictation stopped') +} diff --git a/mobile/src/dictation/mobile-dictation-live-context.ts b/mobile/src/dictation/mobile-dictation-live-context.ts index 1032c48710b..9f5d6126b3d 100644 --- a/mobile/src/dictation/mobile-dictation-live-context.ts +++ b/mobile/src/dictation/mobile-dictation-live-context.ts @@ -7,7 +7,11 @@ export type MobileDictationLive = { readonly engine: MobileDictationEngineShown | null // The paired computer's name; null when another server runs the workspace and the phone lacks its name. readonly computerName: string | null + // Also discards a failed dictation's kept recording. readonly cancel: () => void + // The short reason a dictation failed with its recording kept; Retry plays it to the computer again. + readonly failureReason: string | null + readonly retry: () => void } // The chat mic sits several composers deep (overlay, queued edits); it reads the live dictation from here. diff --git a/mobile/src/dictation/mobile-dictation-setup.ts b/mobile/src/dictation/mobile-dictation-setup.ts index 9e7f034dbd8..421cd3c95ac 100644 --- a/mobile/src/dictation/mobile-dictation-setup.ts +++ b/mobile/src/dictation/mobile-dictation-setup.ts @@ -44,7 +44,7 @@ function speechlessServerMessage(): string { ) } -function isSpeechlessServerError(message: string): boolean { +export function isSpeechlessServerError(message: string): boolean { return message.startsWith('speech_unavailable') } diff --git a/mobile/src/dictation/use-mobile-dictation-live.ts b/mobile/src/dictation/use-mobile-dictation-live.ts index 5027a930c1f..37528abb0cb 100644 --- a/mobile/src/dictation/use-mobile-dictation-live.ts +++ b/mobile/src/dictation/use-mobile-dictation-live.ts @@ -4,10 +4,11 @@ import { useWorkspaceExecutionHost } from '../navigation/workspace-execution-hos import { useHostDisplay } from '../transport/hosts/use-host-display' import { useHostProfile } from '../transport/hosts/use-host-profile' import type { MobileDictationLive } from './mobile-dictation-live-context' +import { dictationFailureReason } from './mobile-dictation-controls-copy' -// What every dictation control reads while one runs: meter, engine, the computer's name and cancel. +// What every dictation control reads while one runs or waits for Retry: meter, engine, the computer's name, cancel and Retry. export function useMobileDictationLive( - dictation: Pick, + dictation: Pick, cancel: () => void, hostId: string ): MobileDictationLive { @@ -20,9 +21,17 @@ export function useMobileDictationLive( const hostTitle = useHostDisplay(hostProfile).title // A workspace routed to another server transcribes there, and the phone lacks that server's name. const computerName = useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null - const { meter, engine } = dictation + const { meter, engine, failure, retry } = dictation + const failureReason = failure === null ? null : dictationFailureReason(failure) return useMemo( - () => ({ meter, engine, computerName, cancel: stableCancel }), - [meter, engine, computerName, stableCancel] + () => ({ + meter, + engine, + computerName, + cancel: stableCancel, + failureReason, + retry: () => void retry() + }), + [meter, engine, computerName, stableCancel, failureReason, retry] ) } diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index c9a18b0d721..524c11ec299 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -20,6 +20,7 @@ import type { } from '../platform/dictation-capture-contract' import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' import type { MobileDictationHeldAudio } from './mobile-dictation-held-audio' +import type { MobileDictationRecording } from './mobile-dictation-recording' import type { RpcClient } from '../transport/rpc/rpc-client-contract' import type { MobileDictationRun } from './mobile-dictation-engine-run' @@ -35,7 +36,7 @@ type MobileDictationAudioChunkQueue = { stopAtAudioLimit: () => void } -// Holding: the microphone runs from the tap while the host's start reply is pending, or for a whole on-device dictation. +// Holding: the microphone runs from the tap while the host's start reply is pending, or while a replay's host start is. export type MobileDictationAudioPhase = 'closed' | 'holding' | 'sending' // Newer hosts report the text so far: shown while speaking, and handed over by a failure instead of dropped. @@ -122,6 +123,7 @@ export function useMobileDictationAudioChunks(capture: DictationCapture, run: Mo failActiveDictation: run.failActiveDictation, transcriptSoFar: run.keptTranscript, heldAudio: run.heldAudio.current, + recording: run.recording.current, // Like desktop: stop as the user would, so what the host kept is transcribed and inserted. stopAtAudioLimit: () => { if (run.status.current === 'recording') { @@ -142,7 +144,7 @@ export function subscribeMobileDictationAudioChunks( getActiveId: () => string | null audioPhase: () => MobileDictationAudioPhase }, - queue: Omit + queue: Omit & { recording: MobileDictationRecording } ): { remove: () => void; sendHeld: () => void } { // Reuse wiring on the microphone hot path instead of allocating per chunk. const chunkQueue = { ...queue, isActive: (id: string) => source.getActiveId() === id } @@ -159,11 +161,15 @@ export function subscribeMobileDictationAudioChunks( } const subscription = capture.onChunk((chunk) => { const dictationId = source.getActiveId() + if (!dictationId) { + return + } + // Kept whatever the phase, so a phone's failure or a failed send can play it to the computer again. + queue.recording.keep(chunk) const phase = source.audioPhase() - if (!dictationId || phase === 'closed') { + if (phase === 'closed') { return } - // An on-device dictation holds its audio with no computer at all. if (phase === 'holding') { queue.heldAudio.hold(chunk) return diff --git a/mobile/src/hooks/mobile-dictation-engine-run.ts b/mobile/src/hooks/mobile-dictation-engine-run.ts index 5abe593ec17..5d7307b19d8 100644 --- a/mobile/src/hooks/mobile-dictation-engine-run.ts +++ b/mobile/src/hooks/mobile-dictation-engine-run.ts @@ -9,11 +9,7 @@ import type { MobileDictationAudioPhase, MobileDictationTranscriptSoFar } from './mobile-dictation-audio-chunk' -import { - cancelDesktopDictation, - settledDesktopDictationWork, - startMobileDictationDesktopSession -} from './mobile-dictation-desktop-start' +import { cancelDesktopDictation } from './mobile-dictation-desktop-start' import { chooseMobileDictationRoute, phoneDictationBusyMessage, @@ -26,6 +22,11 @@ import { } from './mobile-dictation-device-engine' import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' import type { MobileDictationHeldAudio } from './mobile-dictation-held-audio' +import type { MobileDictationRecording } from './mobile-dictation-recording' +import { + replayMobileDictationToHost, + startMobileDictationOnHost +} from './mobile-dictation-host-start' import type { ShowMobileDictationEngine } from './mobile-dictation-engine-shown' // What the engines share with useMobileDictation: its refs (read at call time) and its state transitions. @@ -42,15 +43,18 @@ export type MobileDictationRun = { readonly pendingChunks: RefObject>> readonly pendingAudioBudget: RefObject readonly heldAudio: RefObject + readonly recording: RefObject readonly sendHeldAudio: RefObject<() => void> readonly finish: RefObject<(emptyTranscriptMessage: string) => Promise> readonly onTranscript: RefObject<(text: string) => void> readonly onAudioLimitReached: RefObject<(() => void) | undefined> readonly keptTranscript: MobileDictationTranscriptSoFar - readonly applyStatus: (next: DictationStatus) => void + // keepText: leave the words so far on screen (recording, finishing, or a failure kept for Retry). + readonly applyStatus: (next: DictationStatus, keepText?: boolean) => void readonly showEngine: ShowMobileDictationEngine readonly closeDictationAudio: () => void - readonly reportError: (err: unknown) => void + // kept: the recording stays for Retry, the words so far stay shown, and nothing is inserted yet. + readonly reportError: (err: unknown, kept?: boolean) => void readonly failActiveDictation: (dictationId: string, err: unknown) => void } @@ -80,6 +84,7 @@ export function settleMobileDictation( run.engine.current = { kind: 'host' } run.audioPhase.current = 'closed' run.heldAudio.current.clear() + run.recording.current.clear() run.pendingChunks.current.clear() run.pendingAudioBudget.current.reset() run.applyStatus('idle') @@ -90,48 +95,7 @@ export function settleMobileDictation( } } -export async function startMobileDictationOnHost( - run: MobileDictationRun, - client: RpcClient, - dictationId: string, - generation: number -): Promise { - await settledDesktopDictationWork(client) - if (run.generation.current !== generation) { - return - } - await startMobileDictationDesktopSession({ - client, - dictationId, - generation, - getCurrentGeneration: () => run.generation.current, - getEnabled: () => run.enabled.current, - getActiveId: () => run.activeId.current, - clearActiveId: (id) => { - if (run.activeId.current === id) { - run.activeId.current = null - } - }, - setIdle: () => run.applyStatus('idle'), - commitRecordingStart: (hostDedupesResends) => { - run.audioPhase.current = 'sending' - run.pendingChunks.current.clear() - run.pendingAudioBudget.current.reset(hostDedupesResends) - // A phone dictation stopped or failed at finish is already processing; it never records again. - if (run.status.current !== 'processing') { - run.applyStatus('recording') - } - run.showEngine('computer') - run.sendHeldAudio.current() - if (run.stopWhenStarted.current) { - void run.finish.current(dictationSessionErrorMessage('no-speech')) - } - }, - rollbackRecordingStart: run.closeDictationAudio - }) -} - -// The phone's recognizer failed: replay the audio it kept to the computer, as a start's held audio (seq from 0), and carry on there. +// The phone's recognizer failed: replay the recording to the computer, as a start's held audio (seq from 0), and carry on there. export async function fallBackMobileDictationToHost( run: MobileDictationRun, dictationId: string, @@ -149,25 +113,15 @@ export async function fallBackMobileDictationToHost( !engine.fallback || !client || client.getState() !== 'connected' || - !run.heldAudio.current.complete + !run.recording.current.complete ) { run.failActiveDictation(dictationId, new Error(phoneDictationFailedMessage())) return } - run.engine.current = { kind: 'host' } run.showEngine('computer', true) // A stop that began while the phone was still live is already processing; the computer must finish it too. run.stopWhenStarted.current = finishing || run.status.current === 'processing' - try { - await startMobileDictationOnHost(run, client, dictationId, run.generation.current) - } catch (err) { - // The host start already released the mic and the dictation; keep what the phone heard. - const kept = run.keptTranscript.take(dictationId) - if (kept) { - run.onTranscript.current(kept) - } - run.reportError(err) - } + await replayMobileDictationToHost(run, client, dictationId, run.generation.current) } // Pick the engine before the mic opens, and arm the phone's recognizer so it hears the first word. @@ -229,10 +183,9 @@ export function commitMobileDictationOnDevice( () => void fallBackMobileDictationToHost(run, dictationId, false) ) } - // This phone never replays to the computer, so it keeps no audio. - if (!fallback) { - run.audioPhase.current = 'closed' - } + // The recording keeps the phone's audio for a replay; nothing goes to the computer while the phone hears it. + run.audioPhase.current = 'closed' + run.heldAudio.current.clear() run.applyStatus('recording') run.showEngine('phone') if (run.stopWhenStarted.current) { diff --git a/mobile/src/hooks/mobile-dictation-failure.ts b/mobile/src/hooks/mobile-dictation-failure.ts new file mode 100644 index 00000000000..99ba7a258fa --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-failure.ts @@ -0,0 +1,44 @@ +import { errorMessage } from '../../../src/shared/error-message' +import { + isDictationSetupRequiredError, + isSpeechlessServerError +} from '../dictation/mobile-dictation-setup' +import type { MobileDictationEngine } from './mobile-dictation-device-engine' +import type { MobileDictationRecording } from './mobile-dictation-recording' +import type { MobileDictationRun } from './mobile-dictation-engine-run' + +// Retry plays the recording to the computer, so it needs the whole recording and an engine that may use the computer (not This phone). +export function mobileDictationFailureRetryable( + recording: MobileDictationRecording, + engine: MobileDictationEngine +): boolean { + return recording.complete && recording.bytes > 0 && (engine.kind === 'host' || engine.fallback) +} + +// A computer that needs setup or an update refuses every Retry the same way; those keep today's path (words inserted, setup sheet or toast). +function retryCanSucceed(err: unknown): boolean { + const message = errorMessage(err) + return !isDictationSetupRequiredError(message) && !isSpeechlessServerError(message) +} + +// Never lose speech quietly: a retryable failure keeps the recording and the words so far for Retry; otherwise the words so far are inserted, as before. +export function concludeMobileDictationFailure( + run: Pick< + MobileDictationRun, + 'recording' | 'engine' | 'keptTranscript' | 'onTranscript' | 'reportError' + >, + dictationId: string, + err: unknown, + retryable = mobileDictationFailureRetryable(run.recording.current, run.engine.current) +): void { + if (retryable && retryCanSucceed(err)) { + run.reportError(err, true) + return + } + run.recording.current.clear() + const kept = run.keptTranscript.take(dictationId) + if (kept) { + run.onTranscript.current(kept) + } + run.reportError(err) +} diff --git a/mobile/src/hooks/mobile-dictation-finish.ts b/mobile/src/hooks/mobile-dictation-finish.ts new file mode 100644 index 00000000000..c676c3de41f --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-finish.ts @@ -0,0 +1,77 @@ +import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload' +import type { DictationCapture } from '../platform/dictation-capture-contract' +import { isCurrentMobileDictationFinish } from './mobile-dictation-session-state' +import { finishDesktopDictation } from './mobile-dictation-desktop-start' +import { + finishMobileDictationOnDevice, + settleMobileDictation, + type MobileDictationRun +} from './mobile-dictation-engine-run' + +// Stop listening and settle the dictation with its final text, from whichever engine heard it. +export async function finishMobileDictation( + run: MobileDictationRun, + capture: Pick, + emptyTranscriptMessage: string +): Promise { + const client = run.client.current + const dictationId = run.activeId.current + const engine = run.engine.current + if (!dictationId || (engine.kind === 'host' && !client)) { + return + } + // A replay's host start is still pending: finish once it commits and the kept audio has gone. + if (engine.kind === 'host' && run.audioPhase.current === 'holding') { + run.stopWhenStarted.current = true + if (run.status.current === 'recording') { + run.applyStatus('processing') + // Stop listening now; the tail joins the held audio the replay sends. + void capture.end() + } + return + } + + const generation = run.generation.current + 1 + run.generation.current = generation + run.applyStatus('processing') + const isCurrent = () => + isCurrentMobileDictationFinish( + run.generation.current, + generation, + run.enabled.current, + run.activeId.current, + dictationId + ) + try { + // Keep shutdown inside try for finally cleanup; accept its tail chunks and snapshot pending sends only after end settles, or finish can overtake the last audio. + await capture.end() + if (engine.kind === 'device') { + await finishMobileDictationOnDevice( + run, + engine, + dictationId, + emptyTranscriptMessage, + isCurrent + ) + return + } + run.audioPhase.current = 'closed' + await Promise.allSettled(Array.from(run.pendingChunks.current)) + if (!isCurrent() || !client) { + return + } + const sentSeconds = run.pendingAudioBudget.current.sentAudioSeconds + const finished = await finishDesktopDictation(client, dictationId, sentSeconds) + if (!isCurrent()) { + return + } + const transcript = rpcPayloadMember(finished, 'text') + settleMobileDictation( + run, + typeof transcript === 'string' ? transcript.trim() : '', + emptyTranscriptMessage + ) + } catch (err) { + run.failActiveDictation(dictationId, err) + } +} diff --git a/mobile/src/hooks/mobile-dictation-held-audio.ts b/mobile/src/hooks/mobile-dictation-held-audio.ts index 3e57dfad7d7..21e142a5683 100644 --- a/mobile/src/hooks/mobile-dictation-held-audio.ts +++ b/mobile/src/hooks/mobile-dictation-held-audio.ts @@ -5,10 +5,10 @@ import { } from '../../../src/shared/rpc-contract/speech-params' import type { DictationCaptureChunk } from '../platform/dictation-capture-contract' -// Five minutes of 16 kHz PCM (9.6 MB): what an on-device dictation keeps to replay to the computer if the phone fails. -const MAX_HELD_AUDIO_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * 300 +// Five minutes of 16 kHz PCM (9.6 MB): the longest recording kept for a replay to the computer. +export const MAX_HELD_AUDIO_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * 300 -// Speech not sent yet, in order: held between the tap and the host's start reply, while every send slot is busy, and for a whole on-device dictation. +// Speech not sent yet, in order: held between the tap and the host's start reply, while every send slot is busy, and for a replay of the recording. export class MobileDictationHeldAudio { private held: DictationCaptureChunk[] = [] private heldBytes = 0 diff --git a/mobile/src/hooks/mobile-dictation-host-start.ts b/mobile/src/hooks/mobile-dictation-host-start.ts new file mode 100644 index 00000000000..2ad40864322 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-host-start.ts @@ -0,0 +1,69 @@ +import type { RpcClient } from '../transport/rpc/rpc-client-contract' +import { dictationSessionErrorMessage } from '../dictation/mobile-dictation-setup' +import { + settledDesktopDictationWork, + startMobileDictationDesktopSession +} from './mobile-dictation-desktop-start' +import { concludeMobileDictationFailure } from './mobile-dictation-failure' +import type { MobileDictationRun } from './mobile-dictation-engine-run' + +export async function startMobileDictationOnHost( + run: MobileDictationRun, + client: RpcClient, + dictationId: string, + generation: number, + // A replay (fallback or Retry) keeps the words so far on screen if its start fails. + replay = false +): Promise { + await settledDesktopDictationWork(client) + if (run.generation.current !== generation) { + return + } + await startMobileDictationDesktopSession({ + client, + dictationId, + generation, + getCurrentGeneration: () => run.generation.current, + getEnabled: () => run.enabled.current, + getActiveId: () => run.activeId.current, + clearActiveId: (id) => { + if (run.activeId.current === id) { + run.activeId.current = null + } + }, + setIdle: () => run.applyStatus('idle', replay), + commitRecordingStart: (hostDedupesResends) => { + run.audioPhase.current = 'sending' + run.pendingChunks.current.clear() + run.pendingAudioBudget.current.reset(hostDedupesResends) + // A phone dictation stopped or failed at finish is already processing; it never records again. + if (run.status.current !== 'processing') { + run.applyStatus('recording') + } + run.showEngine('computer') + run.sendHeldAudio.current() + if (run.stopWhenStarted.current) { + void run.finish.current(dictationSessionErrorMessage('no-speech')) + } + }, + rollbackRecordingStart: run.closeDictationAudio + }) +} + +// Play the kept recording to the computer as this dictation's held audio (seq from 0); audio heard after joins it in order until the start commits. +// Fallback and Retry both end here; a failed start keeps the recording for Retry. +export async function replayMobileDictationToHost( + run: MobileDictationRun, + client: RpcClient, + dictationId: string, + generation: number +): Promise { + run.engine.current = { kind: 'host' } + run.recording.current.replayInto(run.heldAudio.current) + run.audioPhase.current = 'holding' + try { + await startMobileDictationOnHost(run, client, dictationId, generation, true) + } catch (err) { + concludeMobileDictationFailure(run, dictationId, err) + } +} diff --git a/mobile/src/hooks/mobile-dictation-recording.ts b/mobile/src/hooks/mobile-dictation-recording.ts new file mode 100644 index 00000000000..872c6231794 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-recording.ts @@ -0,0 +1,47 @@ +import type { DictationCaptureChunk } from '../platform/dictation-capture-contract' +import { MAX_HELD_AUDIO_BYTES, type MobileDictationHeldAudio } from './mobile-dictation-held-audio' + +// Everything one dictation heard, from the tap, whichever engine hears it: replayed to the computer when the phone fails, and kept after a failure for Retry. +export class MobileDictationRecording { + private chunks: DictationCaptureChunk[] = [] + private keptBytes = 0 + private incomplete = false + + keep(chunk: DictationCaptureChunk): void { + if (this.incomplete) { + return + } + // Past the cap, or with audio the page dropped, a replay would lose speech (and the send refuses it), so keep nothing and say it is incomplete. + if (chunk.droppedBytes > 0 || this.keptBytes + chunk.data.byteLength > MAX_HELD_AUDIO_BYTES) { + this.chunks = [] + this.keptBytes = 0 + this.incomplete = true + return + } + this.chunks.push(chunk) + this.keptBytes += chunk.data.byteLength + } + + // False once speech was lost; the recording no longer covers the whole dictation. + get complete(): boolean { + return !this.incomplete + } + + get bytes(): number { + return this.keptBytes + } + + // Queue a copy for sending; the recording itself stays, so a failed replay can be retried. + replayInto(held: MobileDictationHeldAudio): void { + held.clear() + for (const chunk of this.chunks) { + held.hold(chunk) + } + } + + clear(): void { + this.chunks = [] + this.keptBytes = 0 + this.incomplete = false + } +} diff --git a/mobile/src/hooks/mobile-dictation-retry.test.tsx b/mobile/src/hooks/mobile-dictation-retry.test.tsx new file mode 100644 index 00000000000..474685d5a83 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-retry.test.tsx @@ -0,0 +1,166 @@ +// A computer dictation that fails mid-session keeps its whole recording; Retry plays all of it to the computer as a new dictation. +import { createElement } from 'react' +import { act, create } from 'react-test-renderer' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + createFakeRpcClient, + type FakeRpcClient, + type SentRequest +} from '../mobile-web-shell/bridge-host-test-fakes' +import type { + DictationCapture, + DictationCaptureChunk +} from '../platform/dictation-capture-contract' + +type Seam = { + chunkHandlers: Set<(chunk: DictationCaptureChunk) => void> + tail: Uint8Array | null +} + +const seam = vi.hoisted((): Seam => ({ chunkHandlers: new Set(), tail: null })) + +vi.mock('react-native', () => ({ + AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) }, + Platform: { OS: 'ios' } +})) + +// Keep capture identity stable like both real seams; a fresh object on render would rerun teardown and cancel dictation. +vi.mock('../platform/dictation-capture', () => { + const capture: DictationCapture = { + open: async () => ({ ok: true }), + begin: () => true, + end: async () => { + // Page capture reads once more here and may deliver audio. + const tail = seam.tail + seam.tail = null + if (tail !== null) { + for (const handler of seam.chunkHandlers) { + handler({ data: tail, droppedBytes: 0 }) + } + } + }, + release: () => {}, + onChunk: (handler) => { + seam.chunkHandlers.add(handler) + return { + remove: () => { + seam.chunkHandlers.delete(handler) + } + } + }, + onInterruption: () => ({ remove: () => {} }) + } + return { useDictationCapture: () => capture } +}) + +import { useMobileDictation, type UseMobileDictationResult } from './use-mobile-dictation' + +type Reply = (request: SentRequest) => 'hold' | { text?: string } | Error + +// Answer every request; one reply can unblock the next. +async function pump(rpc: FakeRpcClient, sent: SentRequest[], reply: Reply): Promise { + for (let round = 0; round < 30; round += 1) { + for (const request of rpc.requests.splice(0)) { + sent.push(request) + const answer = reply(request) + if (answer === 'hold') { + continue + } + if (answer instanceof Error) { + request.reject(answer) + } else { + request.resolve({ id: 'desktop', ok: true, result: answer }) + } + } + await Promise.resolve() + await Promise.resolve() + } +} + +function paramsOf(request: SentRequest): Record { + const params = request.args[1] + return typeof params === 'object' && params !== null ? { ...params } : {} +} + +function deliver(bytes: number[]): void { + for (const handler of seam.chunkHandlers) { + handler({ data: Uint8Array.from(bytes), droppedBytes: 0 }) + } +} + +const held: { dictation: UseMobileDictationResult | null } = { dictation: null } + +function dictation(): UseMobileDictationResult { + const current = held.dictation + if (current === null) { + throw new Error('nothing mounted') + } + return current +} + +beforeEach(() => { + seam.chunkHandlers.clear() + seam.tail = null + held.dictation = null +}) + +describe('a computer dictation whose finish fails', () => { + it('keeps the recording, and Retry sends all of it again and inserts the words', async () => { + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const inserted: string[] = [] + const errors: { message: string; kept: boolean }[] = [] + function Probe(): null { + held.dictation = useMobileDictation({ + client: rpc, + enabled: true, + onTranscript: (text) => inserted.push(text), + onError: (error, outcome) => errors.push({ message: error.message, kept: outcome.kept }) + }) + return null + } + act(() => { + create(createElement(Probe)) + }) + const firstTry: Reply = (request) => + request.method === 'speech.dictation.finish' ? new Error('connection lost') : {} + await act(async () => { + const started = dictation().start() + await pump(rpc, sent, firstTry) + await started + }) + await act(async () => { + deliver([1, 2, 3, 4]) + await pump(rpc, sent, firstTry) + }) + seam.tail = Uint8Array.from([5, 6]) + await act(async () => { + const stopped = dictation().stop() + await pump(rpc, sent, firstTry) + await stopped + }) + expect(inserted).toEqual([]) + expect(errors).toEqual([{ message: 'connection lost', kept: true }]) + expect(dictation().failure).toEqual({ message: 'connection lost', offline: false }) + + const firstId = paramsOf(sent[0]).dictationId + sent.length = 0 + await act(async () => { + const retried = dictation().retry() + await pump(rpc, sent, (request) => + request.method === 'speech.dictation.finish' ? { text: 'all of it' } : {} + ) + await retried + }) + expect(sent.map((request) => request.method)).toEqual([ + 'speech.dictation.start', + 'speech.dictation.chunk', + 'speech.dictation.finish' + ]) + expect(paramsOf(sent[0]).dictationId).not.toBe(firstId) + // The whole recording, from the first byte, as seq 0 of the new dictation. + expect(paramsOf(sent[1])).toMatchObject({ audioBase64: 'AQIDBAUG', seq: 0 }) + expect(inserted).toEqual(['all of it']) + expect(dictation().failure).toBeNull() + }) +}) diff --git a/mobile/src/hooks/mobile-dictation-retry.ts b/mobile/src/hooks/mobile-dictation-retry.ts new file mode 100644 index 00000000000..7581ae378f5 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-retry.ts @@ -0,0 +1,42 @@ +import type { RpcClient } from '../transport/rpc/rpc-client-contract' +import { createMobileDictationId } from './mobile-dictation-session-state' +import { + concludeMobileDictationFailure, + mobileDictationFailureRetryable +} from './mobile-dictation-failure' +import { + releaseMobileDictationEngine, + type MobileDictationRun +} from './mobile-dictation-engine-run' +import { replayMobileDictationToHost } from './mobile-dictation-host-start' + +// A dictation failed after it began hearing: stop it, and keep its recording for Retry when the computer can hear it again. +export function failActiveMobileDictation( + run: MobileDictationRun, + dictationId: string, + err: unknown +): void { + if (run.activeId.current !== dictationId) { + return + } + run.activeId.current = null + const retryable = mobileDictationFailureRetryable(run.recording.current, run.engine.current) + run.closeDictationAudio() + void releaseMobileDictationEngine(run, run.client.current, dictationId) + concludeMobileDictationFailure(run, dictationId, err, retryable) +} + +// Retry: the kept recording goes to the computer as a new dictation (seq from 0), already stopped, so it finishes as soon as it has been sent. +export async function retryMobileDictationOnHost( + run: MobileDictationRun, + client: RpcClient +): Promise { + const generation = run.generation.current + 1 + run.generation.current = generation + const dictationId = createMobileDictationId() + run.activeId.current = dictationId + run.stopWhenStarted.current = true + run.applyStatus('processing', true) + run.showEngine('computer') + await replayMobileDictationToHost(run, client, dictationId, generation) +} diff --git a/mobile/src/hooks/mobile-dictation-session-state.ts b/mobile/src/hooks/mobile-dictation-session-state.ts index a2ab4abb53e..4b6fa76d157 100644 --- a/mobile/src/hooks/mobile-dictation-session-state.ts +++ b/mobile/src/hooks/mobile-dictation-session-state.ts @@ -9,11 +9,15 @@ export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | // Who is transcribing, for the icon by the timer; movedAt marks when Automatic moved from the phone to the computer. export type MobileDictationEngineShown = { kind: 'phone' | 'computer'; movedAt: number | null } +// offline: the computer was out of reach when it failed, which is the reason the user needs. +export type MobileDictationKeptFailure = { message: string; offline: boolean } + export type UseMobileDictationOptions = { client: RpcClient | null enabled: boolean onTranscript: (text: string) => void - onError?: (error: Error) => void + // kept: the recording is kept for Retry, so the failure shows in place rather than as a toast. + onError?: (error: Error, outcome: { kept: boolean }) => void // The computer's filler switch, for text the phone recognizes itself; unknown means on. removeFillerWords?: boolean // The host's cloud upload is full; the dictation stops and inserts what was kept. @@ -28,10 +32,13 @@ export type UseMobileDictationResult = { // Empty until a newer host reports words; older hosts never do. transcriptSoFar: string error: string | null + // A failure whose recording is kept: Retry plays it to the computer again, cancel discards it. + failure: MobileDictationKeptFailure | null // Null until recording starts on an engine. engine: MobileDictationEngineShown | null start: () => Promise stop: () => Promise + retry: () => Promise cancel: () => Promise meter: MobileDictationMeter } diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index bfc0e9cd3c9..df0d1c6134e 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -13,19 +13,19 @@ import { type MobileDictationAudioPhase } from './mobile-dictation-audio-chunk' import { MobileDictationHeldAudio } from './mobile-dictation-held-audio' -import { isCurrentMobileDictationFinish } from './mobile-dictation-session-state' -import { finishDesktopDictation } from './mobile-dictation-desktop-start' +import { MobileDictationRecording } from './mobile-dictation-recording' +import { failActiveMobileDictation, retryMobileDictationOnHost } from './mobile-dictation-retry' +import { mobileDictationFailureRetryable } from './mobile-dictation-failure' +import { finishMobileDictation } from './mobile-dictation-finish' import type { MobileDictationEngine } from './mobile-dictation-device-engine' import { - finishMobileDictationOnDevice, releaseMobileDictationEngine, - settleMobileDictation, startMobileDictation, type MobileDictationRun } from './mobile-dictation-engine-run' -import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload' import type { DictationStatus, + MobileDictationKeptFailure, UseMobileDictationOptions, UseMobileDictationResult } from './mobile-dictation-session-state' @@ -40,6 +40,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const speech = useOnDeviceSpeech(removeFillerWords) const [status, setStatus] = useState('idle') const [error, setError] = useState(null) + // A failure whose recording is kept for Retry; cleared by Retry, discard and the next dictation. + const [failure, setFailure] = useState(null) + const failureRef = useRef(failure) const [transcriptSoFar, setTranscriptSoFar, keptTranscript] = useMobileDictationTranscriptSoFar() const { shown: shownEngine, @@ -61,6 +64,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget()) const audioPhaseRef = useRef('closed') const heldAudioRef = useRef(new MobileDictationHeldAudio()) + const recordingRef = useRef(new MobileDictationRecording()) const sendHeldAudioRef = useRef(() => {}) const stopWhenStartedRef = useRef(false) const generationRef = useRef(0) @@ -77,27 +81,36 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil }, [client, speech, enabled, onTranscript, onError, onAudioLimitReached]) const applyStatus = useCallback( - (next: DictationStatus) => { + // Kept through processing so the text so far stays in place until the final transcript replaces it in the same render. + (next: DictationStatus, keepText = next === 'recording' || next === 'processing') => { statusRef.current = next setStatus(next) - // Kept through processing so the text so far stays in place until the final transcript replaces it in the same render. - const live = next === 'recording' || next === 'processing' - setTranscriptSoFar((text) => (live ? text : '')) - if (!live) { + setTranscriptSoFar((text) => (keepText ? text : '')) + if (next !== 'recording' && next !== 'processing') { clearEngine() } }, [clearEngine] ) + const setKeptFailure = useCallback((next: MobileDictationKeptFailure | null) => { + failureRef.current = next + setFailure(next) + }, []) + const reportError = useCallback( - (err: unknown) => { + (err: unknown, kept = false) => { const normalized = err instanceof Error ? err : new Error(String(err)) setError(normalized.message) - applyStatus('error') - onErrorRef.current?.(normalized) + applyStatus('error', kept) + setKeptFailure( + kept + ? { message: normalized.message, offline: clientRef.current?.getState() !== 'connected' } + : null + ) + onErrorRef.current?.(normalized, { kept }) }, - [applyStatus] + [applyStatus, setKeptFailure] ) const closeDictationAudio = useCallback(() => { @@ -113,29 +126,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil } }, [capture]) - const failActiveDictation = useCallback( - (dictationId: string, err: unknown) => { - if (activeIdRef.current !== dictationId) { - return - } - activeIdRef.current = null - closeDictationAudio() - void releaseMobileDictationEngine( - { engine: engineRef, speech: speechRef }, - clientRef.current, - dictationId - ) - const kept = keptTranscript.take(dictationId) - if (kept) { - onTranscriptRef.current(kept) - } - reportError(err) - }, - [closeDictationAudio, reportError] - ) - - const run = useMemo( - () => ({ + const run = useMemo(() => { + const created: MobileDictationRun = { status: statusRef, activeId: activeIdRef, client: clientRef, @@ -148,6 +140,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil pendingChunks: pendingChunksRef, pendingAudioBudget: pendingAudioBudgetRef, heldAudio: heldAudioRef, + recording: recordingRef, sendHeldAudio: sendHeldAudioRef, finish: finishRef, onTranscript: onTranscriptRef, @@ -157,77 +150,17 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil showEngine, closeDictationAudio, reportError, - failActiveDictation - }), - [applyStatus, closeDictationAudio, failActiveDictation, keptTranscript, reportError, showEngine] - ) + failActiveDictation: (dictationId, err) => + failActiveMobileDictation(created, dictationId, err) + } + return created + }, [applyStatus, closeDictationAudio, keptTranscript, reportError, showEngine]) useMobileDictationAudioChunks(capture, run) const finishDictation = useCallback( - async (emptyTranscriptMessage: string) => { - const client = clientRef.current - const dictationId = activeIdRef.current - const engine = engineRef.current - if (!dictationId || (engine.kind === 'host' && !client)) { - return - } - // A fallback's host start is still pending: finish once it commits and the kept audio has gone. - if (engine.kind === 'host' && audioPhaseRef.current === 'holding') { - stopWhenStartedRef.current = true - if (statusRef.current === 'recording') { - applyStatus('processing') - // Stop listening now; the tail joins the held audio the replay sends. - void capture.end() - } - return - } - - const generation = generationRef.current + 1 - generationRef.current = generation - applyStatus('processing') - const isCurrent = () => - isCurrentMobileDictationFinish( - generationRef.current, - generation, - enabledRef.current, - activeIdRef.current, - dictationId - ) - try { - // Keep shutdown inside try for finally cleanup; accept its tail chunks and snapshot pending sends only after end settles, or finish can overtake the last audio. - await capture.end() - if (engine.kind === 'device') { - await finishMobileDictationOnDevice( - run, - engine, - dictationId, - emptyTranscriptMessage, - isCurrent - ) - return - } - audioPhaseRef.current = 'closed' - await Promise.allSettled(Array.from(pendingChunksRef.current)) - if (!isCurrent() || !client) { - return - } - const sentSeconds = pendingAudioBudgetRef.current.sentAudioSeconds - const finished = await finishDesktopDictation(client, dictationId, sentSeconds) - if (!isCurrent()) { - return - } - const transcript = rpcPayloadMember(finished, 'text') - settleMobileDictation( - run, - typeof transcript === 'string' ? transcript.trim() : '', - emptyTranscriptMessage - ) - } catch (err) { - failActiveDictation(dictationId, err) - } - }, - [applyStatus, capture, failActiveDictation, run] + (emptyTranscriptMessage: string) => finishMobileDictation(run, capture, emptyTranscriptMessage), + [capture, run] ) useLayoutEffect(() => { finishRef.current = finishDictation @@ -247,8 +180,21 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil return } setError(null) + // A new dictation replaces a failed one's kept recording. + setKeptFailure(null) + recordingRef.current.clear() await startMobileDictation(run, capture) - }, [capture, run]) + }, [capture, run, setKeptFailure]) + + const retry = useCallback(async () => { + const client = clientRef.current + if (!failureRef.current || !client || activeIdRef.current || !enabledRef.current) { + return + } + setError(null) + setKeptFailure(null) + await retryMobileDictationOnHost(run, client) + }, [run, setKeptFailure]) // Only the user’s own cancel (reason:null) ends silently; other interruptions call onError so the tap has a visible outcome. const abandonDictation = useCallback( @@ -256,11 +202,26 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const client = clientRef.current const dictationId = activeIdRef.current const wasUnderway = statusRef.current === 'starting' || statusRef.current === 'recording' + // Only the user's discard drops a kept recording; an interruption while it waits leaves it for Retry. + if (reason !== null && dictationId === null && failureRef.current) { + return + } + // An interruption keeps a recording the computer can hear again, even mid-finish; the user's cancel never does. + const kept = + reason !== null && + dictationId !== null && + mobileDictationFailureRetryable(recordingRef.current, engineRef.current) + if (!kept) { + setKeptFailure(null) + recordingRef.current.clear() + } generationRef.current += 1 activeIdRef.current = null closeDictationAudio() // Settle before the host's cancel reply: a host that went quiet must not hold the mic button in its old state. - if (reason !== null && wasUnderway) { + if (kept) { + reportError(new Error(reason), true) + } else if (reason !== null && wasUnderway) { reportError(new Error(reason)) } else { applyStatus('idle') @@ -268,7 +229,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil } await releaseMobileDictationEngine(run, client, dictationId) }, - [applyStatus, closeDictationAudio, reportError, run] + [applyStatus, closeDictationAudio, reportError, run, setKeptFailure] ) const cancel = useCallback(() => abandonDictation(null), [abandonDictation]) @@ -295,6 +256,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil generationRef.current += 1 activeIdRef.current = null closeDictationAudio() + recordingRef.current.clear() capture.release() void releaseMobileDictationEngine(run, clientRef.current, dictationId) } @@ -307,8 +269,10 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil isProcessing: status === 'processing', transcriptSoFar, error, + failure, engine: shownEngine, start, + retry, stop, cancel, meter: useMobileDictationMeter(capture, status === 'recording') diff --git a/mobile/src/i18n/locales/de.json b/mobile/src/i18n/locales/de.json index 8fdd8ecc62b..4224caf2472 100644 --- a/mobile/src/i18n/locales/de.json +++ b/mobile/src/i18n/locales/de.json @@ -1317,13 +1317,23 @@ "durationSeconds_one": "{{count}} Sekunde", "durationSeconds_other": "{{count}} Sekunden", "durationMinutes_one": "{{count}} Minute", - "durationMinutes_other": "{{count}} Minuten" + "durationMinutes_other": "{{count}} Minuten", + "discard": "Aufnahme verwerfen", + "retry": "Diktat wiederholen", + "retryShort": "Wiederholen" }, "status": { "finishingOn": "Wird auf {{computer}} abgeschlossen", "finishingOnComputer": "Wird auf dem Computer abgeschlossen", "movedTo": "Zu {{computer}} gewechselt", "movedToComputer": "Zum Computer gewechselt" + }, + "failure": { + "connectionLost": "Verbindung unterbrochen", + "connectionSlow": "Verbindung zu langsam", + "phoneStopped": "Auf diesem Telefon gestoppt", + "stopped": "Diktat gestoppt", + "recordingKept": "Aufnahme behalten" } }, "files": { diff --git a/mobile/src/i18n/locales/en.json b/mobile/src/i18n/locales/en.json index df763bf71c0..d75375894eb 100644 --- a/mobile/src/i18n/locales/en.json +++ b/mobile/src/i18n/locales/en.json @@ -534,13 +534,23 @@ "meterOnPhone": "Recording, {{duration}}, on this phone", "meterOn": "Recording, {{duration}}, on {{computer}}", "meterOnComputer": "Recording, {{duration}}, on the computer", - "meter": "Recording, {{duration}}" + "meter": "Recording, {{duration}}", + "discard": "Discard recording", + "retry": "Retry dictation", + "retryShort": "Retry" }, "status": { "finishingOn": "Finishing on {{computer}}", "finishingOnComputer": "Finishing on the computer", "movedTo": "Moved to {{computer}}", "movedToComputer": "Moved to the computer" + }, + "failure": { + "recordingKept": "Recording kept", + "connectionLost": "Connection lost", + "connectionSlow": "Connection too slow", + "phoneStopped": "Stopped on this phone", + "stopped": "Dictation stopped" } }, "home": { diff --git a/mobile/src/i18n/locales/es.json b/mobile/src/i18n/locales/es.json index 93db73377e9..3219dc9de50 100644 --- a/mobile/src/i18n/locales/es.json +++ b/mobile/src/i18n/locales/es.json @@ -1358,13 +1358,23 @@ "durationSeconds_many": "{{count}} segundos", "durationMinutes_one": "{{count}} minuto", "durationMinutes_other": "{{count}} minutos", - "durationMinutes_many": "{{count}} minutos" + "durationMinutes_many": "{{count}} minutos", + "discard": "Descartar grabación", + "retry": "Reintentar dictado", + "retryShort": "Reintentar" }, "status": { "finishingOn": "Terminando en {{computer}}", "finishingOnComputer": "Terminando en el ordenador", "movedTo": "Pasado a {{computer}}", "movedToComputer": "Pasado al ordenador" + }, + "failure": { + "connectionLost": "Conexión perdida", + "connectionSlow": "Conexión demasiado lenta", + "phoneStopped": "Detenido en este teléfono", + "stopped": "Dictado detenido", + "recordingKept": "Grabación guardada" } }, "files": { diff --git a/mobile/src/i18n/locales/fr.json b/mobile/src/i18n/locales/fr.json index 30c9e9c2b4c..76088bbd669 100644 --- a/mobile/src/i18n/locales/fr.json +++ b/mobile/src/i18n/locales/fr.json @@ -1358,13 +1358,23 @@ "durationSeconds_many": "{{count}} secondes", "durationMinutes_one": "{{count}} minute", "durationMinutes_other": "{{count}} minutes", - "durationMinutes_many": "{{count}} minutes" + "durationMinutes_many": "{{count}} minutes", + "discard": "Supprimer l’enregistrement", + "retry": "Relancer la dictée", + "retryShort": "Réessayer" }, "status": { "finishingOn": "Finalisation sur {{computer}}", "finishingOnComputer": "Finalisation sur l’ordinateur", "movedTo": "Passé sur {{computer}}", "movedToComputer": "Passé sur l’ordinateur" + }, + "failure": { + "connectionLost": "Connexion perdue", + "connectionSlow": "Connexion trop lente", + "phoneStopped": "Arrêté sur ce téléphone", + "stopped": "Dictée arrêtée", + "recordingKept": "Enregistrement conservé" } }, "files": { diff --git a/mobile/src/i18n/locales/ja.json b/mobile/src/i18n/locales/ja.json index 7af42d5c585..f265e30f077 100644 --- a/mobile/src/i18n/locales/ja.json +++ b/mobile/src/i18n/locales/ja.json @@ -1313,13 +1313,23 @@ "durationSeconds_one": "{{count}} 秒", "durationSeconds_other": "{{count}} 秒", "durationMinutes_one": "{{count}} 分", - "durationMinutes_other": "{{count}} 分" + "durationMinutes_other": "{{count}} 分", + "discard": "録音を破棄", + "retry": "音声入力を再試行", + "retryShort": "再試行" }, "status": { "finishingOn": "{{computer}} で仕上げています", "finishingOnComputer": "コンピュータで仕上げています", "movedTo": "{{computer}} に切り替えました", "movedToComputer": "コンピュータに切り替えました" + }, + "failure": { + "connectionLost": "接続が切れました", + "connectionSlow": "接続が遅すぎます", + "phoneStopped": "このスマートフォンで停止しました", + "stopped": "音声入力が停止しました", + "recordingKept": "録音は保持されています" } }, "files": { diff --git a/mobile/src/i18n/locales/ko.json b/mobile/src/i18n/locales/ko.json index 5f34699729a..e9f544e21a0 100644 --- a/mobile/src/i18n/locales/ko.json +++ b/mobile/src/i18n/locales/ko.json @@ -1313,13 +1313,23 @@ "durationSeconds_one": "{{count}}초", "durationSeconds_other": "{{count}}초", "durationMinutes_one": "{{count}}분", - "durationMinutes_other": "{{count}}분" + "durationMinutes_other": "{{count}}분", + "discard": "녹음 삭제", + "retry": "받아쓰기 다시 시도", + "retryShort": "다시 시도" }, "status": { "finishingOn": "{{computer}}에서 마무리 중", "finishingOnComputer": "컴퓨터에서 마무리 중", "movedTo": "{{computer}}(으)로 전환됨", "movedToComputer": "컴퓨터로 전환됨" + }, + "failure": { + "connectionLost": "연결 끊김", + "connectionSlow": "연결이 너무 느림", + "phoneStopped": "이 휴대폰에서 중지됨", + "stopped": "받아쓰기 중지됨", + "recordingKept": "녹음 보관됨" } }, "files": { diff --git a/mobile/src/i18n/locales/pt.json b/mobile/src/i18n/locales/pt.json index 5ecd4f2b77d..99fc6e2c73b 100644 --- a/mobile/src/i18n/locales/pt.json +++ b/mobile/src/i18n/locales/pt.json @@ -1358,13 +1358,23 @@ "durationSeconds_many": "{{count}} segundos", "durationMinutes_one": "{{count}} minuto", "durationMinutes_other": "{{count}} minutos", - "durationMinutes_many": "{{count}} minutos" + "durationMinutes_many": "{{count}} minutos", + "discard": "Descartar gravação", + "retry": "Tentar ditado novamente", + "retryShort": "Tentar novamente" }, "status": { "finishingOn": "Finalizando em {{computer}}", "finishingOnComputer": "Finalizando no computador", "movedTo": "Passou para {{computer}}", "movedToComputer": "Passou para o computador" + }, + "failure": { + "connectionLost": "Conexão perdida", + "connectionSlow": "Conexão muito lenta", + "phoneStopped": "Parou neste telefone", + "stopped": "Ditado interrompido", + "recordingKept": "Gravação mantida" } }, "files": { diff --git a/mobile/src/i18n/locales/ru.json b/mobile/src/i18n/locales/ru.json index fc5ba8b987e..dfd13d6fb85 100644 --- a/mobile/src/i18n/locales/ru.json +++ b/mobile/src/i18n/locales/ru.json @@ -1399,13 +1399,23 @@ "durationMinutes_one": "{{count}} минута", "durationMinutes_other": "{{count}} минуты", "durationMinutes_few": "{{count}} минуты", - "durationMinutes_many": "{{count}} минут" + "durationMinutes_many": "{{count}} минут", + "discard": "Удалить запись", + "retry": "Повторить диктовку", + "retryShort": "Повторить" }, "status": { "finishingOn": "Завершается на {{computer}}", "finishingOnComputer": "Завершается на компьютере", "movedTo": "Перешло на {{computer}}", "movedToComputer": "Перешло на компьютер" + }, + "failure": { + "connectionLost": "Соединение потеряно", + "connectionSlow": "Соединение слишком медленное", + "phoneStopped": "Остановлено на этом телефоне", + "stopped": "Диктовка остановлена", + "recordingKept": "Запись сохранена" } }, "files": { diff --git a/mobile/src/i18n/locales/tr.json b/mobile/src/i18n/locales/tr.json index 1648a819495..69f0376ead0 100644 --- a/mobile/src/i18n/locales/tr.json +++ b/mobile/src/i18n/locales/tr.json @@ -1317,13 +1317,23 @@ "durationSeconds_one": "{{count}} saniye", "durationSeconds_other": "{{count}} saniye", "durationMinutes_one": "{{count}} dakika", - "durationMinutes_other": "{{count}} dakika" + "durationMinutes_other": "{{count}} dakika", + "discard": "Kaydı sil", + "retry": "Dikteyi yeniden dene", + "retryShort": "Yeniden dene" }, "status": { "finishingOn": "{{computer}} üzerinde tamamlanıyor", "finishingOnComputer": "Bilgisayarda tamamlanıyor", "movedTo": "{{computer}} cihazına geçildi", "movedToComputer": "Bilgisayara geçildi" + }, + "failure": { + "connectionLost": "Bağlantı kesildi", + "connectionSlow": "Bağlantı çok yavaş", + "phoneStopped": "Bu telefonda durdu", + "stopped": "Dikte durdu", + "recordingKept": "Kayıt saklandı" } }, "files": { diff --git a/mobile/src/i18n/locales/zh-Hant.json b/mobile/src/i18n/locales/zh-Hant.json index c9c3df04598..191d1597ef7 100644 --- a/mobile/src/i18n/locales/zh-Hant.json +++ b/mobile/src/i18n/locales/zh-Hant.json @@ -1313,13 +1313,23 @@ "durationSeconds_one": "{{count}} 秒", "durationSeconds_other": "{{count}} 秒", "durationMinutes_one": "{{count}} 分鐘", - "durationMinutes_other": "{{count}} 分鐘" + "durationMinutes_other": "{{count}} 分鐘", + "discard": "捨棄錄音", + "retry": "重試聽寫", + "retryShort": "重試" }, "status": { "finishingOn": "正在 {{computer}} 上完成", "finishingOnComputer": "正在電腦上完成", "movedTo": "已轉到 {{computer}}", "movedToComputer": "已轉到電腦" + }, + "failure": { + "connectionLost": "連線中斷", + "connectionSlow": "連線太慢", + "phoneStopped": "已在此手機上停止", + "stopped": "聽寫已停止", + "recordingKept": "錄音已保留" } }, "files": { diff --git a/mobile/src/i18n/locales/zh.json b/mobile/src/i18n/locales/zh.json index 611e00ec678..a1723710973 100644 --- a/mobile/src/i18n/locales/zh.json +++ b/mobile/src/i18n/locales/zh.json @@ -1313,13 +1313,23 @@ "durationSeconds_one": "{{count}} 秒", "durationSeconds_other": "{{count}} 秒", "durationMinutes_one": "{{count}} 分钟", - "durationMinutes_other": "{{count}} 分钟" + "durationMinutes_other": "{{count}} 分钟", + "discard": "丢弃录音", + "retry": "重试听写", + "retryShort": "重试" }, "status": { "finishingOn": "正在 {{computer}} 上完成", "finishingOnComputer": "正在电脑上完成", "movedTo": "已转到 {{computer}}", "movedToComputer": "已转到电脑" + }, + "failure": { + "connectionLost": "连接已断开", + "connectionSlow": "连接太慢", + "phoneStopped": "已在此手机上停止", + "stopped": "听写已停止", + "recordingKept": "录音已保留" } }, "files": { diff --git a/mobile/src/session/native-chat/composer/MobileNativeChatComposer.tsx b/mobile/src/session/native-chat/composer/MobileNativeChatComposer.tsx index 2e10e27d24f..b58cdede682 100644 --- a/mobile/src/session/native-chat/composer/MobileNativeChatComposer.tsx +++ b/mobile/src/session/native-chat/composer/MobileNativeChatComposer.tsx @@ -345,7 +345,7 @@ export function MobileNativeChatComposer({ /> {goal.chip} - {micActive || micProcessing ? null : contextRing.ring} {onMicPress ? ( + } return ( void; held: boolean } | null onSend: (intent?: MobileNativeChatSendIntent, keepFocus?: boolean) => void -}): React.JSX.Element { +}): React.JSX.Element | null { + const dictationFailed = useMobileNativeChatDictationFailed() const steerChord = useMobileNativeChatSteerChordPreview({ fieldRef, hasDraft: value.trim() !== '' || attachmentCount > 0, @@ -83,6 +85,10 @@ export function MobileNativeChatComposerSendSlot({ draft: value, attachmentCount }) + // A dictation kept for Retry takes Send's room; a running agent keeps its Stop. + if (dictationFailed && !(showsStop && stop)) { + return null + } if (steerChord.preview && (steerChord.preview === 'steer-queued' || canSend)) { // As desktop, the preview takes Send's slot at once, with no ease. return ( diff --git a/mobile/src/session/native-chat/composer/MobileNativeChatDictationLeading.tsx b/mobile/src/session/native-chat/composer/MobileNativeChatDictationLeading.tsx index 272ac466892..806609bf8f6 100644 --- a/mobile/src/session/native-chat/composer/MobileNativeChatDictationLeading.tsx +++ b/mobile/src/session/native-chat/composer/MobileNativeChatDictationLeading.tsx @@ -3,28 +3,45 @@ import { View } from 'react-native' import { MobileDictationLiveContext } from '../../../dictation/mobile-dictation-live-context' import { MobileDictationCancelButton, + MobileDictationFailureNote, MobileDictationFinishingNote, MobileDictationRecordingMeter } from '../../../dictation/MobileDictationLiveControls' import { mobileNativeChatComposerStyles as styles } from './mobile-native-chat-composer-styles' -// While dictating, the row's leading side swaps attach and the pickers for ✕ and the meter, so the composer keeps its height. +// While dictating, the row's leading side swaps attach and the pickers for ✕ and the meter (or a kept failure), so the composer keeps its height. export function MobileNativeChatDictationLeading({ micActive, micStarting = false, micProcessing = false, cancelRef, + trailing, children }: { micActive: boolean micStarting?: boolean micProcessing?: boolean cancelRef: RefObject + // The context ring after the leading controls; it steps aside with them. + trailing: ReactNode children: ReactNode }): React.JSX.Element { const live = useContext(MobileDictationLiveContext) + if (live?.failureReason != null) { + return ( + + + + + ) + } if (!live || (!micActive && !micProcessing)) { - return {children} + return ( + <> + {children} + {trailing} + + ) } return ( @@ -42,3 +59,8 @@ export function MobileNativeChatDictationLeading({ ) } + +// A dictation failed with its recording kept: Send gives its room to Retry. +export function useMobileNativeChatDictationFailed(): boolean { + return useContext(MobileDictationLiveContext)?.failureReason != null +} diff --git a/mobile/src/session/use-mobile-session-native-chat-dictation.ts b/mobile/src/session/use-mobile-session-native-chat-dictation.ts index aaafe9b0f95..ffd2a1e30ac 100644 --- a/mobile/src/session/use-mobile-session-native-chat-dictation.ts +++ b/mobile/src/session/use-mobile-session-native-chat-dictation.ts @@ -166,7 +166,12 @@ export function useMobileSessionNativeChatDictation( } })().catch(() => keepUndeliveredDictation(text)) }, - onError: (err) => { + onError: (err, { kept }) => { + // A kept failure shows in place with Retry, which still needs the dictation's target. + if (kept) { + triggerError() + return + } dictationRouteContextRef.current = null reportDictationFailure(err) }, diff --git a/mobile/src/terminal/session/MobileTerminalInputActions.tsx b/mobile/src/terminal/session/MobileTerminalInputActions.tsx index ca3a69e4ef2..3730693bdad 100644 --- a/mobile/src/terminal/session/MobileTerminalInputActions.tsx +++ b/mobile/src/terminal/session/MobileTerminalInputActions.tsx @@ -1,4 +1,4 @@ -import type { RefObject } from 'react' +import { useContext, type RefObject } from 'react' import { ActivityIndicator, Pressable, @@ -17,7 +17,11 @@ import { useMobileColorScheme } from '../../theme/use-mobile-color-scheme' import { keepHeldPressThroughLongPress } from '../../platform/held-press-long-press' import { dictationControlsCopy } from '../../dictation/mobile-dictation-controls-copy' import { useMobileDictationCancelDrop } from '../../dictation/use-mobile-dictation-cancel-drop' -import { MobileDictationMicIcon } from '../../dictation/MobileDictationLiveControls' +import { + MobileDictationMicIcon, + MobileDictationRetryButton +} from '../../dictation/MobileDictationLiveControls' +import { MobileDictationLiveContext } from '../../dictation/mobile-dictation-live-context' type DictationState = { readonly isStarting: boolean @@ -54,11 +58,20 @@ type MobileTerminalInputActionsProps = MobileTerminalDictationButtonProps & export function MobileTerminalInputActions(props: MobileTerminalInputActionsProps) { const { isStarting, isRecording, isProcessing } = props.dictation - // Attach steps aside while dictating so the words get the field's room. + const live = useContext(MobileDictationLiveContext) + // A kept failure is cleared by every start and Retry, so it never shows while dictating. + const failureReason = live?.failureReason ?? null + // Attach steps aside while dictating or holding a failed recording, so the field gets the room. return ( <> - {isStarting || isRecording || isProcessing ? null : } - + {isStarting || isRecording || isProcessing || failureReason !== null ? null : ( + + )} + {failureReason !== null && live ? ( + + ) : ( + + )} ) } diff --git a/mobile/src/terminal/session/MobileTerminalLiveInputStatus.tsx b/mobile/src/terminal/session/MobileTerminalLiveInputStatus.tsx index a60cf7965ba..45d018b21eb 100644 --- a/mobile/src/terminal/session/MobileTerminalLiveInputStatus.tsx +++ b/mobile/src/terminal/session/MobileTerminalLiveInputStatus.tsx @@ -12,6 +12,7 @@ import { import { MobileDictationLiveContext } from '../../dictation/mobile-dictation-live-context' import { MobileDictationCancelButton, + MobileDictationFailureNote, MobileDictationFinishingNote, MobileDictationRecordingMeter } from '../../dictation/MobileDictationLiveControls' @@ -37,6 +38,16 @@ export function MobileTerminalLiveInputStatus({ const t = useMobileTranslation() const live = useContext(MobileDictationLiveContext) const dictationTitle = mobileDictationStatusTitle(dictation) + if (live?.failureReason != null) { + return ( + + + + + + + ) + } if (!dictationTitle) { return isAttaching ? (