feat(mobile): keep a failed dictation's recording and offer Retry

This commit is contained in:
Kelvin Amoaba
2026-11-09 09:50:54 +00:00
parent 1b78d1d1d4
commit cc48a9f6a3
34 changed files with 908 additions and 214 deletions
@@ -1,14 +1,15 @@
import { useEffect, useMemo, useState, type Ref } from 'react'
import { ActivityIndicator, Pressable, Text, View } from 'react-native'
import { Laptop, Mic, Smartphone, Square, X } from 'lucide-react-native'
import { CircleAlert, Laptop, Mic, RotateCcw, Smartphone, Square, X } from 'lucide-react-native'
import type { MobileDictationMeter } from '../hooks/mobile-dictation-meter'
import type { MobileDictationEngineShown } from '../hooks/mobile-dictation-session-state'
import { useMobileTranslation } from '../i18n/use-mobile-translation'
import {
COMPOSER_CONTROL_HIT_SLOP,
COMPOSER_CONTROL_SIZE,
composerInputStyles
} from '../components/composer-input/composer-input-styles'
import { colors, spacing, typography } from '../theme/mobile-theme'
import { colors, radii, spacing, typography } from '../theme/mobile-theme'
import { themedStyleSheet } from '../theme/themed-styles'
import { useMobileColorScheme } from '../theme/use-mobile-color-scheme'
import {
@@ -27,11 +28,14 @@ const FINISHING_NOTE_DELAY_MS = 300
export function MobileDictationCancelButton({
onCancel,
targetRef,
compact = false
compact = false,
discards = false
}: {
onCancel: () => void
targetRef?: Ref<View>
compact?: boolean
// After a failure, ✕ throws the kept recording away.
discards?: boolean
}): React.JSX.Element {
useMobileColorScheme()
useMobileTranslation()
@@ -39,7 +43,9 @@ export function MobileDictationCancelButton({
<Pressable
ref={targetRef}
accessibilityRole="button"
accessibilityLabel={dictationControlsCopy.cancel()}
accessibilityLabel={
discards ? dictationControlsCopy.discard() : dictationControlsCopy.cancel()
}
style={({ pressed }) => [
composerInputStyles.controlButton,
compact && styles.compactCancel,
@@ -86,6 +92,77 @@ export function MobileDictationMicIcon({
)
}
// The failure that kept its recording: a reason and "Recording kept", on two lines in chat and one in the terminal.
export function MobileDictationFailureNote({
reason,
compact = false
}: {
reason: string
compact?: boolean
}): React.JSX.Element {
useMobileColorScheme()
useMobileTranslation()
const kept = dictationControlsCopy.recordingKept()
return (
<View style={styles.row} accessible accessibilityLabel={`${reason}. ${kept}.`}>
<CircleAlert
pointerEvents="none"
size={compact ? 15 : 17}
color={colors.statusRed}
strokeWidth={2.2}
/>
{compact ? (
<Text style={styles.failureCompact} numberOfLines={1}>
{`${reason} · ${kept}`}
</Text>
) : (
<View style={styles.failureLines}>
<Text style={styles.failureReason} numberOfLines={1}>
{reason}
</Text>
<Text style={styles.failureKept} numberOfLines={1}>
{kept}
</Text>
</View>
)}
</View>
)
}
// Retry sits where Stop was: a labelled pill in chat, the round control in the terminal.
export function MobileDictationRetryButton({
onRetry,
labelled
}: {
onRetry: () => void
labelled: boolean
}): React.JSX.Element {
useMobileColorScheme()
useMobileTranslation()
return (
<Pressable
accessibilityRole="button"
accessibilityLabel={dictationControlsCopy.retry()}
style={({ pressed }) => [
labelled ? styles.retryPill : composerInputStyles.controlButton,
pressed && composerInputStyles.pressed
]}
hitSlop={COMPOSER_CONTROL_HIT_SLOP}
onPress={onRetry}
>
<RotateCcw
pointerEvents="none"
size={15}
color={labelled ? colors.bgBase : colors.textPrimary}
strokeWidth={2.4}
/>
{labelled ? (
<Text style={styles.retryLabel}>{dictationControlsCopy.retryShort()}</Text>
) : null}
</Pressable>
)
}
function EngineIcon({ kind, color }: { kind: MobileDictationEngineShown['kind']; color: string }) {
return kind === 'phone' ? (
<Smartphone pointerEvents="none" size={15} color={color} strokeWidth={2} />
@@ -190,5 +267,38 @@ const styles = themedStyleSheet(() => ({
flexShrink: 1,
color: colors.textSecondary,
fontSize: typography.metaSize + 1
},
failureLines: {
flexShrink: 1,
minWidth: 0
},
failureReason: {
color: colors.textPrimary,
fontSize: typography.metaSize + 1,
fontWeight: '500'
},
failureKept: {
color: colors.textSecondary,
fontSize: typography.metaSize
},
failureCompact: {
flexShrink: 1,
color: colors.textPrimary,
fontSize: typography.metaSize
},
retryPill: {
height: COMPOSER_CONTROL_SIZE,
borderRadius: radii.pill,
alignItems: 'center',
justifyContent: 'center',
flexDirection: 'row',
gap: 6,
paddingHorizontal: spacing.md,
backgroundColor: colors.surfaceBright
},
retryLabel: {
color: colors.bgBase,
fontSize: typography.bodySize,
fontWeight: '600'
}
}))
@@ -1,5 +1,11 @@
import { translate } from '../i18n/mobile-i18n'
import type { MobileDictationEngineShown } from '../hooks/mobile-dictation-session-state'
import type {
MobileDictationEngineShown,
MobileDictationKeptFailure
} from '../hooks/mobile-dictation-session-state'
import { MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE } from '../hooks/mobile-dictation-pending-audio-budget'
import { phoneDictationFailedMessage } from '../hooks/mobile-dictation-device-engine'
import { dictationFailureMessage, dictationSessionLostMessage } from './mobile-dictation-setup'
// Active: starting or recording, so the mic is a Stop.
export type MobileDictationMicState = { starting: boolean; active: boolean; processing: boolean }
@@ -11,6 +17,10 @@ export const dictationControlsCopy = {
stop: () => translate('dictation.controls.stop', 'Stop and insert'),
finishing: () => translate('dictation.controls.finishing', 'Finishing dictation'),
cancel: () => translate('dictation.controls.cancel', 'Cancel dictation'),
discard: () => translate('dictation.controls.discard', 'Discard recording'),
retry: () => translate('dictation.controls.retry', 'Retry dictation'),
retryShort: () => translate('dictation.controls.retryShort', 'Retry'),
recordingKept: () => translate('dictation.failure.recordingKept', 'Recording kept'),
mic: ({ starting, active, processing }: MobileDictationMicState) =>
starting
? dictationControlsCopy.starting()
@@ -79,3 +89,17 @@ export function dictationMeterLabel(
}
return translate('dictation.controls.meter', 'Recording, {{duration}}', { duration })
}
// The few words a kept failure shows in the composer row; the full message is too long for it.
export function dictationFailureReason({ message, offline }: MobileDictationKeptFailure): string {
if (offline || dictationFailureMessage(message) === dictationSessionLostMessage()) {
return translate('dictation.failure.connectionLost', 'Connection lost')
}
if (message === MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE) {
return translate('dictation.failure.connectionSlow', 'Connection too slow')
}
if (message === phoneDictationFailedMessage()) {
return translate('dictation.failure.phoneStopped', 'Stopped on this phone')
}
return translate('dictation.failure.stopped', 'Dictation stopped')
}
@@ -7,7 +7,11 @@ export type MobileDictationLive = {
readonly engine: MobileDictationEngineShown | null
// The paired computer's name; null when another server runs the workspace and the phone lacks its name.
readonly computerName: string | null
// Also discards a failed dictation's kept recording.
readonly cancel: () => void
// The short reason a dictation failed with its recording kept; Retry plays it to the computer again.
readonly failureReason: string | null
readonly retry: () => void
}
// The chat mic sits several composers deep (overlay, queued edits); it reads the live dictation from here.
@@ -44,7 +44,7 @@ function speechlessServerMessage(): string {
)
}
function isSpeechlessServerError(message: string): boolean {
export function isSpeechlessServerError(message: string): boolean {
return message.startsWith('speech_unavailable')
}
@@ -4,10 +4,11 @@ import { useWorkspaceExecutionHost } from '../navigation/workspace-execution-hos
import { useHostDisplay } from '../transport/hosts/use-host-display'
import { useHostProfile } from '../transport/hosts/use-host-profile'
import type { MobileDictationLive } from './mobile-dictation-live-context'
import { dictationFailureReason } from './mobile-dictation-controls-copy'
// What every dictation control reads while one runs: meter, engine, the computer's name and cancel.
// What every dictation control reads while one runs or waits for Retry: meter, engine, the computer's name, cancel and Retry.
export function useMobileDictationLive(
dictation: Pick<UseMobileDictationResult, 'meter' | 'engine'>,
dictation: Pick<UseMobileDictationResult, 'meter' | 'engine' | 'failure' | 'retry'>,
cancel: () => void,
hostId: string
): MobileDictationLive {
@@ -20,9 +21,17 @@ export function useMobileDictationLive(
const hostTitle = useHostDisplay(hostProfile).title
// A workspace routed to another server transcribes there, and the phone lacks that server's name.
const computerName = useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
const { meter, engine } = dictation
const { meter, engine, failure, retry } = dictation
const failureReason = failure === null ? null : dictationFailureReason(failure)
return useMemo(
() => ({ meter, engine, computerName, cancel: stableCancel }),
[meter, engine, computerName, stableCancel]
() => ({
meter,
engine,
computerName,
cancel: stableCancel,
failureReason,
retry: () => void retry()
}),
[meter, engine, computerName, stableCancel, failureReason, retry]
)
}
@@ -20,6 +20,7 @@ import type {
} from '../platform/dictation-capture-contract'
import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget'
import type { MobileDictationHeldAudio } from './mobile-dictation-held-audio'
import type { MobileDictationRecording } from './mobile-dictation-recording'
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
import type { MobileDictationRun } from './mobile-dictation-engine-run'
@@ -35,7 +36,7 @@ type MobileDictationAudioChunkQueue = {
stopAtAudioLimit: () => void
}
// Holding: the microphone runs from the tap while the host's start reply is pending, or for a whole on-device dictation.
// Holding: the microphone runs from the tap while the host's start reply is pending, or while a replay's host start is.
export type MobileDictationAudioPhase = 'closed' | 'holding' | 'sending'
// Newer hosts report the text so far: shown while speaking, and handed over by a failure instead of dropped.
@@ -122,6 +123,7 @@ export function useMobileDictationAudioChunks(capture: DictationCapture, run: Mo
failActiveDictation: run.failActiveDictation,
transcriptSoFar: run.keptTranscript,
heldAudio: run.heldAudio.current,
recording: run.recording.current,
// Like desktop: stop as the user would, so what the host kept is transcribed and inserted.
stopAtAudioLimit: () => {
if (run.status.current === 'recording') {
@@ -142,7 +144,7 @@ export function subscribeMobileDictationAudioChunks(
getActiveId: () => string | null
audioPhase: () => MobileDictationAudioPhase
},
queue: Omit<MobileDictationAudioChunkQueue, 'isActive'>
queue: Omit<MobileDictationAudioChunkQueue, 'isActive'> & { recording: MobileDictationRecording }
): { remove: () => void; sendHeld: () => void } {
// Reuse wiring on the microphone hot path instead of allocating per chunk.
const chunkQueue = { ...queue, isActive: (id: string) => source.getActiveId() === id }
@@ -159,11 +161,15 @@ export function subscribeMobileDictationAudioChunks(
}
const subscription = capture.onChunk((chunk) => {
const dictationId = source.getActiveId()
if (!dictationId) {
return
}
// Kept whatever the phase, so a phone's failure or a failed send can play it to the computer again.
queue.recording.keep(chunk)
const phase = source.audioPhase()
if (!dictationId || phase === 'closed') {
if (phase === 'closed') {
return
}
// An on-device dictation holds its audio with no computer at all.
if (phase === 'holding') {
queue.heldAudio.hold(chunk)
return
+18 -65
View File
@@ -9,11 +9,7 @@ import type {
MobileDictationAudioPhase,
MobileDictationTranscriptSoFar
} from './mobile-dictation-audio-chunk'
import {
cancelDesktopDictation,
settledDesktopDictationWork,
startMobileDictationDesktopSession
} from './mobile-dictation-desktop-start'
import { cancelDesktopDictation } from './mobile-dictation-desktop-start'
import {
chooseMobileDictationRoute,
phoneDictationBusyMessage,
@@ -26,6 +22,11 @@ import {
} from './mobile-dictation-device-engine'
import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget'
import type { MobileDictationHeldAudio } from './mobile-dictation-held-audio'
import type { MobileDictationRecording } from './mobile-dictation-recording'
import {
replayMobileDictationToHost,
startMobileDictationOnHost
} from './mobile-dictation-host-start'
import type { ShowMobileDictationEngine } from './mobile-dictation-engine-shown'
// What the engines share with useMobileDictation: its refs (read at call time) and its state transitions.
@@ -42,15 +43,18 @@ export type MobileDictationRun = {
readonly pendingChunks: RefObject<Set<Promise<void>>>
readonly pendingAudioBudget: RefObject<MobileDictationPendingAudioBudget>
readonly heldAudio: RefObject<MobileDictationHeldAudio>
readonly recording: RefObject<MobileDictationRecording>
readonly sendHeldAudio: RefObject<() => void>
readonly finish: RefObject<(emptyTranscriptMessage: string) => Promise<void>>
readonly onTranscript: RefObject<(text: string) => void>
readonly onAudioLimitReached: RefObject<(() => void) | undefined>
readonly keptTranscript: MobileDictationTranscriptSoFar
readonly applyStatus: (next: DictationStatus) => void
// keepText: leave the words so far on screen (recording, finishing, or a failure kept for Retry).
readonly applyStatus: (next: DictationStatus, keepText?: boolean) => void
readonly showEngine: ShowMobileDictationEngine
readonly closeDictationAudio: () => void
readonly reportError: (err: unknown) => void
// kept: the recording stays for Retry, the words so far stay shown, and nothing is inserted yet.
readonly reportError: (err: unknown, kept?: boolean) => void
readonly failActiveDictation: (dictationId: string, err: unknown) => void
}
@@ -80,6 +84,7 @@ export function settleMobileDictation(
run.engine.current = { kind: 'host' }
run.audioPhase.current = 'closed'
run.heldAudio.current.clear()
run.recording.current.clear()
run.pendingChunks.current.clear()
run.pendingAudioBudget.current.reset()
run.applyStatus('idle')
@@ -90,48 +95,7 @@ export function settleMobileDictation(
}
}
export async function startMobileDictationOnHost(
run: MobileDictationRun,
client: RpcClient,
dictationId: string,
generation: number
): Promise<void> {
await settledDesktopDictationWork(client)
if (run.generation.current !== generation) {
return
}
await startMobileDictationDesktopSession({
client,
dictationId,
generation,
getCurrentGeneration: () => run.generation.current,
getEnabled: () => run.enabled.current,
getActiveId: () => run.activeId.current,
clearActiveId: (id) => {
if (run.activeId.current === id) {
run.activeId.current = null
}
},
setIdle: () => run.applyStatus('idle'),
commitRecordingStart: (hostDedupesResends) => {
run.audioPhase.current = 'sending'
run.pendingChunks.current.clear()
run.pendingAudioBudget.current.reset(hostDedupesResends)
// A phone dictation stopped or failed at finish is already processing; it never records again.
if (run.status.current !== 'processing') {
run.applyStatus('recording')
}
run.showEngine('computer')
run.sendHeldAudio.current()
if (run.stopWhenStarted.current) {
void run.finish.current(dictationSessionErrorMessage('no-speech'))
}
},
rollbackRecordingStart: run.closeDictationAudio
})
}
// The phone's recognizer failed: replay the audio it kept to the computer, as a start's held audio (seq from 0), and carry on there.
// The phone's recognizer failed: replay the recording to the computer, as a start's held audio (seq from 0), and carry on there.
export async function fallBackMobileDictationToHost(
run: MobileDictationRun,
dictationId: string,
@@ -149,25 +113,15 @@ export async function fallBackMobileDictationToHost(
!engine.fallback ||
!client ||
client.getState() !== 'connected' ||
!run.heldAudio.current.complete
!run.recording.current.complete
) {
run.failActiveDictation(dictationId, new Error(phoneDictationFailedMessage()))
return
}
run.engine.current = { kind: 'host' }
run.showEngine('computer', true)
// A stop that began while the phone was still live is already processing; the computer must finish it too.
run.stopWhenStarted.current = finishing || run.status.current === 'processing'
try {
await startMobileDictationOnHost(run, client, dictationId, run.generation.current)
} catch (err) {
// The host start already released the mic and the dictation; keep what the phone heard.
const kept = run.keptTranscript.take(dictationId)
if (kept) {
run.onTranscript.current(kept)
}
run.reportError(err)
}
await replayMobileDictationToHost(run, client, dictationId, run.generation.current)
}
// Pick the engine before the mic opens, and arm the phone's recognizer so it hears the first word.
@@ -229,10 +183,9 @@ export function commitMobileDictationOnDevice(
() => void fallBackMobileDictationToHost(run, dictationId, false)
)
}
// This phone never replays to the computer, so it keeps no audio.
if (!fallback) {
run.audioPhase.current = 'closed'
}
// The recording keeps the phone's audio for a replay; nothing goes to the computer while the phone hears it.
run.audioPhase.current = 'closed'
run.heldAudio.current.clear()
run.applyStatus('recording')
run.showEngine('phone')
if (run.stopWhenStarted.current) {
@@ -0,0 +1,44 @@
import { errorMessage } from '../../../src/shared/error-message'
import {
isDictationSetupRequiredError,
isSpeechlessServerError
} from '../dictation/mobile-dictation-setup'
import type { MobileDictationEngine } from './mobile-dictation-device-engine'
import type { MobileDictationRecording } from './mobile-dictation-recording'
import type { MobileDictationRun } from './mobile-dictation-engine-run'
// Retry plays the recording to the computer, so it needs the whole recording and an engine that may use the computer (not This phone).
export function mobileDictationFailureRetryable(
recording: MobileDictationRecording,
engine: MobileDictationEngine
): boolean {
return recording.complete && recording.bytes > 0 && (engine.kind === 'host' || engine.fallback)
}
// A computer that needs setup or an update refuses every Retry the same way; those keep today's path (words inserted, setup sheet or toast).
function retryCanSucceed(err: unknown): boolean {
const message = errorMessage(err)
return !isDictationSetupRequiredError(message) && !isSpeechlessServerError(message)
}
// Never lose speech quietly: a retryable failure keeps the recording and the words so far for Retry; otherwise the words so far are inserted, as before.
export function concludeMobileDictationFailure(
run: Pick<
MobileDictationRun,
'recording' | 'engine' | 'keptTranscript' | 'onTranscript' | 'reportError'
>,
dictationId: string,
err: unknown,
retryable = mobileDictationFailureRetryable(run.recording.current, run.engine.current)
): void {
if (retryable && retryCanSucceed(err)) {
run.reportError(err, true)
return
}
run.recording.current.clear()
const kept = run.keptTranscript.take(dictationId)
if (kept) {
run.onTranscript.current(kept)
}
run.reportError(err)
}
@@ -0,0 +1,77 @@
import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload'
import type { DictationCapture } from '../platform/dictation-capture-contract'
import { isCurrentMobileDictationFinish } from './mobile-dictation-session-state'
import { finishDesktopDictation } from './mobile-dictation-desktop-start'
import {
finishMobileDictationOnDevice,
settleMobileDictation,
type MobileDictationRun
} from './mobile-dictation-engine-run'
// Stop listening and settle the dictation with its final text, from whichever engine heard it.
export async function finishMobileDictation(
run: MobileDictationRun,
capture: Pick<DictationCapture, 'end'>,
emptyTranscriptMessage: string
): Promise<void> {
const client = run.client.current
const dictationId = run.activeId.current
const engine = run.engine.current
if (!dictationId || (engine.kind === 'host' && !client)) {
return
}
// A replay's host start is still pending: finish once it commits and the kept audio has gone.
if (engine.kind === 'host' && run.audioPhase.current === 'holding') {
run.stopWhenStarted.current = true
if (run.status.current === 'recording') {
run.applyStatus('processing')
// Stop listening now; the tail joins the held audio the replay sends.
void capture.end()
}
return
}
const generation = run.generation.current + 1
run.generation.current = generation
run.applyStatus('processing')
const isCurrent = () =>
isCurrentMobileDictationFinish(
run.generation.current,
generation,
run.enabled.current,
run.activeId.current,
dictationId
)
try {
// Keep shutdown inside try for finally cleanup; accept its tail chunks and snapshot pending sends only after end settles, or finish can overtake the last audio.
await capture.end()
if (engine.kind === 'device') {
await finishMobileDictationOnDevice(
run,
engine,
dictationId,
emptyTranscriptMessage,
isCurrent
)
return
}
run.audioPhase.current = 'closed'
await Promise.allSettled(Array.from(run.pendingChunks.current))
if (!isCurrent() || !client) {
return
}
const sentSeconds = run.pendingAudioBudget.current.sentAudioSeconds
const finished = await finishDesktopDictation(client, dictationId, sentSeconds)
if (!isCurrent()) {
return
}
const transcript = rpcPayloadMember(finished, 'text')
settleMobileDictation(
run,
typeof transcript === 'string' ? transcript.trim() : '',
emptyTranscriptMessage
)
} catch (err) {
run.failActiveDictation(dictationId, err)
}
}
@@ -5,10 +5,10 @@ import {
} from '../../../src/shared/rpc-contract/speech-params'
import type { DictationCaptureChunk } from '../platform/dictation-capture-contract'
// Five minutes of 16 kHz PCM (9.6 MB): what an on-device dictation keeps to replay to the computer if the phone fails.
const MAX_HELD_AUDIO_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * 300
// Five minutes of 16 kHz PCM (9.6 MB): the longest recording kept for a replay to the computer.
export const MAX_HELD_AUDIO_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * 300
// Speech not sent yet, in order: held between the tap and the host's start reply, while every send slot is busy, and for a whole on-device dictation.
// Speech not sent yet, in order: held between the tap and the host's start reply, while every send slot is busy, and for a replay of the recording.
export class MobileDictationHeldAudio {
private held: DictationCaptureChunk[] = []
private heldBytes = 0
@@ -0,0 +1,69 @@
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
import { dictationSessionErrorMessage } from '../dictation/mobile-dictation-setup'
import {
settledDesktopDictationWork,
startMobileDictationDesktopSession
} from './mobile-dictation-desktop-start'
import { concludeMobileDictationFailure } from './mobile-dictation-failure'
import type { MobileDictationRun } from './mobile-dictation-engine-run'
export async function startMobileDictationOnHost(
run: MobileDictationRun,
client: RpcClient,
dictationId: string,
generation: number,
// A replay (fallback or Retry) keeps the words so far on screen if its start fails.
replay = false
): Promise<void> {
await settledDesktopDictationWork(client)
if (run.generation.current !== generation) {
return
}
await startMobileDictationDesktopSession({
client,
dictationId,
generation,
getCurrentGeneration: () => run.generation.current,
getEnabled: () => run.enabled.current,
getActiveId: () => run.activeId.current,
clearActiveId: (id) => {
if (run.activeId.current === id) {
run.activeId.current = null
}
},
setIdle: () => run.applyStatus('idle', replay),
commitRecordingStart: (hostDedupesResends) => {
run.audioPhase.current = 'sending'
run.pendingChunks.current.clear()
run.pendingAudioBudget.current.reset(hostDedupesResends)
// A phone dictation stopped or failed at finish is already processing; it never records again.
if (run.status.current !== 'processing') {
run.applyStatus('recording')
}
run.showEngine('computer')
run.sendHeldAudio.current()
if (run.stopWhenStarted.current) {
void run.finish.current(dictationSessionErrorMessage('no-speech'))
}
},
rollbackRecordingStart: run.closeDictationAudio
})
}
// Play the kept recording to the computer as this dictation's held audio (seq from 0); audio heard after joins it in order until the start commits.
// Fallback and Retry both end here; a failed start keeps the recording for Retry.
export async function replayMobileDictationToHost(
run: MobileDictationRun,
client: RpcClient,
dictationId: string,
generation: number
): Promise<void> {
run.engine.current = { kind: 'host' }
run.recording.current.replayInto(run.heldAudio.current)
run.audioPhase.current = 'holding'
try {
await startMobileDictationOnHost(run, client, dictationId, generation, true)
} catch (err) {
concludeMobileDictationFailure(run, dictationId, err)
}
}
@@ -0,0 +1,47 @@
import type { DictationCaptureChunk } from '../platform/dictation-capture-contract'
import { MAX_HELD_AUDIO_BYTES, type MobileDictationHeldAudio } from './mobile-dictation-held-audio'
// Everything one dictation heard, from the tap, whichever engine hears it: replayed to the computer when the phone fails, and kept after a failure for Retry.
export class MobileDictationRecording {
private chunks: DictationCaptureChunk[] = []
private keptBytes = 0
private incomplete = false
keep(chunk: DictationCaptureChunk): void {
if (this.incomplete) {
return
}
// Past the cap, or with audio the page dropped, a replay would lose speech (and the send refuses it), so keep nothing and say it is incomplete.
if (chunk.droppedBytes > 0 || this.keptBytes + chunk.data.byteLength > MAX_HELD_AUDIO_BYTES) {
this.chunks = []
this.keptBytes = 0
this.incomplete = true
return
}
this.chunks.push(chunk)
this.keptBytes += chunk.data.byteLength
}
// False once speech was lost; the recording no longer covers the whole dictation.
get complete(): boolean {
return !this.incomplete
}
get bytes(): number {
return this.keptBytes
}
// Queue a copy for sending; the recording itself stays, so a failed replay can be retried.
replayInto(held: MobileDictationHeldAudio): void {
held.clear()
for (const chunk of this.chunks) {
held.hold(chunk)
}
}
clear(): void {
this.chunks = []
this.keptBytes = 0
this.incomplete = false
}
}
@@ -0,0 +1,166 @@
// A computer dictation that fails mid-session keeps its whole recording; Retry plays all of it to the computer as a new dictation.
import { createElement } from 'react'
import { act, create } from 'react-test-renderer'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
createFakeRpcClient,
type FakeRpcClient,
type SentRequest
} from '../mobile-web-shell/bridge-host-test-fakes'
import type {
DictationCapture,
DictationCaptureChunk
} from '../platform/dictation-capture-contract'
type Seam = {
chunkHandlers: Set<(chunk: DictationCaptureChunk) => void>
tail: Uint8Array | null
}
const seam = vi.hoisted((): Seam => ({ chunkHandlers: new Set(), tail: null }))
vi.mock('react-native', () => ({
AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) },
Platform: { OS: 'ios' }
}))
// Keep capture identity stable like both real seams; a fresh object on render would rerun teardown and cancel dictation.
vi.mock('../platform/dictation-capture', () => {
const capture: DictationCapture = {
open: async () => ({ ok: true }),
begin: () => true,
end: async () => {
// Page capture reads once more here and may deliver audio.
const tail = seam.tail
seam.tail = null
if (tail !== null) {
for (const handler of seam.chunkHandlers) {
handler({ data: tail, droppedBytes: 0 })
}
}
},
release: () => {},
onChunk: (handler) => {
seam.chunkHandlers.add(handler)
return {
remove: () => {
seam.chunkHandlers.delete(handler)
}
}
},
onInterruption: () => ({ remove: () => {} })
}
return { useDictationCapture: () => capture }
})
import { useMobileDictation, type UseMobileDictationResult } from './use-mobile-dictation'
type Reply = (request: SentRequest) => 'hold' | { text?: string } | Error
// Answer every request; one reply can unblock the next.
async function pump(rpc: FakeRpcClient, sent: SentRequest[], reply: Reply): Promise<void> {
for (let round = 0; round < 30; round += 1) {
for (const request of rpc.requests.splice(0)) {
sent.push(request)
const answer = reply(request)
if (answer === 'hold') {
continue
}
if (answer instanceof Error) {
request.reject(answer)
} else {
request.resolve({ id: 'desktop', ok: true, result: answer })
}
}
await Promise.resolve()
await Promise.resolve()
}
}
function paramsOf(request: SentRequest): Record<string, unknown> {
const params = request.args[1]
return typeof params === 'object' && params !== null ? { ...params } : {}
}
function deliver(bytes: number[]): void {
for (const handler of seam.chunkHandlers) {
handler({ data: Uint8Array.from(bytes), droppedBytes: 0 })
}
}
const held: { dictation: UseMobileDictationResult | null } = { dictation: null }
function dictation(): UseMobileDictationResult {
const current = held.dictation
if (current === null) {
throw new Error('nothing mounted')
}
return current
}
beforeEach(() => {
seam.chunkHandlers.clear()
seam.tail = null
held.dictation = null
})
describe('a computer dictation whose finish fails', () => {
it('keeps the recording, and Retry sends all of it again and inserts the words', async () => {
const rpc = createFakeRpcClient()
const sent: SentRequest[] = []
const inserted: string[] = []
const errors: { message: string; kept: boolean }[] = []
function Probe(): null {
held.dictation = useMobileDictation({
client: rpc,
enabled: true,
onTranscript: (text) => inserted.push(text),
onError: (error, outcome) => errors.push({ message: error.message, kept: outcome.kept })
})
return null
}
act(() => {
create(createElement(Probe))
})
const firstTry: Reply = (request) =>
request.method === 'speech.dictation.finish' ? new Error('connection lost') : {}
await act(async () => {
const started = dictation().start()
await pump(rpc, sent, firstTry)
await started
})
await act(async () => {
deliver([1, 2, 3, 4])
await pump(rpc, sent, firstTry)
})
seam.tail = Uint8Array.from([5, 6])
await act(async () => {
const stopped = dictation().stop()
await pump(rpc, sent, firstTry)
await stopped
})
expect(inserted).toEqual([])
expect(errors).toEqual([{ message: 'connection lost', kept: true }])
expect(dictation().failure).toEqual({ message: 'connection lost', offline: false })
const firstId = paramsOf(sent[0]).dictationId
sent.length = 0
await act(async () => {
const retried = dictation().retry()
await pump(rpc, sent, (request) =>
request.method === 'speech.dictation.finish' ? { text: 'all of it' } : {}
)
await retried
})
expect(sent.map((request) => request.method)).toEqual([
'speech.dictation.start',
'speech.dictation.chunk',
'speech.dictation.finish'
])
expect(paramsOf(sent[0]).dictationId).not.toBe(firstId)
// The whole recording, from the first byte, as seq 0 of the new dictation.
expect(paramsOf(sent[1])).toMatchObject({ audioBase64: 'AQIDBAUG', seq: 0 })
expect(inserted).toEqual(['all of it'])
expect(dictation().failure).toBeNull()
})
})
@@ -0,0 +1,42 @@
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
import { createMobileDictationId } from './mobile-dictation-session-state'
import {
concludeMobileDictationFailure,
mobileDictationFailureRetryable
} from './mobile-dictation-failure'
import {
releaseMobileDictationEngine,
type MobileDictationRun
} from './mobile-dictation-engine-run'
import { replayMobileDictationToHost } from './mobile-dictation-host-start'
// A dictation failed after it began hearing: stop it, and keep its recording for Retry when the computer can hear it again.
export function failActiveMobileDictation(
run: MobileDictationRun,
dictationId: string,
err: unknown
): void {
if (run.activeId.current !== dictationId) {
return
}
run.activeId.current = null
const retryable = mobileDictationFailureRetryable(run.recording.current, run.engine.current)
run.closeDictationAudio()
void releaseMobileDictationEngine(run, run.client.current, dictationId)
concludeMobileDictationFailure(run, dictationId, err, retryable)
}
// Retry: the kept recording goes to the computer as a new dictation (seq from 0), already stopped, so it finishes as soon as it has been sent.
export async function retryMobileDictationOnHost(
run: MobileDictationRun,
client: RpcClient
): Promise<void> {
const generation = run.generation.current + 1
run.generation.current = generation
const dictationId = createMobileDictationId()
run.activeId.current = dictationId
run.stopWhenStarted.current = true
run.applyStatus('processing', true)
run.showEngine('computer')
await replayMobileDictationToHost(run, client, dictationId, generation)
}
@@ -9,11 +9,15 @@ export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' |
// Who is transcribing, for the icon by the timer; movedAt marks when Automatic moved from the phone to the computer.
export type MobileDictationEngineShown = { kind: 'phone' | 'computer'; movedAt: number | null }
// offline: the computer was out of reach when it failed, which is the reason the user needs.
export type MobileDictationKeptFailure = { message: string; offline: boolean }
export type UseMobileDictationOptions = {
client: RpcClient | null
enabled: boolean
onTranscript: (text: string) => void
onError?: (error: Error) => void
// kept: the recording is kept for Retry, so the failure shows in place rather than as a toast.
onError?: (error: Error, outcome: { kept: boolean }) => void
// The computer's filler switch, for text the phone recognizes itself; unknown means on.
removeFillerWords?: boolean
// The host's cloud upload is full; the dictation stops and inserts what was kept.
@@ -28,10 +32,13 @@ export type UseMobileDictationResult = {
// Empty until a newer host reports words; older hosts never do.
transcriptSoFar: string
error: string | null
// A failure whose recording is kept: Retry plays it to the computer again, cancel discards it.
failure: MobileDictationKeptFailure | null
// Null until recording starts on an engine.
engine: MobileDictationEngineShown | null
start: () => Promise<void>
stop: () => Promise<void>
retry: () => Promise<void>
cancel: () => Promise<void>
meter: MobileDictationMeter
}
+71 -107
View File
@@ -13,19 +13,19 @@ import {
type MobileDictationAudioPhase
} from './mobile-dictation-audio-chunk'
import { MobileDictationHeldAudio } from './mobile-dictation-held-audio'
import { isCurrentMobileDictationFinish } from './mobile-dictation-session-state'
import { finishDesktopDictation } from './mobile-dictation-desktop-start'
import { MobileDictationRecording } from './mobile-dictation-recording'
import { failActiveMobileDictation, retryMobileDictationOnHost } from './mobile-dictation-retry'
import { mobileDictationFailureRetryable } from './mobile-dictation-failure'
import { finishMobileDictation } from './mobile-dictation-finish'
import type { MobileDictationEngine } from './mobile-dictation-device-engine'
import {
finishMobileDictationOnDevice,
releaseMobileDictationEngine,
settleMobileDictation,
startMobileDictation,
type MobileDictationRun
} from './mobile-dictation-engine-run'
import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload'
import type {
DictationStatus,
MobileDictationKeptFailure,
UseMobileDictationOptions,
UseMobileDictationResult
} from './mobile-dictation-session-state'
@@ -40,6 +40,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
const speech = useOnDeviceSpeech(removeFillerWords)
const [status, setStatus] = useState<DictationStatus>('idle')
const [error, setError] = useState<string | null>(null)
// A failure whose recording is kept for Retry; cleared by Retry, discard and the next dictation.
const [failure, setFailure] = useState<MobileDictationKeptFailure | null>(null)
const failureRef = useRef(failure)
const [transcriptSoFar, setTranscriptSoFar, keptTranscript] = useMobileDictationTranscriptSoFar()
const {
shown: shownEngine,
@@ -61,6 +64,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget())
const audioPhaseRef = useRef<MobileDictationAudioPhase>('closed')
const heldAudioRef = useRef(new MobileDictationHeldAudio())
const recordingRef = useRef(new MobileDictationRecording())
const sendHeldAudioRef = useRef(() => {})
const stopWhenStartedRef = useRef(false)
const generationRef = useRef(0)
@@ -77,27 +81,36 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
}, [client, speech, enabled, onTranscript, onError, onAudioLimitReached])
const applyStatus = useCallback(
(next: DictationStatus) => {
// Kept through processing so the text so far stays in place until the final transcript replaces it in the same render.
(next: DictationStatus, keepText = next === 'recording' || next === 'processing') => {
statusRef.current = next
setStatus(next)
// Kept through processing so the text so far stays in place until the final transcript replaces it in the same render.
const live = next === 'recording' || next === 'processing'
setTranscriptSoFar((text) => (live ? text : ''))
if (!live) {
setTranscriptSoFar((text) => (keepText ? text : ''))
if (next !== 'recording' && next !== 'processing') {
clearEngine()
}
},
[clearEngine]
)
const setKeptFailure = useCallback((next: MobileDictationKeptFailure | null) => {
failureRef.current = next
setFailure(next)
}, [])
const reportError = useCallback(
(err: unknown) => {
(err: unknown, kept = false) => {
const normalized = err instanceof Error ? err : new Error(String(err))
setError(normalized.message)
applyStatus('error')
onErrorRef.current?.(normalized)
applyStatus('error', kept)
setKeptFailure(
kept
? { message: normalized.message, offline: clientRef.current?.getState() !== 'connected' }
: null
)
onErrorRef.current?.(normalized, { kept })
},
[applyStatus]
[applyStatus, setKeptFailure]
)
const closeDictationAudio = useCallback(() => {
@@ -113,29 +126,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
}
}, [capture])
const failActiveDictation = useCallback(
(dictationId: string, err: unknown) => {
if (activeIdRef.current !== dictationId) {
return
}
activeIdRef.current = null
closeDictationAudio()
void releaseMobileDictationEngine(
{ engine: engineRef, speech: speechRef },
clientRef.current,
dictationId
)
const kept = keptTranscript.take(dictationId)
if (kept) {
onTranscriptRef.current(kept)
}
reportError(err)
},
[closeDictationAudio, reportError]
)
const run = useMemo<MobileDictationRun>(
() => ({
const run = useMemo<MobileDictationRun>(() => {
const created: MobileDictationRun = {
status: statusRef,
activeId: activeIdRef,
client: clientRef,
@@ -148,6 +140,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
pendingChunks: pendingChunksRef,
pendingAudioBudget: pendingAudioBudgetRef,
heldAudio: heldAudioRef,
recording: recordingRef,
sendHeldAudio: sendHeldAudioRef,
finish: finishRef,
onTranscript: onTranscriptRef,
@@ -157,77 +150,17 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
showEngine,
closeDictationAudio,
reportError,
failActiveDictation
}),
[applyStatus, closeDictationAudio, failActiveDictation, keptTranscript, reportError, showEngine]
)
failActiveDictation: (dictationId, err) =>
failActiveMobileDictation(created, dictationId, err)
}
return created
}, [applyStatus, closeDictationAudio, keptTranscript, reportError, showEngine])
useMobileDictationAudioChunks(capture, run)
const finishDictation = useCallback(
async (emptyTranscriptMessage: string) => {
const client = clientRef.current
const dictationId = activeIdRef.current
const engine = engineRef.current
if (!dictationId || (engine.kind === 'host' && !client)) {
return
}
// A fallback's host start is still pending: finish once it commits and the kept audio has gone.
if (engine.kind === 'host' && audioPhaseRef.current === 'holding') {
stopWhenStartedRef.current = true
if (statusRef.current === 'recording') {
applyStatus('processing')
// Stop listening now; the tail joins the held audio the replay sends.
void capture.end()
}
return
}
const generation = generationRef.current + 1
generationRef.current = generation
applyStatus('processing')
const isCurrent = () =>
isCurrentMobileDictationFinish(
generationRef.current,
generation,
enabledRef.current,
activeIdRef.current,
dictationId
)
try {
// Keep shutdown inside try for finally cleanup; accept its tail chunks and snapshot pending sends only after end settles, or finish can overtake the last audio.
await capture.end()
if (engine.kind === 'device') {
await finishMobileDictationOnDevice(
run,
engine,
dictationId,
emptyTranscriptMessage,
isCurrent
)
return
}
audioPhaseRef.current = 'closed'
await Promise.allSettled(Array.from(pendingChunksRef.current))
if (!isCurrent() || !client) {
return
}
const sentSeconds = pendingAudioBudgetRef.current.sentAudioSeconds
const finished = await finishDesktopDictation(client, dictationId, sentSeconds)
if (!isCurrent()) {
return
}
const transcript = rpcPayloadMember(finished, 'text')
settleMobileDictation(
run,
typeof transcript === 'string' ? transcript.trim() : '',
emptyTranscriptMessage
)
} catch (err) {
failActiveDictation(dictationId, err)
}
},
[applyStatus, capture, failActiveDictation, run]
(emptyTranscriptMessage: string) => finishMobileDictation(run, capture, emptyTranscriptMessage),
[capture, run]
)
useLayoutEffect(() => {
finishRef.current = finishDictation
@@ -247,8 +180,21 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
return
}
setError(null)
// A new dictation replaces a failed one's kept recording.
setKeptFailure(null)
recordingRef.current.clear()
await startMobileDictation(run, capture)
}, [capture, run])
}, [capture, run, setKeptFailure])
const retry = useCallback(async () => {
const client = clientRef.current
if (!failureRef.current || !client || activeIdRef.current || !enabledRef.current) {
return
}
setError(null)
setKeptFailure(null)
await retryMobileDictationOnHost(run, client)
}, [run, setKeptFailure])
// Only the user’s own cancel (reason:null) ends silently; other interruptions call onError so the tap has a visible outcome.
const abandonDictation = useCallback(
@@ -256,11 +202,26 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
const client = clientRef.current
const dictationId = activeIdRef.current
const wasUnderway = statusRef.current === 'starting' || statusRef.current === 'recording'
// Only the user's discard drops a kept recording; an interruption while it waits leaves it for Retry.
if (reason !== null && dictationId === null && failureRef.current) {
return
}
// An interruption keeps a recording the computer can hear again, even mid-finish; the user's cancel never does.
const kept =
reason !== null &&
dictationId !== null &&
mobileDictationFailureRetryable(recordingRef.current, engineRef.current)
if (!kept) {
setKeptFailure(null)
recordingRef.current.clear()
}
generationRef.current += 1
activeIdRef.current = null
closeDictationAudio()
// Settle before the host's cancel reply: a host that went quiet must not hold the mic button in its old state.
if (reason !== null && wasUnderway) {
if (kept) {
reportError(new Error(reason), true)
} else if (reason !== null && wasUnderway) {
reportError(new Error(reason))
} else {
applyStatus('idle')
@@ -268,7 +229,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
}
await releaseMobileDictationEngine(run, client, dictationId)
},
[applyStatus, closeDictationAudio, reportError, run]
[applyStatus, closeDictationAudio, reportError, run, setKeptFailure]
)
const cancel = useCallback(() => abandonDictation(null), [abandonDictation])
@@ -295,6 +256,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
generationRef.current += 1
activeIdRef.current = null
closeDictationAudio()
recordingRef.current.clear()
capture.release()
void releaseMobileDictationEngine(run, clientRef.current, dictationId)
}
@@ -307,8 +269,10 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
isProcessing: status === 'processing',
transcriptSoFar,
error,
failure,
engine: shownEngine,
start,
retry,
stop,
cancel,
meter: useMobileDictationMeter(capture, status === 'recording')
+11 -1
View File
@@ -1317,13 +1317,23 @@
"durationSeconds_one": "{{count}} Sekunde",
"durationSeconds_other": "{{count}} Sekunden",
"durationMinutes_one": "{{count}} Minute",
"durationMinutes_other": "{{count}} Minuten"
"durationMinutes_other": "{{count}} Minuten",
"discard": "Aufnahme verwerfen",
"retry": "Diktat wiederholen",
"retryShort": "Wiederholen"
},
"status": {
"finishingOn": "Wird auf {{computer}} abgeschlossen",
"finishingOnComputer": "Wird auf dem Computer abgeschlossen",
"movedTo": "Zu {{computer}} gewechselt",
"movedToComputer": "Zum Computer gewechselt"
},
"failure": {
"connectionLost": "Verbindung unterbrochen",
"connectionSlow": "Verbindung zu langsam",
"phoneStopped": "Auf diesem Telefon gestoppt",
"stopped": "Diktat gestoppt",
"recordingKept": "Aufnahme behalten"
}
},
"files": {
+11 -1
View File
@@ -534,13 +534,23 @@
"meterOnPhone": "Recording, {{duration}}, on this phone",
"meterOn": "Recording, {{duration}}, on {{computer}}",
"meterOnComputer": "Recording, {{duration}}, on the computer",
"meter": "Recording, {{duration}}"
"meter": "Recording, {{duration}}",
"discard": "Discard recording",
"retry": "Retry dictation",
"retryShort": "Retry"
},
"status": {
"finishingOn": "Finishing on {{computer}}",
"finishingOnComputer": "Finishing on the computer",
"movedTo": "Moved to {{computer}}",
"movedToComputer": "Moved to the computer"
},
"failure": {
"recordingKept": "Recording kept",
"connectionLost": "Connection lost",
"connectionSlow": "Connection too slow",
"phoneStopped": "Stopped on this phone",
"stopped": "Dictation stopped"
}
},
"home": {
+11 -1
View File
@@ -1358,13 +1358,23 @@
"durationSeconds_many": "{{count}} segundos",
"durationMinutes_one": "{{count}} minuto",
"durationMinutes_other": "{{count}} minutos",
"durationMinutes_many": "{{count}} minutos"
"durationMinutes_many": "{{count}} minutos",
"discard": "Descartar grabación",
"retry": "Reintentar dictado",
"retryShort": "Reintentar"
},
"status": {
"finishingOn": "Terminando en {{computer}}",
"finishingOnComputer": "Terminando en el ordenador",
"movedTo": "Pasado a {{computer}}",
"movedToComputer": "Pasado al ordenador"
},
"failure": {
"connectionLost": "Conexión perdida",
"connectionSlow": "Conexión demasiado lenta",
"phoneStopped": "Detenido en este teléfono",
"stopped": "Dictado detenido",
"recordingKept": "Grabación guardada"
}
},
"files": {
+11 -1
View File
@@ -1358,13 +1358,23 @@
"durationSeconds_many": "{{count}} secondes",
"durationMinutes_one": "{{count}} minute",
"durationMinutes_other": "{{count}} minutes",
"durationMinutes_many": "{{count}} minutes"
"durationMinutes_many": "{{count}} minutes",
"discard": "Supprimer l’enregistrement",
"retry": "Relancer la dictée",
"retryShort": "Réessayer"
},
"status": {
"finishingOn": "Finalisation sur {{computer}}",
"finishingOnComputer": "Finalisation sur l’ordinateur",
"movedTo": "Passé sur {{computer}}",
"movedToComputer": "Passé sur l’ordinateur"
},
"failure": {
"connectionLost": "Connexion perdue",
"connectionSlow": "Connexion trop lente",
"phoneStopped": "Arrêté sur ce téléphone",
"stopped": "Dictée arrêtée",
"recordingKept": "Enregistrement conservé"
}
},
"files": {
+11 -1
View File
@@ -1313,13 +1313,23 @@
"durationSeconds_one": "{{count}} 秒",
"durationSeconds_other": "{{count}} 秒",
"durationMinutes_one": "{{count}} 分",
"durationMinutes_other": "{{count}} 分"
"durationMinutes_other": "{{count}} 分",
"discard": "録音を破棄",
"retry": "音声入力を再試行",
"retryShort": "再試行"
},
"status": {
"finishingOn": "{{computer}} で仕上げています",
"finishingOnComputer": "コンピュータで仕上げています",
"movedTo": "{{computer}} に切り替えました",
"movedToComputer": "コンピュータに切り替えました"
},
"failure": {
"connectionLost": "接続が切れました",
"connectionSlow": "接続が遅すぎます",
"phoneStopped": "このスマートフォンで停止しました",
"stopped": "音声入力が停止しました",
"recordingKept": "録音は保持されています"
}
},
"files": {
+11 -1
View File
@@ -1313,13 +1313,23 @@
"durationSeconds_one": "{{count}}초",
"durationSeconds_other": "{{count}}초",
"durationMinutes_one": "{{count}}분",
"durationMinutes_other": "{{count}}분"
"durationMinutes_other": "{{count}}분",
"discard": "녹음 삭제",
"retry": "받아쓰기 다시 시도",
"retryShort": "다시 시도"
},
"status": {
"finishingOn": "{{computer}}에서 마무리 중",
"finishingOnComputer": "컴퓨터에서 마무리 중",
"movedTo": "{{computer}}(으)로 전환됨",
"movedToComputer": "컴퓨터로 전환됨"
},
"failure": {
"connectionLost": "연결 끊김",
"connectionSlow": "연결이 너무 느림",
"phoneStopped": "이 휴대폰에서 중지됨",
"stopped": "받아쓰기 중지됨",
"recordingKept": "녹음 보관됨"
}
},
"files": {
+11 -1
View File
@@ -1358,13 +1358,23 @@
"durationSeconds_many": "{{count}} segundos",
"durationMinutes_one": "{{count}} minuto",
"durationMinutes_other": "{{count}} minutos",
"durationMinutes_many": "{{count}} minutos"
"durationMinutes_many": "{{count}} minutos",
"discard": "Descartar gravação",
"retry": "Tentar ditado novamente",
"retryShort": "Tentar novamente"
},
"status": {
"finishingOn": "Finalizando em {{computer}}",
"finishingOnComputer": "Finalizando no computador",
"movedTo": "Passou para {{computer}}",
"movedToComputer": "Passou para o computador"
},
"failure": {
"connectionLost": "Conexão perdida",
"connectionSlow": "Conexão muito lenta",
"phoneStopped": "Parou neste telefone",
"stopped": "Ditado interrompido",
"recordingKept": "Gravação mantida"
}
},
"files": {
+11 -1
View File
@@ -1399,13 +1399,23 @@
"durationMinutes_one": "{{count}} минута",
"durationMinutes_other": "{{count}} минуты",
"durationMinutes_few": "{{count}} минуты",
"durationMinutes_many": "{{count}} минут"
"durationMinutes_many": "{{count}} минут",
"discard": "Удалить запись",
"retry": "Повторить диктовку",
"retryShort": "Повторить"
},
"status": {
"finishingOn": "Завершается на {{computer}}",
"finishingOnComputer": "Завершается на компьютере",
"movedTo": "Перешло на {{computer}}",
"movedToComputer": "Перешло на компьютер"
},
"failure": {
"connectionLost": "Соединение потеряно",
"connectionSlow": "Соединение слишком медленное",
"phoneStopped": "Остановлено на этом телефоне",
"stopped": "Диктовка остановлена",
"recordingKept": "Запись сохранена"
}
},
"files": {
+11 -1
View File
@@ -1317,13 +1317,23 @@
"durationSeconds_one": "{{count}} saniye",
"durationSeconds_other": "{{count}} saniye",
"durationMinutes_one": "{{count}} dakika",
"durationMinutes_other": "{{count}} dakika"
"durationMinutes_other": "{{count}} dakika",
"discard": "Kaydı sil",
"retry": "Dikteyi yeniden dene",
"retryShort": "Yeniden dene"
},
"status": {
"finishingOn": "{{computer}} üzerinde tamamlanıyor",
"finishingOnComputer": "Bilgisayarda tamamlanıyor",
"movedTo": "{{computer}} cihazına geçildi",
"movedToComputer": "Bilgisayara geçildi"
},
"failure": {
"connectionLost": "Bağlantı kesildi",
"connectionSlow": "Bağlantı çok yavaş",
"phoneStopped": "Bu telefonda durdu",
"stopped": "Dikte durdu",
"recordingKept": "Kayıt saklandı"
}
},
"files": {
+11 -1
View File
@@ -1313,13 +1313,23 @@
"durationSeconds_one": "{{count}} 秒",
"durationSeconds_other": "{{count}} 秒",
"durationMinutes_one": "{{count}} 分鐘",
"durationMinutes_other": "{{count}} 分鐘"
"durationMinutes_other": "{{count}} 分鐘",
"discard": "捨棄錄音",
"retry": "重試聽寫",
"retryShort": "重試"
},
"status": {
"finishingOn": "正在 {{computer}} 上完成",
"finishingOnComputer": "正在電腦上完成",
"movedTo": "已轉到 {{computer}}",
"movedToComputer": "已轉到電腦"
},
"failure": {
"connectionLost": "連線中斷",
"connectionSlow": "連線太慢",
"phoneStopped": "已在此手機上停止",
"stopped": "聽寫已停止",
"recordingKept": "錄音已保留"
}
},
"files": {
+11 -1
View File
@@ -1313,13 +1313,23 @@
"durationSeconds_one": "{{count}} 秒",
"durationSeconds_other": "{{count}} 秒",
"durationMinutes_one": "{{count}} 分钟",
"durationMinutes_other": "{{count}} 分钟"
"durationMinutes_other": "{{count}} 分钟",
"discard": "丢弃录音",
"retry": "重试听写",
"retryShort": "重试"
},
"status": {
"finishingOn": "正在 {{computer}} 上完成",
"finishingOnComputer": "正在电脑上完成",
"movedTo": "已转到 {{computer}}",
"movedToComputer": "已转到电脑"
},
"failure": {
"connectionLost": "连接已断开",
"connectionSlow": "连接太慢",
"phoneStopped": "已在此手机上停止",
"stopped": "听写已停止",
"recordingKept": "录音已保留"
}
},
"files": {
@@ -345,7 +345,7 @@ export function MobileNativeChatComposer({
/>
<View style={inputStyles.actionRow} testID="native-chat-composer-actions">
<MobileNativeChatDictationLeading
{...{ micActive, micStarting, micProcessing, cancelRef }}
{...{ micActive, micStarting, micProcessing, cancelRef, trailing: contextRing.ring }}
>
<MobileNativeChatComposerAttachButton
onAttachImage={onAttachImage}
@@ -373,7 +373,6 @@ export function MobileNativeChatComposer({
<MobileNativeChatComposerPlanChip options={sessionOptions} inputRef={fieldRef} />
{goal.chip}
</MobileNativeChatDictationLeading>
{micActive || micProcessing ? null : contextRing.ring}
{onMicPress ? (
<MobileNativeChatComposerMic
{...{ onMicPress, micActive, micStarting, micProcessing, dictationMode }}
@@ -2,7 +2,10 @@ import { Pressable, type View } from 'react-native'
import { useContext, type RefObject } from 'react'
import { MobileDictationLiveContext } from '../../../dictation/mobile-dictation-live-context'
import { dictationControlsCopy } from '../../../dictation/mobile-dictation-controls-copy'
import { MobileDictationMicIcon } from '../../../dictation/MobileDictationLiveControls'
import {
MobileDictationMicIcon,
MobileDictationRetryButton
} from '../../../dictation/MobileDictationLiveControls'
import { useMobileDictationCancelDrop } from '../../../dictation/use-mobile-dictation-cancel-drop'
import { useMobileTranslation } from '../../../i18n/use-mobile-translation'
import { keepHeldPressThroughLongPress } from '../../../platform/held-press-long-press'
@@ -43,6 +46,9 @@ export function MobileNativeChatComposerMic({
useMobileTranslation()
const live = useContext(MobileDictationLiveContext)
const release = useMobileDictationCancelDrop(cancelRef, live?.cancel, onMicPressOut)
if (live?.failureReason != null) {
return <MobileDictationRetryButton onRetry={live.retry} labelled />
}
return (
<Pressable
accessibilityRole="button"
@@ -16,6 +16,7 @@ import {
} from './MobileNativeChatSendButton'
import type { MobileNativeChatSteerHead } from './use-mobile-native-chat-composer-tray'
import { useMobileNativeChatSteerChordPreview } from './use-mobile-native-chat-steer-chord-preview'
import { useMobileNativeChatDictationFailed } from './MobileNativeChatDictationLeading'
/** Send's slot outside a queued edit: Send, Stop where Send can't deliver, or what a held Cmd/Ctrl would do. */
export function MobileNativeChatComposerSendSlot({
@@ -51,7 +52,8 @@ export function MobileNativeChatComposerSendSlot({
attaching: boolean
stop: { onStop?: () => void; held: boolean } | null
onSend: (intent?: MobileNativeChatSendIntent, keepFocus?: boolean) => void
}): React.JSX.Element {
}): React.JSX.Element | null {
const dictationFailed = useMobileNativeChatDictationFailed()
const steerChord = useMobileNativeChatSteerChordPreview({
fieldRef,
hasDraft: value.trim() !== '' || attachmentCount > 0,
@@ -83,6 +85,10 @@ export function MobileNativeChatComposerSendSlot({
draft: value,
attachmentCount
})
// A dictation kept for Retry takes Send's room; a running agent keeps its Stop.
if (dictationFailed && !(showsStop && stop)) {
return null
}
if (steerChord.preview && (steerChord.preview === 'steer-queued' || canSend)) {
// As desktop, the preview takes Send's slot at once, with no ease.
return (
@@ -3,28 +3,45 @@ import { View } from 'react-native'
import { MobileDictationLiveContext } from '../../../dictation/mobile-dictation-live-context'
import {
MobileDictationCancelButton,
MobileDictationFailureNote,
MobileDictationFinishingNote,
MobileDictationRecordingMeter
} from '../../../dictation/MobileDictationLiveControls'
import { mobileNativeChatComposerStyles as styles } from './mobile-native-chat-composer-styles'
// While dictating, the row's leading side swaps attach and the pickers for ✕ and the meter, so the composer keeps its height.
// While dictating, the row's leading side swaps attach and the pickers for ✕ and the meter (or a kept failure), so the composer keeps its height.
export function MobileNativeChatDictationLeading({
micActive,
micStarting = false,
micProcessing = false,
cancelRef,
trailing,
children
}: {
micActive: boolean
micStarting?: boolean
micProcessing?: boolean
cancelRef: RefObject<View | null>
// The context ring after the leading controls; it steps aside with them.
trailing: ReactNode
children: ReactNode
}): React.JSX.Element {
const live = useContext(MobileDictationLiveContext)
if (live?.failureReason != null) {
return (
<View style={[styles.leadingControls, styles.dictationLeading]}>
<MobileDictationCancelButton onCancel={live.cancel} discards />
<MobileDictationFailureNote reason={live.failureReason} />
</View>
)
}
if (!live || (!micActive && !micProcessing)) {
return <View style={styles.leadingControls}>{children}</View>
return (
<>
<View style={styles.leadingControls}>{children}</View>
{trailing}
</>
)
}
return (
<View style={[styles.leadingControls, styles.dictationLeading]}>
@@ -42,3 +59,8 @@ export function MobileNativeChatDictationLeading({
</View>
)
}
// A dictation failed with its recording kept: Send gives its room to Retry.
export function useMobileNativeChatDictationFailed(): boolean {
return useContext(MobileDictationLiveContext)?.failureReason != null
}
@@ -166,7 +166,12 @@ export function useMobileSessionNativeChatDictation(
}
})().catch(() => keepUndeliveredDictation(text))
},
onError: (err) => {
onError: (err, { kept }) => {
// A kept failure shows in place with Retry, which still needs the dictation's target.
if (kept) {
triggerError()
return
}
dictationRouteContextRef.current = null
reportDictationFailure(err)
},
@@ -1,4 +1,4 @@
import type { RefObject } from 'react'
import { useContext, type RefObject } from 'react'
import {
ActivityIndicator,
Pressable,
@@ -17,7 +17,11 @@ import { useMobileColorScheme } from '../../theme/use-mobile-color-scheme'
import { keepHeldPressThroughLongPress } from '../../platform/held-press-long-press'
import { dictationControlsCopy } from '../../dictation/mobile-dictation-controls-copy'
import { useMobileDictationCancelDrop } from '../../dictation/use-mobile-dictation-cancel-drop'
import { MobileDictationMicIcon } from '../../dictation/MobileDictationLiveControls'
import {
MobileDictationMicIcon,
MobileDictationRetryButton
} from '../../dictation/MobileDictationLiveControls'
import { MobileDictationLiveContext } from '../../dictation/mobile-dictation-live-context'
type DictationState = {
readonly isStarting: boolean
@@ -54,11 +58,20 @@ type MobileTerminalInputActionsProps = MobileTerminalDictationButtonProps &
export function MobileTerminalInputActions(props: MobileTerminalInputActionsProps) {
const { isStarting, isRecording, isProcessing } = props.dictation
// Attach steps aside while dictating so the words get the field's room.
const live = useContext(MobileDictationLiveContext)
// A kept failure is cleared by every start and Retry, so it never shows while dictating.
const failureReason = live?.failureReason ?? null
// Attach steps aside while dictating or holding a failed recording, so the field gets the room.
return (
<>
{isStarting || isRecording || isProcessing ? null : <MobileTerminalAttachButton {...props} />}
<MobileTerminalDictationButton {...props} />
{isStarting || isRecording || isProcessing || failureReason !== null ? null : (
<MobileTerminalAttachButton {...props} />
)}
{failureReason !== null && live ? (
<MobileDictationRetryButton onRetry={live.retry} labelled={false} />
) : (
<MobileTerminalDictationButton {...props} />
)}
</>
)
}
@@ -12,6 +12,7 @@ import {
import { MobileDictationLiveContext } from '../../dictation/mobile-dictation-live-context'
import {
MobileDictationCancelButton,
MobileDictationFailureNote,
MobileDictationFinishingNote,
MobileDictationRecordingMeter
} from '../../dictation/MobileDictationLiveControls'
@@ -37,6 +38,16 @@ export function MobileTerminalLiveInputStatus({
const t = useMobileTranslation()
const live = useContext(MobileDictationLiveContext)
const dictationTitle = mobileDictationStatusTitle(dictation)
if (live?.failureReason != null) {
return (
<View style={[styles.status, styles.dictating]} pointerEvents="box-none">
<MobileDictationCancelButton onCancel={live.cancel} compact discards />
<View style={styles.dictationLine} pointerEvents="none">
<MobileDictationFailureNote reason={live.failureReason} compact />
</View>
</View>
)
}
if (!dictationTitle) {
return isAttaching ? (
<View style={styles.status} pointerEvents="none">