mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
feat(mobile): keep a failed dictation's recording and offer Retry
This commit is contained in:
@@ -1,14 +1,15 @@
|
||||
import { useEffect, useMemo, useState, type Ref } from 'react'
|
||||
import { ActivityIndicator, Pressable, Text, View } from 'react-native'
|
||||
import { Laptop, Mic, Smartphone, Square, X } from 'lucide-react-native'
|
||||
import { CircleAlert, Laptop, Mic, RotateCcw, Smartphone, Square, X } from 'lucide-react-native'
|
||||
import type { MobileDictationMeter } from '../hooks/mobile-dictation-meter'
|
||||
import type { MobileDictationEngineShown } from '../hooks/mobile-dictation-session-state'
|
||||
import { useMobileTranslation } from '../i18n/use-mobile-translation'
|
||||
import {
|
||||
COMPOSER_CONTROL_HIT_SLOP,
|
||||
COMPOSER_CONTROL_SIZE,
|
||||
composerInputStyles
|
||||
} from '../components/composer-input/composer-input-styles'
|
||||
import { colors, spacing, typography } from '../theme/mobile-theme'
|
||||
import { colors, radii, spacing, typography } from '../theme/mobile-theme'
|
||||
import { themedStyleSheet } from '../theme/themed-styles'
|
||||
import { useMobileColorScheme } from '../theme/use-mobile-color-scheme'
|
||||
import {
|
||||
@@ -27,11 +28,14 @@ const FINISHING_NOTE_DELAY_MS = 300
|
||||
export function MobileDictationCancelButton({
|
||||
onCancel,
|
||||
targetRef,
|
||||
compact = false
|
||||
compact = false,
|
||||
discards = false
|
||||
}: {
|
||||
onCancel: () => void
|
||||
targetRef?: Ref<View>
|
||||
compact?: boolean
|
||||
// After a failure, ✕ throws the kept recording away.
|
||||
discards?: boolean
|
||||
}): React.JSX.Element {
|
||||
useMobileColorScheme()
|
||||
useMobileTranslation()
|
||||
@@ -39,7 +43,9 @@ export function MobileDictationCancelButton({
|
||||
<Pressable
|
||||
ref={targetRef}
|
||||
accessibilityRole="button"
|
||||
accessibilityLabel={dictationControlsCopy.cancel()}
|
||||
accessibilityLabel={
|
||||
discards ? dictationControlsCopy.discard() : dictationControlsCopy.cancel()
|
||||
}
|
||||
style={({ pressed }) => [
|
||||
composerInputStyles.controlButton,
|
||||
compact && styles.compactCancel,
|
||||
@@ -86,6 +92,77 @@ export function MobileDictationMicIcon({
|
||||
)
|
||||
}
|
||||
|
||||
// The failure that kept its recording: a reason and "Recording kept", on two lines in chat and one in the terminal.
|
||||
export function MobileDictationFailureNote({
|
||||
reason,
|
||||
compact = false
|
||||
}: {
|
||||
reason: string
|
||||
compact?: boolean
|
||||
}): React.JSX.Element {
|
||||
useMobileColorScheme()
|
||||
useMobileTranslation()
|
||||
const kept = dictationControlsCopy.recordingKept()
|
||||
return (
|
||||
<View style={styles.row} accessible accessibilityLabel={`${reason}. ${kept}.`}>
|
||||
<CircleAlert
|
||||
pointerEvents="none"
|
||||
size={compact ? 15 : 17}
|
||||
color={colors.statusRed}
|
||||
strokeWidth={2.2}
|
||||
/>
|
||||
{compact ? (
|
||||
<Text style={styles.failureCompact} numberOfLines={1}>
|
||||
{`${reason} · ${kept}`}
|
||||
</Text>
|
||||
) : (
|
||||
<View style={styles.failureLines}>
|
||||
<Text style={styles.failureReason} numberOfLines={1}>
|
||||
{reason}
|
||||
</Text>
|
||||
<Text style={styles.failureKept} numberOfLines={1}>
|
||||
{kept}
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
)
|
||||
}
|
||||
|
||||
// Retry sits where Stop was: a labelled pill in chat, the round control in the terminal.
|
||||
export function MobileDictationRetryButton({
|
||||
onRetry,
|
||||
labelled
|
||||
}: {
|
||||
onRetry: () => void
|
||||
labelled: boolean
|
||||
}): React.JSX.Element {
|
||||
useMobileColorScheme()
|
||||
useMobileTranslation()
|
||||
return (
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
accessibilityLabel={dictationControlsCopy.retry()}
|
||||
style={({ pressed }) => [
|
||||
labelled ? styles.retryPill : composerInputStyles.controlButton,
|
||||
pressed && composerInputStyles.pressed
|
||||
]}
|
||||
hitSlop={COMPOSER_CONTROL_HIT_SLOP}
|
||||
onPress={onRetry}
|
||||
>
|
||||
<RotateCcw
|
||||
pointerEvents="none"
|
||||
size={15}
|
||||
color={labelled ? colors.bgBase : colors.textPrimary}
|
||||
strokeWidth={2.4}
|
||||
/>
|
||||
{labelled ? (
|
||||
<Text style={styles.retryLabel}>{dictationControlsCopy.retryShort()}</Text>
|
||||
) : null}
|
||||
</Pressable>
|
||||
)
|
||||
}
|
||||
|
||||
function EngineIcon({ kind, color }: { kind: MobileDictationEngineShown['kind']; color: string }) {
|
||||
return kind === 'phone' ? (
|
||||
<Smartphone pointerEvents="none" size={15} color={color} strokeWidth={2} />
|
||||
@@ -190,5 +267,38 @@ const styles = themedStyleSheet(() => ({
|
||||
flexShrink: 1,
|
||||
color: colors.textSecondary,
|
||||
fontSize: typography.metaSize + 1
|
||||
},
|
||||
failureLines: {
|
||||
flexShrink: 1,
|
||||
minWidth: 0
|
||||
},
|
||||
failureReason: {
|
||||
color: colors.textPrimary,
|
||||
fontSize: typography.metaSize + 1,
|
||||
fontWeight: '500'
|
||||
},
|
||||
failureKept: {
|
||||
color: colors.textSecondary,
|
||||
fontSize: typography.metaSize
|
||||
},
|
||||
failureCompact: {
|
||||
flexShrink: 1,
|
||||
color: colors.textPrimary,
|
||||
fontSize: typography.metaSize
|
||||
},
|
||||
retryPill: {
|
||||
height: COMPOSER_CONTROL_SIZE,
|
||||
borderRadius: radii.pill,
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
flexDirection: 'row',
|
||||
gap: 6,
|
||||
paddingHorizontal: spacing.md,
|
||||
backgroundColor: colors.surfaceBright
|
||||
},
|
||||
retryLabel: {
|
||||
color: colors.bgBase,
|
||||
fontSize: typography.bodySize,
|
||||
fontWeight: '600'
|
||||
}
|
||||
}))
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
import { translate } from '../i18n/mobile-i18n'
|
||||
import type { MobileDictationEngineShown } from '../hooks/mobile-dictation-session-state'
|
||||
import type {
|
||||
MobileDictationEngineShown,
|
||||
MobileDictationKeptFailure
|
||||
} from '../hooks/mobile-dictation-session-state'
|
||||
import { MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE } from '../hooks/mobile-dictation-pending-audio-budget'
|
||||
import { phoneDictationFailedMessage } from '../hooks/mobile-dictation-device-engine'
|
||||
import { dictationFailureMessage, dictationSessionLostMessage } from './mobile-dictation-setup'
|
||||
|
||||
// Active: starting or recording, so the mic is a Stop.
|
||||
export type MobileDictationMicState = { starting: boolean; active: boolean; processing: boolean }
|
||||
@@ -11,6 +17,10 @@ export const dictationControlsCopy = {
|
||||
stop: () => translate('dictation.controls.stop', 'Stop and insert'),
|
||||
finishing: () => translate('dictation.controls.finishing', 'Finishing dictation'),
|
||||
cancel: () => translate('dictation.controls.cancel', 'Cancel dictation'),
|
||||
discard: () => translate('dictation.controls.discard', 'Discard recording'),
|
||||
retry: () => translate('dictation.controls.retry', 'Retry dictation'),
|
||||
retryShort: () => translate('dictation.controls.retryShort', 'Retry'),
|
||||
recordingKept: () => translate('dictation.failure.recordingKept', 'Recording kept'),
|
||||
mic: ({ starting, active, processing }: MobileDictationMicState) =>
|
||||
starting
|
||||
? dictationControlsCopy.starting()
|
||||
@@ -79,3 +89,17 @@ export function dictationMeterLabel(
|
||||
}
|
||||
return translate('dictation.controls.meter', 'Recording, {{duration}}', { duration })
|
||||
}
|
||||
|
||||
// The few words a kept failure shows in the composer row; the full message is too long for it.
|
||||
export function dictationFailureReason({ message, offline }: MobileDictationKeptFailure): string {
|
||||
if (offline || dictationFailureMessage(message) === dictationSessionLostMessage()) {
|
||||
return translate('dictation.failure.connectionLost', 'Connection lost')
|
||||
}
|
||||
if (message === MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE) {
|
||||
return translate('dictation.failure.connectionSlow', 'Connection too slow')
|
||||
}
|
||||
if (message === phoneDictationFailedMessage()) {
|
||||
return translate('dictation.failure.phoneStopped', 'Stopped on this phone')
|
||||
}
|
||||
return translate('dictation.failure.stopped', 'Dictation stopped')
|
||||
}
|
||||
|
||||
@@ -7,7 +7,11 @@ export type MobileDictationLive = {
|
||||
readonly engine: MobileDictationEngineShown | null
|
||||
// The paired computer's name; null when another server runs the workspace and the phone lacks its name.
|
||||
readonly computerName: string | null
|
||||
// Also discards a failed dictation's kept recording.
|
||||
readonly cancel: () => void
|
||||
// The short reason a dictation failed with its recording kept; Retry plays it to the computer again.
|
||||
readonly failureReason: string | null
|
||||
readonly retry: () => void
|
||||
}
|
||||
|
||||
// The chat mic sits several composers deep (overlay, queued edits); it reads the live dictation from here.
|
||||
|
||||
@@ -44,7 +44,7 @@ function speechlessServerMessage(): string {
|
||||
)
|
||||
}
|
||||
|
||||
function isSpeechlessServerError(message: string): boolean {
|
||||
export function isSpeechlessServerError(message: string): boolean {
|
||||
return message.startsWith('speech_unavailable')
|
||||
}
|
||||
|
||||
|
||||
@@ -4,10 +4,11 @@ import { useWorkspaceExecutionHost } from '../navigation/workspace-execution-hos
|
||||
import { useHostDisplay } from '../transport/hosts/use-host-display'
|
||||
import { useHostProfile } from '../transport/hosts/use-host-profile'
|
||||
import type { MobileDictationLive } from './mobile-dictation-live-context'
|
||||
import { dictationFailureReason } from './mobile-dictation-controls-copy'
|
||||
|
||||
// What every dictation control reads while one runs: meter, engine, the computer's name and cancel.
|
||||
// What every dictation control reads while one runs or waits for Retry: meter, engine, the computer's name, cancel and Retry.
|
||||
export function useMobileDictationLive(
|
||||
dictation: Pick<UseMobileDictationResult, 'meter' | 'engine'>,
|
||||
dictation: Pick<UseMobileDictationResult, 'meter' | 'engine' | 'failure' | 'retry'>,
|
||||
cancel: () => void,
|
||||
hostId: string
|
||||
): MobileDictationLive {
|
||||
@@ -20,9 +21,17 @@ export function useMobileDictationLive(
|
||||
const hostTitle = useHostDisplay(hostProfile).title
|
||||
// A workspace routed to another server transcribes there, and the phone lacks that server's name.
|
||||
const computerName = useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
|
||||
const { meter, engine } = dictation
|
||||
const { meter, engine, failure, retry } = dictation
|
||||
const failureReason = failure === null ? null : dictationFailureReason(failure)
|
||||
return useMemo(
|
||||
() => ({ meter, engine, computerName, cancel: stableCancel }),
|
||||
[meter, engine, computerName, stableCancel]
|
||||
() => ({
|
||||
meter,
|
||||
engine,
|
||||
computerName,
|
||||
cancel: stableCancel,
|
||||
failureReason,
|
||||
retry: () => void retry()
|
||||
}),
|
||||
[meter, engine, computerName, stableCancel, failureReason, retry]
|
||||
)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import type {
|
||||
} from '../platform/dictation-capture-contract'
|
||||
import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget'
|
||||
import type { MobileDictationHeldAudio } from './mobile-dictation-held-audio'
|
||||
import type { MobileDictationRecording } from './mobile-dictation-recording'
|
||||
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
|
||||
import type { MobileDictationRun } from './mobile-dictation-engine-run'
|
||||
|
||||
@@ -35,7 +36,7 @@ type MobileDictationAudioChunkQueue = {
|
||||
stopAtAudioLimit: () => void
|
||||
}
|
||||
|
||||
// Holding: the microphone runs from the tap while the host's start reply is pending, or for a whole on-device dictation.
|
||||
// Holding: the microphone runs from the tap while the host's start reply is pending, or while a replay's host start is.
|
||||
export type MobileDictationAudioPhase = 'closed' | 'holding' | 'sending'
|
||||
|
||||
// Newer hosts report the text so far: shown while speaking, and handed over by a failure instead of dropped.
|
||||
@@ -122,6 +123,7 @@ export function useMobileDictationAudioChunks(capture: DictationCapture, run: Mo
|
||||
failActiveDictation: run.failActiveDictation,
|
||||
transcriptSoFar: run.keptTranscript,
|
||||
heldAudio: run.heldAudio.current,
|
||||
recording: run.recording.current,
|
||||
// Like desktop: stop as the user would, so what the host kept is transcribed and inserted.
|
||||
stopAtAudioLimit: () => {
|
||||
if (run.status.current === 'recording') {
|
||||
@@ -142,7 +144,7 @@ export function subscribeMobileDictationAudioChunks(
|
||||
getActiveId: () => string | null
|
||||
audioPhase: () => MobileDictationAudioPhase
|
||||
},
|
||||
queue: Omit<MobileDictationAudioChunkQueue, 'isActive'>
|
||||
queue: Omit<MobileDictationAudioChunkQueue, 'isActive'> & { recording: MobileDictationRecording }
|
||||
): { remove: () => void; sendHeld: () => void } {
|
||||
// Reuse wiring on the microphone hot path instead of allocating per chunk.
|
||||
const chunkQueue = { ...queue, isActive: (id: string) => source.getActiveId() === id }
|
||||
@@ -159,11 +161,15 @@ export function subscribeMobileDictationAudioChunks(
|
||||
}
|
||||
const subscription = capture.onChunk((chunk) => {
|
||||
const dictationId = source.getActiveId()
|
||||
if (!dictationId) {
|
||||
return
|
||||
}
|
||||
// Kept whatever the phase, so a phone's failure or a failed send can play it to the computer again.
|
||||
queue.recording.keep(chunk)
|
||||
const phase = source.audioPhase()
|
||||
if (!dictationId || phase === 'closed') {
|
||||
if (phase === 'closed') {
|
||||
return
|
||||
}
|
||||
// An on-device dictation holds its audio with no computer at all.
|
||||
if (phase === 'holding') {
|
||||
queue.heldAudio.hold(chunk)
|
||||
return
|
||||
|
||||
@@ -9,11 +9,7 @@ import type {
|
||||
MobileDictationAudioPhase,
|
||||
MobileDictationTranscriptSoFar
|
||||
} from './mobile-dictation-audio-chunk'
|
||||
import {
|
||||
cancelDesktopDictation,
|
||||
settledDesktopDictationWork,
|
||||
startMobileDictationDesktopSession
|
||||
} from './mobile-dictation-desktop-start'
|
||||
import { cancelDesktopDictation } from './mobile-dictation-desktop-start'
|
||||
import {
|
||||
chooseMobileDictationRoute,
|
||||
phoneDictationBusyMessage,
|
||||
@@ -26,6 +22,11 @@ import {
|
||||
} from './mobile-dictation-device-engine'
|
||||
import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget'
|
||||
import type { MobileDictationHeldAudio } from './mobile-dictation-held-audio'
|
||||
import type { MobileDictationRecording } from './mobile-dictation-recording'
|
||||
import {
|
||||
replayMobileDictationToHost,
|
||||
startMobileDictationOnHost
|
||||
} from './mobile-dictation-host-start'
|
||||
import type { ShowMobileDictationEngine } from './mobile-dictation-engine-shown'
|
||||
|
||||
// What the engines share with useMobileDictation: its refs (read at call time) and its state transitions.
|
||||
@@ -42,15 +43,18 @@ export type MobileDictationRun = {
|
||||
readonly pendingChunks: RefObject<Set<Promise<void>>>
|
||||
readonly pendingAudioBudget: RefObject<MobileDictationPendingAudioBudget>
|
||||
readonly heldAudio: RefObject<MobileDictationHeldAudio>
|
||||
readonly recording: RefObject<MobileDictationRecording>
|
||||
readonly sendHeldAudio: RefObject<() => void>
|
||||
readonly finish: RefObject<(emptyTranscriptMessage: string) => Promise<void>>
|
||||
readonly onTranscript: RefObject<(text: string) => void>
|
||||
readonly onAudioLimitReached: RefObject<(() => void) | undefined>
|
||||
readonly keptTranscript: MobileDictationTranscriptSoFar
|
||||
readonly applyStatus: (next: DictationStatus) => void
|
||||
// keepText: leave the words so far on screen (recording, finishing, or a failure kept for Retry).
|
||||
readonly applyStatus: (next: DictationStatus, keepText?: boolean) => void
|
||||
readonly showEngine: ShowMobileDictationEngine
|
||||
readonly closeDictationAudio: () => void
|
||||
readonly reportError: (err: unknown) => void
|
||||
// kept: the recording stays for Retry, the words so far stay shown, and nothing is inserted yet.
|
||||
readonly reportError: (err: unknown, kept?: boolean) => void
|
||||
readonly failActiveDictation: (dictationId: string, err: unknown) => void
|
||||
}
|
||||
|
||||
@@ -80,6 +84,7 @@ export function settleMobileDictation(
|
||||
run.engine.current = { kind: 'host' }
|
||||
run.audioPhase.current = 'closed'
|
||||
run.heldAudio.current.clear()
|
||||
run.recording.current.clear()
|
||||
run.pendingChunks.current.clear()
|
||||
run.pendingAudioBudget.current.reset()
|
||||
run.applyStatus('idle')
|
||||
@@ -90,48 +95,7 @@ export function settleMobileDictation(
|
||||
}
|
||||
}
|
||||
|
||||
export async function startMobileDictationOnHost(
|
||||
run: MobileDictationRun,
|
||||
client: RpcClient,
|
||||
dictationId: string,
|
||||
generation: number
|
||||
): Promise<void> {
|
||||
await settledDesktopDictationWork(client)
|
||||
if (run.generation.current !== generation) {
|
||||
return
|
||||
}
|
||||
await startMobileDictationDesktopSession({
|
||||
client,
|
||||
dictationId,
|
||||
generation,
|
||||
getCurrentGeneration: () => run.generation.current,
|
||||
getEnabled: () => run.enabled.current,
|
||||
getActiveId: () => run.activeId.current,
|
||||
clearActiveId: (id) => {
|
||||
if (run.activeId.current === id) {
|
||||
run.activeId.current = null
|
||||
}
|
||||
},
|
||||
setIdle: () => run.applyStatus('idle'),
|
||||
commitRecordingStart: (hostDedupesResends) => {
|
||||
run.audioPhase.current = 'sending'
|
||||
run.pendingChunks.current.clear()
|
||||
run.pendingAudioBudget.current.reset(hostDedupesResends)
|
||||
// A phone dictation stopped or failed at finish is already processing; it never records again.
|
||||
if (run.status.current !== 'processing') {
|
||||
run.applyStatus('recording')
|
||||
}
|
||||
run.showEngine('computer')
|
||||
run.sendHeldAudio.current()
|
||||
if (run.stopWhenStarted.current) {
|
||||
void run.finish.current(dictationSessionErrorMessage('no-speech'))
|
||||
}
|
||||
},
|
||||
rollbackRecordingStart: run.closeDictationAudio
|
||||
})
|
||||
}
|
||||
|
||||
// The phone's recognizer failed: replay the audio it kept to the computer, as a start's held audio (seq from 0), and carry on there.
|
||||
// The phone's recognizer failed: replay the recording to the computer, as a start's held audio (seq from 0), and carry on there.
|
||||
export async function fallBackMobileDictationToHost(
|
||||
run: MobileDictationRun,
|
||||
dictationId: string,
|
||||
@@ -149,25 +113,15 @@ export async function fallBackMobileDictationToHost(
|
||||
!engine.fallback ||
|
||||
!client ||
|
||||
client.getState() !== 'connected' ||
|
||||
!run.heldAudio.current.complete
|
||||
!run.recording.current.complete
|
||||
) {
|
||||
run.failActiveDictation(dictationId, new Error(phoneDictationFailedMessage()))
|
||||
return
|
||||
}
|
||||
run.engine.current = { kind: 'host' }
|
||||
run.showEngine('computer', true)
|
||||
// A stop that began while the phone was still live is already processing; the computer must finish it too.
|
||||
run.stopWhenStarted.current = finishing || run.status.current === 'processing'
|
||||
try {
|
||||
await startMobileDictationOnHost(run, client, dictationId, run.generation.current)
|
||||
} catch (err) {
|
||||
// The host start already released the mic and the dictation; keep what the phone heard.
|
||||
const kept = run.keptTranscript.take(dictationId)
|
||||
if (kept) {
|
||||
run.onTranscript.current(kept)
|
||||
}
|
||||
run.reportError(err)
|
||||
}
|
||||
await replayMobileDictationToHost(run, client, dictationId, run.generation.current)
|
||||
}
|
||||
|
||||
// Pick the engine before the mic opens, and arm the phone's recognizer so it hears the first word.
|
||||
@@ -229,10 +183,9 @@ export function commitMobileDictationOnDevice(
|
||||
() => void fallBackMobileDictationToHost(run, dictationId, false)
|
||||
)
|
||||
}
|
||||
// This phone never replays to the computer, so it keeps no audio.
|
||||
if (!fallback) {
|
||||
run.audioPhase.current = 'closed'
|
||||
}
|
||||
// The recording keeps the phone's audio for a replay; nothing goes to the computer while the phone hears it.
|
||||
run.audioPhase.current = 'closed'
|
||||
run.heldAudio.current.clear()
|
||||
run.applyStatus('recording')
|
||||
run.showEngine('phone')
|
||||
if (run.stopWhenStarted.current) {
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { errorMessage } from '../../../src/shared/error-message'
|
||||
import {
|
||||
isDictationSetupRequiredError,
|
||||
isSpeechlessServerError
|
||||
} from '../dictation/mobile-dictation-setup'
|
||||
import type { MobileDictationEngine } from './mobile-dictation-device-engine'
|
||||
import type { MobileDictationRecording } from './mobile-dictation-recording'
|
||||
import type { MobileDictationRun } from './mobile-dictation-engine-run'
|
||||
|
||||
// Retry plays the recording to the computer, so it needs the whole recording and an engine that may use the computer (not This phone).
|
||||
export function mobileDictationFailureRetryable(
|
||||
recording: MobileDictationRecording,
|
||||
engine: MobileDictationEngine
|
||||
): boolean {
|
||||
return recording.complete && recording.bytes > 0 && (engine.kind === 'host' || engine.fallback)
|
||||
}
|
||||
|
||||
// A computer that needs setup or an update refuses every Retry the same way; those keep today's path (words inserted, setup sheet or toast).
|
||||
function retryCanSucceed(err: unknown): boolean {
|
||||
const message = errorMessage(err)
|
||||
return !isDictationSetupRequiredError(message) && !isSpeechlessServerError(message)
|
||||
}
|
||||
|
||||
// Never lose speech quietly: a retryable failure keeps the recording and the words so far for Retry; otherwise the words so far are inserted, as before.
|
||||
export function concludeMobileDictationFailure(
|
||||
run: Pick<
|
||||
MobileDictationRun,
|
||||
'recording' | 'engine' | 'keptTranscript' | 'onTranscript' | 'reportError'
|
||||
>,
|
||||
dictationId: string,
|
||||
err: unknown,
|
||||
retryable = mobileDictationFailureRetryable(run.recording.current, run.engine.current)
|
||||
): void {
|
||||
if (retryable && retryCanSucceed(err)) {
|
||||
run.reportError(err, true)
|
||||
return
|
||||
}
|
||||
run.recording.current.clear()
|
||||
const kept = run.keptTranscript.take(dictationId)
|
||||
if (kept) {
|
||||
run.onTranscript.current(kept)
|
||||
}
|
||||
run.reportError(err)
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload'
|
||||
import type { DictationCapture } from '../platform/dictation-capture-contract'
|
||||
import { isCurrentMobileDictationFinish } from './mobile-dictation-session-state'
|
||||
import { finishDesktopDictation } from './mobile-dictation-desktop-start'
|
||||
import {
|
||||
finishMobileDictationOnDevice,
|
||||
settleMobileDictation,
|
||||
type MobileDictationRun
|
||||
} from './mobile-dictation-engine-run'
|
||||
|
||||
// Stop listening and settle the dictation with its final text, from whichever engine heard it.
|
||||
export async function finishMobileDictation(
|
||||
run: MobileDictationRun,
|
||||
capture: Pick<DictationCapture, 'end'>,
|
||||
emptyTranscriptMessage: string
|
||||
): Promise<void> {
|
||||
const client = run.client.current
|
||||
const dictationId = run.activeId.current
|
||||
const engine = run.engine.current
|
||||
if (!dictationId || (engine.kind === 'host' && !client)) {
|
||||
return
|
||||
}
|
||||
// A replay's host start is still pending: finish once it commits and the kept audio has gone.
|
||||
if (engine.kind === 'host' && run.audioPhase.current === 'holding') {
|
||||
run.stopWhenStarted.current = true
|
||||
if (run.status.current === 'recording') {
|
||||
run.applyStatus('processing')
|
||||
// Stop listening now; the tail joins the held audio the replay sends.
|
||||
void capture.end()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const generation = run.generation.current + 1
|
||||
run.generation.current = generation
|
||||
run.applyStatus('processing')
|
||||
const isCurrent = () =>
|
||||
isCurrentMobileDictationFinish(
|
||||
run.generation.current,
|
||||
generation,
|
||||
run.enabled.current,
|
||||
run.activeId.current,
|
||||
dictationId
|
||||
)
|
||||
try {
|
||||
// Keep shutdown inside try for finally cleanup; accept its tail chunks and snapshot pending sends only after end settles, or finish can overtake the last audio.
|
||||
await capture.end()
|
||||
if (engine.kind === 'device') {
|
||||
await finishMobileDictationOnDevice(
|
||||
run,
|
||||
engine,
|
||||
dictationId,
|
||||
emptyTranscriptMessage,
|
||||
isCurrent
|
||||
)
|
||||
return
|
||||
}
|
||||
run.audioPhase.current = 'closed'
|
||||
await Promise.allSettled(Array.from(run.pendingChunks.current))
|
||||
if (!isCurrent() || !client) {
|
||||
return
|
||||
}
|
||||
const sentSeconds = run.pendingAudioBudget.current.sentAudioSeconds
|
||||
const finished = await finishDesktopDictation(client, dictationId, sentSeconds)
|
||||
if (!isCurrent()) {
|
||||
return
|
||||
}
|
||||
const transcript = rpcPayloadMember(finished, 'text')
|
||||
settleMobileDictation(
|
||||
run,
|
||||
typeof transcript === 'string' ? transcript.trim() : '',
|
||||
emptyTranscriptMessage
|
||||
)
|
||||
} catch (err) {
|
||||
run.failActiveDictation(dictationId, err)
|
||||
}
|
||||
}
|
||||
@@ -5,10 +5,10 @@ import {
|
||||
} from '../../../src/shared/rpc-contract/speech-params'
|
||||
import type { DictationCaptureChunk } from '../platform/dictation-capture-contract'
|
||||
|
||||
// Five minutes of 16 kHz PCM (9.6 MB): what an on-device dictation keeps to replay to the computer if the phone fails.
|
||||
const MAX_HELD_AUDIO_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * 300
|
||||
// Five minutes of 16 kHz PCM (9.6 MB): the longest recording kept for a replay to the computer.
|
||||
export const MAX_HELD_AUDIO_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * 300
|
||||
|
||||
// Speech not sent yet, in order: held between the tap and the host's start reply, while every send slot is busy, and for a whole on-device dictation.
|
||||
// Speech not sent yet, in order: held between the tap and the host's start reply, while every send slot is busy, and for a replay of the recording.
|
||||
export class MobileDictationHeldAudio {
|
||||
private held: DictationCaptureChunk[] = []
|
||||
private heldBytes = 0
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
|
||||
import { dictationSessionErrorMessage } from '../dictation/mobile-dictation-setup'
|
||||
import {
|
||||
settledDesktopDictationWork,
|
||||
startMobileDictationDesktopSession
|
||||
} from './mobile-dictation-desktop-start'
|
||||
import { concludeMobileDictationFailure } from './mobile-dictation-failure'
|
||||
import type { MobileDictationRun } from './mobile-dictation-engine-run'
|
||||
|
||||
export async function startMobileDictationOnHost(
|
||||
run: MobileDictationRun,
|
||||
client: RpcClient,
|
||||
dictationId: string,
|
||||
generation: number,
|
||||
// A replay (fallback or Retry) keeps the words so far on screen if its start fails.
|
||||
replay = false
|
||||
): Promise<void> {
|
||||
await settledDesktopDictationWork(client)
|
||||
if (run.generation.current !== generation) {
|
||||
return
|
||||
}
|
||||
await startMobileDictationDesktopSession({
|
||||
client,
|
||||
dictationId,
|
||||
generation,
|
||||
getCurrentGeneration: () => run.generation.current,
|
||||
getEnabled: () => run.enabled.current,
|
||||
getActiveId: () => run.activeId.current,
|
||||
clearActiveId: (id) => {
|
||||
if (run.activeId.current === id) {
|
||||
run.activeId.current = null
|
||||
}
|
||||
},
|
||||
setIdle: () => run.applyStatus('idle', replay),
|
||||
commitRecordingStart: (hostDedupesResends) => {
|
||||
run.audioPhase.current = 'sending'
|
||||
run.pendingChunks.current.clear()
|
||||
run.pendingAudioBudget.current.reset(hostDedupesResends)
|
||||
// A phone dictation stopped or failed at finish is already processing; it never records again.
|
||||
if (run.status.current !== 'processing') {
|
||||
run.applyStatus('recording')
|
||||
}
|
||||
run.showEngine('computer')
|
||||
run.sendHeldAudio.current()
|
||||
if (run.stopWhenStarted.current) {
|
||||
void run.finish.current(dictationSessionErrorMessage('no-speech'))
|
||||
}
|
||||
},
|
||||
rollbackRecordingStart: run.closeDictationAudio
|
||||
})
|
||||
}
|
||||
|
||||
// Play the kept recording to the computer as this dictation's held audio (seq from 0); audio heard after joins it in order until the start commits.
|
||||
// Fallback and Retry both end here; a failed start keeps the recording for Retry.
|
||||
export async function replayMobileDictationToHost(
|
||||
run: MobileDictationRun,
|
||||
client: RpcClient,
|
||||
dictationId: string,
|
||||
generation: number
|
||||
): Promise<void> {
|
||||
run.engine.current = { kind: 'host' }
|
||||
run.recording.current.replayInto(run.heldAudio.current)
|
||||
run.audioPhase.current = 'holding'
|
||||
try {
|
||||
await startMobileDictationOnHost(run, client, dictationId, generation, true)
|
||||
} catch (err) {
|
||||
concludeMobileDictationFailure(run, dictationId, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { DictationCaptureChunk } from '../platform/dictation-capture-contract'
|
||||
import { MAX_HELD_AUDIO_BYTES, type MobileDictationHeldAudio } from './mobile-dictation-held-audio'
|
||||
|
||||
// Everything one dictation heard, from the tap, whichever engine hears it: replayed to the computer when the phone fails, and kept after a failure for Retry.
|
||||
export class MobileDictationRecording {
|
||||
private chunks: DictationCaptureChunk[] = []
|
||||
private keptBytes = 0
|
||||
private incomplete = false
|
||||
|
||||
keep(chunk: DictationCaptureChunk): void {
|
||||
if (this.incomplete) {
|
||||
return
|
||||
}
|
||||
// Past the cap, or with audio the page dropped, a replay would lose speech (and the send refuses it), so keep nothing and say it is incomplete.
|
||||
if (chunk.droppedBytes > 0 || this.keptBytes + chunk.data.byteLength > MAX_HELD_AUDIO_BYTES) {
|
||||
this.chunks = []
|
||||
this.keptBytes = 0
|
||||
this.incomplete = true
|
||||
return
|
||||
}
|
||||
this.chunks.push(chunk)
|
||||
this.keptBytes += chunk.data.byteLength
|
||||
}
|
||||
|
||||
// False once speech was lost; the recording no longer covers the whole dictation.
|
||||
get complete(): boolean {
|
||||
return !this.incomplete
|
||||
}
|
||||
|
||||
get bytes(): number {
|
||||
return this.keptBytes
|
||||
}
|
||||
|
||||
// Queue a copy for sending; the recording itself stays, so a failed replay can be retried.
|
||||
replayInto(held: MobileDictationHeldAudio): void {
|
||||
held.clear()
|
||||
for (const chunk of this.chunks) {
|
||||
held.hold(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.chunks = []
|
||||
this.keptBytes = 0
|
||||
this.incomplete = false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// A computer dictation that fails mid-session keeps its whole recording; Retry plays all of it to the computer as a new dictation.
|
||||
import { createElement } from 'react'
|
||||
import { act, create } from 'react-test-renderer'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
createFakeRpcClient,
|
||||
type FakeRpcClient,
|
||||
type SentRequest
|
||||
} from '../mobile-web-shell/bridge-host-test-fakes'
|
||||
import type {
|
||||
DictationCapture,
|
||||
DictationCaptureChunk
|
||||
} from '../platform/dictation-capture-contract'
|
||||
|
||||
type Seam = {
|
||||
chunkHandlers: Set<(chunk: DictationCaptureChunk) => void>
|
||||
tail: Uint8Array | null
|
||||
}
|
||||
|
||||
const seam = vi.hoisted((): Seam => ({ chunkHandlers: new Set(), tail: null }))
|
||||
|
||||
vi.mock('react-native', () => ({
|
||||
AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) },
|
||||
Platform: { OS: 'ios' }
|
||||
}))
|
||||
|
||||
// Keep capture identity stable like both real seams; a fresh object on render would rerun teardown and cancel dictation.
|
||||
vi.mock('../platform/dictation-capture', () => {
|
||||
const capture: DictationCapture = {
|
||||
open: async () => ({ ok: true }),
|
||||
begin: () => true,
|
||||
end: async () => {
|
||||
// Page capture reads once more here and may deliver audio.
|
||||
const tail = seam.tail
|
||||
seam.tail = null
|
||||
if (tail !== null) {
|
||||
for (const handler of seam.chunkHandlers) {
|
||||
handler({ data: tail, droppedBytes: 0 })
|
||||
}
|
||||
}
|
||||
},
|
||||
release: () => {},
|
||||
onChunk: (handler) => {
|
||||
seam.chunkHandlers.add(handler)
|
||||
return {
|
||||
remove: () => {
|
||||
seam.chunkHandlers.delete(handler)
|
||||
}
|
||||
}
|
||||
},
|
||||
onInterruption: () => ({ remove: () => {} })
|
||||
}
|
||||
return { useDictationCapture: () => capture }
|
||||
})
|
||||
|
||||
import { useMobileDictation, type UseMobileDictationResult } from './use-mobile-dictation'
|
||||
|
||||
type Reply = (request: SentRequest) => 'hold' | { text?: string } | Error
|
||||
|
||||
// Answer every request; one reply can unblock the next.
|
||||
async function pump(rpc: FakeRpcClient, sent: SentRequest[], reply: Reply): Promise<void> {
|
||||
for (let round = 0; round < 30; round += 1) {
|
||||
for (const request of rpc.requests.splice(0)) {
|
||||
sent.push(request)
|
||||
const answer = reply(request)
|
||||
if (answer === 'hold') {
|
||||
continue
|
||||
}
|
||||
if (answer instanceof Error) {
|
||||
request.reject(answer)
|
||||
} else {
|
||||
request.resolve({ id: 'desktop', ok: true, result: answer })
|
||||
}
|
||||
}
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
function paramsOf(request: SentRequest): Record<string, unknown> {
|
||||
const params = request.args[1]
|
||||
return typeof params === 'object' && params !== null ? { ...params } : {}
|
||||
}
|
||||
|
||||
function deliver(bytes: number[]): void {
|
||||
for (const handler of seam.chunkHandlers) {
|
||||
handler({ data: Uint8Array.from(bytes), droppedBytes: 0 })
|
||||
}
|
||||
}
|
||||
|
||||
const held: { dictation: UseMobileDictationResult | null } = { dictation: null }
|
||||
|
||||
function dictation(): UseMobileDictationResult {
|
||||
const current = held.dictation
|
||||
if (current === null) {
|
||||
throw new Error('nothing mounted')
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
seam.chunkHandlers.clear()
|
||||
seam.tail = null
|
||||
held.dictation = null
|
||||
})
|
||||
|
||||
describe('a computer dictation whose finish fails', () => {
|
||||
it('keeps the recording, and Retry sends all of it again and inserts the words', async () => {
|
||||
const rpc = createFakeRpcClient()
|
||||
const sent: SentRequest[] = []
|
||||
const inserted: string[] = []
|
||||
const errors: { message: string; kept: boolean }[] = []
|
||||
function Probe(): null {
|
||||
held.dictation = useMobileDictation({
|
||||
client: rpc,
|
||||
enabled: true,
|
||||
onTranscript: (text) => inserted.push(text),
|
||||
onError: (error, outcome) => errors.push({ message: error.message, kept: outcome.kept })
|
||||
})
|
||||
return null
|
||||
}
|
||||
act(() => {
|
||||
create(createElement(Probe))
|
||||
})
|
||||
const firstTry: Reply = (request) =>
|
||||
request.method === 'speech.dictation.finish' ? new Error('connection lost') : {}
|
||||
await act(async () => {
|
||||
const started = dictation().start()
|
||||
await pump(rpc, sent, firstTry)
|
||||
await started
|
||||
})
|
||||
await act(async () => {
|
||||
deliver([1, 2, 3, 4])
|
||||
await pump(rpc, sent, firstTry)
|
||||
})
|
||||
seam.tail = Uint8Array.from([5, 6])
|
||||
await act(async () => {
|
||||
const stopped = dictation().stop()
|
||||
await pump(rpc, sent, firstTry)
|
||||
await stopped
|
||||
})
|
||||
expect(inserted).toEqual([])
|
||||
expect(errors).toEqual([{ message: 'connection lost', kept: true }])
|
||||
expect(dictation().failure).toEqual({ message: 'connection lost', offline: false })
|
||||
|
||||
const firstId = paramsOf(sent[0]).dictationId
|
||||
sent.length = 0
|
||||
await act(async () => {
|
||||
const retried = dictation().retry()
|
||||
await pump(rpc, sent, (request) =>
|
||||
request.method === 'speech.dictation.finish' ? { text: 'all of it' } : {}
|
||||
)
|
||||
await retried
|
||||
})
|
||||
expect(sent.map((request) => request.method)).toEqual([
|
||||
'speech.dictation.start',
|
||||
'speech.dictation.chunk',
|
||||
'speech.dictation.finish'
|
||||
])
|
||||
expect(paramsOf(sent[0]).dictationId).not.toBe(firstId)
|
||||
// The whole recording, from the first byte, as seq 0 of the new dictation.
|
||||
expect(paramsOf(sent[1])).toMatchObject({ audioBase64: 'AQIDBAUG', seq: 0 })
|
||||
expect(inserted).toEqual(['all of it'])
|
||||
expect(dictation().failure).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,42 @@
|
||||
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
|
||||
import { createMobileDictationId } from './mobile-dictation-session-state'
|
||||
import {
|
||||
concludeMobileDictationFailure,
|
||||
mobileDictationFailureRetryable
|
||||
} from './mobile-dictation-failure'
|
||||
import {
|
||||
releaseMobileDictationEngine,
|
||||
type MobileDictationRun
|
||||
} from './mobile-dictation-engine-run'
|
||||
import { replayMobileDictationToHost } from './mobile-dictation-host-start'
|
||||
|
||||
// A dictation failed after it began hearing: stop it, and keep its recording for Retry when the computer can hear it again.
|
||||
export function failActiveMobileDictation(
|
||||
run: MobileDictationRun,
|
||||
dictationId: string,
|
||||
err: unknown
|
||||
): void {
|
||||
if (run.activeId.current !== dictationId) {
|
||||
return
|
||||
}
|
||||
run.activeId.current = null
|
||||
const retryable = mobileDictationFailureRetryable(run.recording.current, run.engine.current)
|
||||
run.closeDictationAudio()
|
||||
void releaseMobileDictationEngine(run, run.client.current, dictationId)
|
||||
concludeMobileDictationFailure(run, dictationId, err, retryable)
|
||||
}
|
||||
|
||||
// Retry: the kept recording goes to the computer as a new dictation (seq from 0), already stopped, so it finishes as soon as it has been sent.
|
||||
export async function retryMobileDictationOnHost(
|
||||
run: MobileDictationRun,
|
||||
client: RpcClient
|
||||
): Promise<void> {
|
||||
const generation = run.generation.current + 1
|
||||
run.generation.current = generation
|
||||
const dictationId = createMobileDictationId()
|
||||
run.activeId.current = dictationId
|
||||
run.stopWhenStarted.current = true
|
||||
run.applyStatus('processing', true)
|
||||
run.showEngine('computer')
|
||||
await replayMobileDictationToHost(run, client, dictationId, generation)
|
||||
}
|
||||
@@ -9,11 +9,15 @@ export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' |
|
||||
// Who is transcribing, for the icon by the timer; movedAt marks when Automatic moved from the phone to the computer.
|
||||
export type MobileDictationEngineShown = { kind: 'phone' | 'computer'; movedAt: number | null }
|
||||
|
||||
// offline: the computer was out of reach when it failed, which is the reason the user needs.
|
||||
export type MobileDictationKeptFailure = { message: string; offline: boolean }
|
||||
|
||||
export type UseMobileDictationOptions = {
|
||||
client: RpcClient | null
|
||||
enabled: boolean
|
||||
onTranscript: (text: string) => void
|
||||
onError?: (error: Error) => void
|
||||
// kept: the recording is kept for Retry, so the failure shows in place rather than as a toast.
|
||||
onError?: (error: Error, outcome: { kept: boolean }) => void
|
||||
// The computer's filler switch, for text the phone recognizes itself; unknown means on.
|
||||
removeFillerWords?: boolean
|
||||
// The host's cloud upload is full; the dictation stops and inserts what was kept.
|
||||
@@ -28,10 +32,13 @@ export type UseMobileDictationResult = {
|
||||
// Empty until a newer host reports words; older hosts never do.
|
||||
transcriptSoFar: string
|
||||
error: string | null
|
||||
// A failure whose recording is kept: Retry plays it to the computer again, cancel discards it.
|
||||
failure: MobileDictationKeptFailure | null
|
||||
// Null until recording starts on an engine.
|
||||
engine: MobileDictationEngineShown | null
|
||||
start: () => Promise<void>
|
||||
stop: () => Promise<void>
|
||||
retry: () => Promise<void>
|
||||
cancel: () => Promise<void>
|
||||
meter: MobileDictationMeter
|
||||
}
|
||||
|
||||
@@ -13,19 +13,19 @@ import {
|
||||
type MobileDictationAudioPhase
|
||||
} from './mobile-dictation-audio-chunk'
|
||||
import { MobileDictationHeldAudio } from './mobile-dictation-held-audio'
|
||||
import { isCurrentMobileDictationFinish } from './mobile-dictation-session-state'
|
||||
import { finishDesktopDictation } from './mobile-dictation-desktop-start'
|
||||
import { MobileDictationRecording } from './mobile-dictation-recording'
|
||||
import { failActiveMobileDictation, retryMobileDictationOnHost } from './mobile-dictation-retry'
|
||||
import { mobileDictationFailureRetryable } from './mobile-dictation-failure'
|
||||
import { finishMobileDictation } from './mobile-dictation-finish'
|
||||
import type { MobileDictationEngine } from './mobile-dictation-device-engine'
|
||||
import {
|
||||
finishMobileDictationOnDevice,
|
||||
releaseMobileDictationEngine,
|
||||
settleMobileDictation,
|
||||
startMobileDictation,
|
||||
type MobileDictationRun
|
||||
} from './mobile-dictation-engine-run'
|
||||
import { rpcPayloadMember } from '../transport/rpc/rpc-reader-payload'
|
||||
import type {
|
||||
DictationStatus,
|
||||
MobileDictationKeptFailure,
|
||||
UseMobileDictationOptions,
|
||||
UseMobileDictationResult
|
||||
} from './mobile-dictation-session-state'
|
||||
@@ -40,6 +40,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
const speech = useOnDeviceSpeech(removeFillerWords)
|
||||
const [status, setStatus] = useState<DictationStatus>('idle')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
// A failure whose recording is kept for Retry; cleared by Retry, discard and the next dictation.
|
||||
const [failure, setFailure] = useState<MobileDictationKeptFailure | null>(null)
|
||||
const failureRef = useRef(failure)
|
||||
const [transcriptSoFar, setTranscriptSoFar, keptTranscript] = useMobileDictationTranscriptSoFar()
|
||||
const {
|
||||
shown: shownEngine,
|
||||
@@ -61,6 +64,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget())
|
||||
const audioPhaseRef = useRef<MobileDictationAudioPhase>('closed')
|
||||
const heldAudioRef = useRef(new MobileDictationHeldAudio())
|
||||
const recordingRef = useRef(new MobileDictationRecording())
|
||||
const sendHeldAudioRef = useRef(() => {})
|
||||
const stopWhenStartedRef = useRef(false)
|
||||
const generationRef = useRef(0)
|
||||
@@ -77,27 +81,36 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
}, [client, speech, enabled, onTranscript, onError, onAudioLimitReached])
|
||||
|
||||
const applyStatus = useCallback(
|
||||
(next: DictationStatus) => {
|
||||
// Kept through processing so the text so far stays in place until the final transcript replaces it in the same render.
|
||||
(next: DictationStatus, keepText = next === 'recording' || next === 'processing') => {
|
||||
statusRef.current = next
|
||||
setStatus(next)
|
||||
// Kept through processing so the text so far stays in place until the final transcript replaces it in the same render.
|
||||
const live = next === 'recording' || next === 'processing'
|
||||
setTranscriptSoFar((text) => (live ? text : ''))
|
||||
if (!live) {
|
||||
setTranscriptSoFar((text) => (keepText ? text : ''))
|
||||
if (next !== 'recording' && next !== 'processing') {
|
||||
clearEngine()
|
||||
}
|
||||
},
|
||||
[clearEngine]
|
||||
)
|
||||
|
||||
const setKeptFailure = useCallback((next: MobileDictationKeptFailure | null) => {
|
||||
failureRef.current = next
|
||||
setFailure(next)
|
||||
}, [])
|
||||
|
||||
const reportError = useCallback(
|
||||
(err: unknown) => {
|
||||
(err: unknown, kept = false) => {
|
||||
const normalized = err instanceof Error ? err : new Error(String(err))
|
||||
setError(normalized.message)
|
||||
applyStatus('error')
|
||||
onErrorRef.current?.(normalized)
|
||||
applyStatus('error', kept)
|
||||
setKeptFailure(
|
||||
kept
|
||||
? { message: normalized.message, offline: clientRef.current?.getState() !== 'connected' }
|
||||
: null
|
||||
)
|
||||
onErrorRef.current?.(normalized, { kept })
|
||||
},
|
||||
[applyStatus]
|
||||
[applyStatus, setKeptFailure]
|
||||
)
|
||||
|
||||
const closeDictationAudio = useCallback(() => {
|
||||
@@ -113,29 +126,8 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
}
|
||||
}, [capture])
|
||||
|
||||
const failActiveDictation = useCallback(
|
||||
(dictationId: string, err: unknown) => {
|
||||
if (activeIdRef.current !== dictationId) {
|
||||
return
|
||||
}
|
||||
activeIdRef.current = null
|
||||
closeDictationAudio()
|
||||
void releaseMobileDictationEngine(
|
||||
{ engine: engineRef, speech: speechRef },
|
||||
clientRef.current,
|
||||
dictationId
|
||||
)
|
||||
const kept = keptTranscript.take(dictationId)
|
||||
if (kept) {
|
||||
onTranscriptRef.current(kept)
|
||||
}
|
||||
reportError(err)
|
||||
},
|
||||
[closeDictationAudio, reportError]
|
||||
)
|
||||
|
||||
const run = useMemo<MobileDictationRun>(
|
||||
() => ({
|
||||
const run = useMemo<MobileDictationRun>(() => {
|
||||
const created: MobileDictationRun = {
|
||||
status: statusRef,
|
||||
activeId: activeIdRef,
|
||||
client: clientRef,
|
||||
@@ -148,6 +140,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
pendingChunks: pendingChunksRef,
|
||||
pendingAudioBudget: pendingAudioBudgetRef,
|
||||
heldAudio: heldAudioRef,
|
||||
recording: recordingRef,
|
||||
sendHeldAudio: sendHeldAudioRef,
|
||||
finish: finishRef,
|
||||
onTranscript: onTranscriptRef,
|
||||
@@ -157,77 +150,17 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
showEngine,
|
||||
closeDictationAudio,
|
||||
reportError,
|
||||
failActiveDictation
|
||||
}),
|
||||
[applyStatus, closeDictationAudio, failActiveDictation, keptTranscript, reportError, showEngine]
|
||||
)
|
||||
failActiveDictation: (dictationId, err) =>
|
||||
failActiveMobileDictation(created, dictationId, err)
|
||||
}
|
||||
return created
|
||||
}, [applyStatus, closeDictationAudio, keptTranscript, reportError, showEngine])
|
||||
|
||||
useMobileDictationAudioChunks(capture, run)
|
||||
|
||||
const finishDictation = useCallback(
|
||||
async (emptyTranscriptMessage: string) => {
|
||||
const client = clientRef.current
|
||||
const dictationId = activeIdRef.current
|
||||
const engine = engineRef.current
|
||||
if (!dictationId || (engine.kind === 'host' && !client)) {
|
||||
return
|
||||
}
|
||||
// A fallback's host start is still pending: finish once it commits and the kept audio has gone.
|
||||
if (engine.kind === 'host' && audioPhaseRef.current === 'holding') {
|
||||
stopWhenStartedRef.current = true
|
||||
if (statusRef.current === 'recording') {
|
||||
applyStatus('processing')
|
||||
// Stop listening now; the tail joins the held audio the replay sends.
|
||||
void capture.end()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const generation = generationRef.current + 1
|
||||
generationRef.current = generation
|
||||
applyStatus('processing')
|
||||
const isCurrent = () =>
|
||||
isCurrentMobileDictationFinish(
|
||||
generationRef.current,
|
||||
generation,
|
||||
enabledRef.current,
|
||||
activeIdRef.current,
|
||||
dictationId
|
||||
)
|
||||
try {
|
||||
// Keep shutdown inside try for finally cleanup; accept its tail chunks and snapshot pending sends only after end settles, or finish can overtake the last audio.
|
||||
await capture.end()
|
||||
if (engine.kind === 'device') {
|
||||
await finishMobileDictationOnDevice(
|
||||
run,
|
||||
engine,
|
||||
dictationId,
|
||||
emptyTranscriptMessage,
|
||||
isCurrent
|
||||
)
|
||||
return
|
||||
}
|
||||
audioPhaseRef.current = 'closed'
|
||||
await Promise.allSettled(Array.from(pendingChunksRef.current))
|
||||
if (!isCurrent() || !client) {
|
||||
return
|
||||
}
|
||||
const sentSeconds = pendingAudioBudgetRef.current.sentAudioSeconds
|
||||
const finished = await finishDesktopDictation(client, dictationId, sentSeconds)
|
||||
if (!isCurrent()) {
|
||||
return
|
||||
}
|
||||
const transcript = rpcPayloadMember(finished, 'text')
|
||||
settleMobileDictation(
|
||||
run,
|
||||
typeof transcript === 'string' ? transcript.trim() : '',
|
||||
emptyTranscriptMessage
|
||||
)
|
||||
} catch (err) {
|
||||
failActiveDictation(dictationId, err)
|
||||
}
|
||||
},
|
||||
[applyStatus, capture, failActiveDictation, run]
|
||||
(emptyTranscriptMessage: string) => finishMobileDictation(run, capture, emptyTranscriptMessage),
|
||||
[capture, run]
|
||||
)
|
||||
useLayoutEffect(() => {
|
||||
finishRef.current = finishDictation
|
||||
@@ -247,8 +180,21 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
return
|
||||
}
|
||||
setError(null)
|
||||
// A new dictation replaces a failed one's kept recording.
|
||||
setKeptFailure(null)
|
||||
recordingRef.current.clear()
|
||||
await startMobileDictation(run, capture)
|
||||
}, [capture, run])
|
||||
}, [capture, run, setKeptFailure])
|
||||
|
||||
const retry = useCallback(async () => {
|
||||
const client = clientRef.current
|
||||
if (!failureRef.current || !client || activeIdRef.current || !enabledRef.current) {
|
||||
return
|
||||
}
|
||||
setError(null)
|
||||
setKeptFailure(null)
|
||||
await retryMobileDictationOnHost(run, client)
|
||||
}, [run, setKeptFailure])
|
||||
|
||||
// Only the user’s own cancel (reason:null) ends silently; other interruptions call onError so the tap has a visible outcome.
|
||||
const abandonDictation = useCallback(
|
||||
@@ -256,11 +202,26 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
const client = clientRef.current
|
||||
const dictationId = activeIdRef.current
|
||||
const wasUnderway = statusRef.current === 'starting' || statusRef.current === 'recording'
|
||||
// Only the user's discard drops a kept recording; an interruption while it waits leaves it for Retry.
|
||||
if (reason !== null && dictationId === null && failureRef.current) {
|
||||
return
|
||||
}
|
||||
// An interruption keeps a recording the computer can hear again, even mid-finish; the user's cancel never does.
|
||||
const kept =
|
||||
reason !== null &&
|
||||
dictationId !== null &&
|
||||
mobileDictationFailureRetryable(recordingRef.current, engineRef.current)
|
||||
if (!kept) {
|
||||
setKeptFailure(null)
|
||||
recordingRef.current.clear()
|
||||
}
|
||||
generationRef.current += 1
|
||||
activeIdRef.current = null
|
||||
closeDictationAudio()
|
||||
// Settle before the host's cancel reply: a host that went quiet must not hold the mic button in its old state.
|
||||
if (reason !== null && wasUnderway) {
|
||||
if (kept) {
|
||||
reportError(new Error(reason), true)
|
||||
} else if (reason !== null && wasUnderway) {
|
||||
reportError(new Error(reason))
|
||||
} else {
|
||||
applyStatus('idle')
|
||||
@@ -268,7 +229,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
}
|
||||
await releaseMobileDictationEngine(run, client, dictationId)
|
||||
},
|
||||
[applyStatus, closeDictationAudio, reportError, run]
|
||||
[applyStatus, closeDictationAudio, reportError, run, setKeptFailure]
|
||||
)
|
||||
|
||||
const cancel = useCallback(() => abandonDictation(null), [abandonDictation])
|
||||
@@ -295,6 +256,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
generationRef.current += 1
|
||||
activeIdRef.current = null
|
||||
closeDictationAudio()
|
||||
recordingRef.current.clear()
|
||||
capture.release()
|
||||
void releaseMobileDictationEngine(run, clientRef.current, dictationId)
|
||||
}
|
||||
@@ -307,8 +269,10 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
isProcessing: status === 'processing',
|
||||
transcriptSoFar,
|
||||
error,
|
||||
failure,
|
||||
engine: shownEngine,
|
||||
start,
|
||||
retry,
|
||||
stop,
|
||||
cancel,
|
||||
meter: useMobileDictationMeter(capture, status === 'recording')
|
||||
|
||||
@@ -1317,13 +1317,23 @@
|
||||
"durationSeconds_one": "{{count}} Sekunde",
|
||||
"durationSeconds_other": "{{count}} Sekunden",
|
||||
"durationMinutes_one": "{{count}} Minute",
|
||||
"durationMinutes_other": "{{count}} Minuten"
|
||||
"durationMinutes_other": "{{count}} Minuten",
|
||||
"discard": "Aufnahme verwerfen",
|
||||
"retry": "Diktat wiederholen",
|
||||
"retryShort": "Wiederholen"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "Wird auf {{computer}} abgeschlossen",
|
||||
"finishingOnComputer": "Wird auf dem Computer abgeschlossen",
|
||||
"movedTo": "Zu {{computer}} gewechselt",
|
||||
"movedToComputer": "Zum Computer gewechselt"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "Verbindung unterbrochen",
|
||||
"connectionSlow": "Verbindung zu langsam",
|
||||
"phoneStopped": "Auf diesem Telefon gestoppt",
|
||||
"stopped": "Diktat gestoppt",
|
||||
"recordingKept": "Aufnahme behalten"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -534,13 +534,23 @@
|
||||
"meterOnPhone": "Recording, {{duration}}, on this phone",
|
||||
"meterOn": "Recording, {{duration}}, on {{computer}}",
|
||||
"meterOnComputer": "Recording, {{duration}}, on the computer",
|
||||
"meter": "Recording, {{duration}}"
|
||||
"meter": "Recording, {{duration}}",
|
||||
"discard": "Discard recording",
|
||||
"retry": "Retry dictation",
|
||||
"retryShort": "Retry"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "Finishing on {{computer}}",
|
||||
"finishingOnComputer": "Finishing on the computer",
|
||||
"movedTo": "Moved to {{computer}}",
|
||||
"movedToComputer": "Moved to the computer"
|
||||
},
|
||||
"failure": {
|
||||
"recordingKept": "Recording kept",
|
||||
"connectionLost": "Connection lost",
|
||||
"connectionSlow": "Connection too slow",
|
||||
"phoneStopped": "Stopped on this phone",
|
||||
"stopped": "Dictation stopped"
|
||||
}
|
||||
},
|
||||
"home": {
|
||||
|
||||
@@ -1358,13 +1358,23 @@
|
||||
"durationSeconds_many": "{{count}} segundos",
|
||||
"durationMinutes_one": "{{count}} minuto",
|
||||
"durationMinutes_other": "{{count}} minutos",
|
||||
"durationMinutes_many": "{{count}} minutos"
|
||||
"durationMinutes_many": "{{count}} minutos",
|
||||
"discard": "Descartar grabación",
|
||||
"retry": "Reintentar dictado",
|
||||
"retryShort": "Reintentar"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "Terminando en {{computer}}",
|
||||
"finishingOnComputer": "Terminando en el ordenador",
|
||||
"movedTo": "Pasado a {{computer}}",
|
||||
"movedToComputer": "Pasado al ordenador"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "Conexión perdida",
|
||||
"connectionSlow": "Conexión demasiado lenta",
|
||||
"phoneStopped": "Detenido en este teléfono",
|
||||
"stopped": "Dictado detenido",
|
||||
"recordingKept": "Grabación guardada"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1358,13 +1358,23 @@
|
||||
"durationSeconds_many": "{{count}} secondes",
|
||||
"durationMinutes_one": "{{count}} minute",
|
||||
"durationMinutes_other": "{{count}} minutes",
|
||||
"durationMinutes_many": "{{count}} minutes"
|
||||
"durationMinutes_many": "{{count}} minutes",
|
||||
"discard": "Supprimer l’enregistrement",
|
||||
"retry": "Relancer la dictée",
|
||||
"retryShort": "Réessayer"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "Finalisation sur {{computer}}",
|
||||
"finishingOnComputer": "Finalisation sur l’ordinateur",
|
||||
"movedTo": "Passé sur {{computer}}",
|
||||
"movedToComputer": "Passé sur l’ordinateur"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "Connexion perdue",
|
||||
"connectionSlow": "Connexion trop lente",
|
||||
"phoneStopped": "Arrêté sur ce téléphone",
|
||||
"stopped": "Dictée arrêtée",
|
||||
"recordingKept": "Enregistrement conservé"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1313,13 +1313,23 @@
|
||||
"durationSeconds_one": "{{count}} 秒",
|
||||
"durationSeconds_other": "{{count}} 秒",
|
||||
"durationMinutes_one": "{{count}} 分",
|
||||
"durationMinutes_other": "{{count}} 分"
|
||||
"durationMinutes_other": "{{count}} 分",
|
||||
"discard": "録音を破棄",
|
||||
"retry": "音声入力を再試行",
|
||||
"retryShort": "再試行"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "{{computer}} で仕上げています",
|
||||
"finishingOnComputer": "コンピュータで仕上げています",
|
||||
"movedTo": "{{computer}} に切り替えました",
|
||||
"movedToComputer": "コンピュータに切り替えました"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "接続が切れました",
|
||||
"connectionSlow": "接続が遅すぎます",
|
||||
"phoneStopped": "このスマートフォンで停止しました",
|
||||
"stopped": "音声入力が停止しました",
|
||||
"recordingKept": "録音は保持されています"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1313,13 +1313,23 @@
|
||||
"durationSeconds_one": "{{count}}초",
|
||||
"durationSeconds_other": "{{count}}초",
|
||||
"durationMinutes_one": "{{count}}분",
|
||||
"durationMinutes_other": "{{count}}분"
|
||||
"durationMinutes_other": "{{count}}분",
|
||||
"discard": "녹음 삭제",
|
||||
"retry": "받아쓰기 다시 시도",
|
||||
"retryShort": "다시 시도"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "{{computer}}에서 마무리 중",
|
||||
"finishingOnComputer": "컴퓨터에서 마무리 중",
|
||||
"movedTo": "{{computer}}(으)로 전환됨",
|
||||
"movedToComputer": "컴퓨터로 전환됨"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "연결 끊김",
|
||||
"connectionSlow": "연결이 너무 느림",
|
||||
"phoneStopped": "이 휴대폰에서 중지됨",
|
||||
"stopped": "받아쓰기 중지됨",
|
||||
"recordingKept": "녹음 보관됨"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1358,13 +1358,23 @@
|
||||
"durationSeconds_many": "{{count}} segundos",
|
||||
"durationMinutes_one": "{{count}} minuto",
|
||||
"durationMinutes_other": "{{count}} minutos",
|
||||
"durationMinutes_many": "{{count}} minutos"
|
||||
"durationMinutes_many": "{{count}} minutos",
|
||||
"discard": "Descartar gravação",
|
||||
"retry": "Tentar ditado novamente",
|
||||
"retryShort": "Tentar novamente"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "Finalizando em {{computer}}",
|
||||
"finishingOnComputer": "Finalizando no computador",
|
||||
"movedTo": "Passou para {{computer}}",
|
||||
"movedToComputer": "Passou para o computador"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "Conexão perdida",
|
||||
"connectionSlow": "Conexão muito lenta",
|
||||
"phoneStopped": "Parou neste telefone",
|
||||
"stopped": "Ditado interrompido",
|
||||
"recordingKept": "Gravação mantida"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1399,13 +1399,23 @@
|
||||
"durationMinutes_one": "{{count}} минута",
|
||||
"durationMinutes_other": "{{count}} минуты",
|
||||
"durationMinutes_few": "{{count}} минуты",
|
||||
"durationMinutes_many": "{{count}} минут"
|
||||
"durationMinutes_many": "{{count}} минут",
|
||||
"discard": "Удалить запись",
|
||||
"retry": "Повторить диктовку",
|
||||
"retryShort": "Повторить"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "Завершается на {{computer}}",
|
||||
"finishingOnComputer": "Завершается на компьютере",
|
||||
"movedTo": "Перешло на {{computer}}",
|
||||
"movedToComputer": "Перешло на компьютер"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "Соединение потеряно",
|
||||
"connectionSlow": "Соединение слишком медленное",
|
||||
"phoneStopped": "Остановлено на этом телефоне",
|
||||
"stopped": "Диктовка остановлена",
|
||||
"recordingKept": "Запись сохранена"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1317,13 +1317,23 @@
|
||||
"durationSeconds_one": "{{count}} saniye",
|
||||
"durationSeconds_other": "{{count}} saniye",
|
||||
"durationMinutes_one": "{{count}} dakika",
|
||||
"durationMinutes_other": "{{count}} dakika"
|
||||
"durationMinutes_other": "{{count}} dakika",
|
||||
"discard": "Kaydı sil",
|
||||
"retry": "Dikteyi yeniden dene",
|
||||
"retryShort": "Yeniden dene"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "{{computer}} üzerinde tamamlanıyor",
|
||||
"finishingOnComputer": "Bilgisayarda tamamlanıyor",
|
||||
"movedTo": "{{computer}} cihazına geçildi",
|
||||
"movedToComputer": "Bilgisayara geçildi"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "Bağlantı kesildi",
|
||||
"connectionSlow": "Bağlantı çok yavaş",
|
||||
"phoneStopped": "Bu telefonda durdu",
|
||||
"stopped": "Dikte durdu",
|
||||
"recordingKept": "Kayıt saklandı"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1313,13 +1313,23 @@
|
||||
"durationSeconds_one": "{{count}} 秒",
|
||||
"durationSeconds_other": "{{count}} 秒",
|
||||
"durationMinutes_one": "{{count}} 分鐘",
|
||||
"durationMinutes_other": "{{count}} 分鐘"
|
||||
"durationMinutes_other": "{{count}} 分鐘",
|
||||
"discard": "捨棄錄音",
|
||||
"retry": "重試聽寫",
|
||||
"retryShort": "重試"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "正在 {{computer}} 上完成",
|
||||
"finishingOnComputer": "正在電腦上完成",
|
||||
"movedTo": "已轉到 {{computer}}",
|
||||
"movedToComputer": "已轉到電腦"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "連線中斷",
|
||||
"connectionSlow": "連線太慢",
|
||||
"phoneStopped": "已在此手機上停止",
|
||||
"stopped": "聽寫已停止",
|
||||
"recordingKept": "錄音已保留"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -1313,13 +1313,23 @@
|
||||
"durationSeconds_one": "{{count}} 秒",
|
||||
"durationSeconds_other": "{{count}} 秒",
|
||||
"durationMinutes_one": "{{count}} 分钟",
|
||||
"durationMinutes_other": "{{count}} 分钟"
|
||||
"durationMinutes_other": "{{count}} 分钟",
|
||||
"discard": "丢弃录音",
|
||||
"retry": "重试听写",
|
||||
"retryShort": "重试"
|
||||
},
|
||||
"status": {
|
||||
"finishingOn": "正在 {{computer}} 上完成",
|
||||
"finishingOnComputer": "正在电脑上完成",
|
||||
"movedTo": "已转到 {{computer}}",
|
||||
"movedToComputer": "已转到电脑"
|
||||
},
|
||||
"failure": {
|
||||
"connectionLost": "连接已断开",
|
||||
"connectionSlow": "连接太慢",
|
||||
"phoneStopped": "已在此手机上停止",
|
||||
"stopped": "听写已停止",
|
||||
"recordingKept": "录音已保留"
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
|
||||
@@ -345,7 +345,7 @@ export function MobileNativeChatComposer({
|
||||
/>
|
||||
<View style={inputStyles.actionRow} testID="native-chat-composer-actions">
|
||||
<MobileNativeChatDictationLeading
|
||||
{...{ micActive, micStarting, micProcessing, cancelRef }}
|
||||
{...{ micActive, micStarting, micProcessing, cancelRef, trailing: contextRing.ring }}
|
||||
>
|
||||
<MobileNativeChatComposerAttachButton
|
||||
onAttachImage={onAttachImage}
|
||||
@@ -373,7 +373,6 @@ export function MobileNativeChatComposer({
|
||||
<MobileNativeChatComposerPlanChip options={sessionOptions} inputRef={fieldRef} />
|
||||
{goal.chip}
|
||||
</MobileNativeChatDictationLeading>
|
||||
{micActive || micProcessing ? null : contextRing.ring}
|
||||
{onMicPress ? (
|
||||
<MobileNativeChatComposerMic
|
||||
{...{ onMicPress, micActive, micStarting, micProcessing, dictationMode }}
|
||||
|
||||
@@ -2,7 +2,10 @@ import { Pressable, type View } from 'react-native'
|
||||
import { useContext, type RefObject } from 'react'
|
||||
import { MobileDictationLiveContext } from '../../../dictation/mobile-dictation-live-context'
|
||||
import { dictationControlsCopy } from '../../../dictation/mobile-dictation-controls-copy'
|
||||
import { MobileDictationMicIcon } from '../../../dictation/MobileDictationLiveControls'
|
||||
import {
|
||||
MobileDictationMicIcon,
|
||||
MobileDictationRetryButton
|
||||
} from '../../../dictation/MobileDictationLiveControls'
|
||||
import { useMobileDictationCancelDrop } from '../../../dictation/use-mobile-dictation-cancel-drop'
|
||||
import { useMobileTranslation } from '../../../i18n/use-mobile-translation'
|
||||
import { keepHeldPressThroughLongPress } from '../../../platform/held-press-long-press'
|
||||
@@ -43,6 +46,9 @@ export function MobileNativeChatComposerMic({
|
||||
useMobileTranslation()
|
||||
const live = useContext(MobileDictationLiveContext)
|
||||
const release = useMobileDictationCancelDrop(cancelRef, live?.cancel, onMicPressOut)
|
||||
if (live?.failureReason != null) {
|
||||
return <MobileDictationRetryButton onRetry={live.retry} labelled />
|
||||
}
|
||||
return (
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
} from './MobileNativeChatSendButton'
|
||||
import type { MobileNativeChatSteerHead } from './use-mobile-native-chat-composer-tray'
|
||||
import { useMobileNativeChatSteerChordPreview } from './use-mobile-native-chat-steer-chord-preview'
|
||||
import { useMobileNativeChatDictationFailed } from './MobileNativeChatDictationLeading'
|
||||
|
||||
/** Send's slot outside a queued edit: Send, Stop where Send can't deliver, or what a held Cmd/Ctrl would do. */
|
||||
export function MobileNativeChatComposerSendSlot({
|
||||
@@ -51,7 +52,8 @@ export function MobileNativeChatComposerSendSlot({
|
||||
attaching: boolean
|
||||
stop: { onStop?: () => void; held: boolean } | null
|
||||
onSend: (intent?: MobileNativeChatSendIntent, keepFocus?: boolean) => void
|
||||
}): React.JSX.Element {
|
||||
}): React.JSX.Element | null {
|
||||
const dictationFailed = useMobileNativeChatDictationFailed()
|
||||
const steerChord = useMobileNativeChatSteerChordPreview({
|
||||
fieldRef,
|
||||
hasDraft: value.trim() !== '' || attachmentCount > 0,
|
||||
@@ -83,6 +85,10 @@ export function MobileNativeChatComposerSendSlot({
|
||||
draft: value,
|
||||
attachmentCount
|
||||
})
|
||||
// A dictation kept for Retry takes Send's room; a running agent keeps its Stop.
|
||||
if (dictationFailed && !(showsStop && stop)) {
|
||||
return null
|
||||
}
|
||||
if (steerChord.preview && (steerChord.preview === 'steer-queued' || canSend)) {
|
||||
// As desktop, the preview takes Send's slot at once, with no ease.
|
||||
return (
|
||||
|
||||
@@ -3,28 +3,45 @@ import { View } from 'react-native'
|
||||
import { MobileDictationLiveContext } from '../../../dictation/mobile-dictation-live-context'
|
||||
import {
|
||||
MobileDictationCancelButton,
|
||||
MobileDictationFailureNote,
|
||||
MobileDictationFinishingNote,
|
||||
MobileDictationRecordingMeter
|
||||
} from '../../../dictation/MobileDictationLiveControls'
|
||||
import { mobileNativeChatComposerStyles as styles } from './mobile-native-chat-composer-styles'
|
||||
|
||||
// While dictating, the row's leading side swaps attach and the pickers for ✕ and the meter, so the composer keeps its height.
|
||||
// While dictating, the row's leading side swaps attach and the pickers for ✕ and the meter (or a kept failure), so the composer keeps its height.
|
||||
export function MobileNativeChatDictationLeading({
|
||||
micActive,
|
||||
micStarting = false,
|
||||
micProcessing = false,
|
||||
cancelRef,
|
||||
trailing,
|
||||
children
|
||||
}: {
|
||||
micActive: boolean
|
||||
micStarting?: boolean
|
||||
micProcessing?: boolean
|
||||
cancelRef: RefObject<View | null>
|
||||
// The context ring after the leading controls; it steps aside with them.
|
||||
trailing: ReactNode
|
||||
children: ReactNode
|
||||
}): React.JSX.Element {
|
||||
const live = useContext(MobileDictationLiveContext)
|
||||
if (live?.failureReason != null) {
|
||||
return (
|
||||
<View style={[styles.leadingControls, styles.dictationLeading]}>
|
||||
<MobileDictationCancelButton onCancel={live.cancel} discards />
|
||||
<MobileDictationFailureNote reason={live.failureReason} />
|
||||
</View>
|
||||
)
|
||||
}
|
||||
if (!live || (!micActive && !micProcessing)) {
|
||||
return <View style={styles.leadingControls}>{children}</View>
|
||||
return (
|
||||
<>
|
||||
<View style={styles.leadingControls}>{children}</View>
|
||||
{trailing}
|
||||
</>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<View style={[styles.leadingControls, styles.dictationLeading]}>
|
||||
@@ -42,3 +59,8 @@ export function MobileNativeChatDictationLeading({
|
||||
</View>
|
||||
)
|
||||
}
|
||||
|
||||
// A dictation failed with its recording kept: Send gives its room to Retry.
|
||||
export function useMobileNativeChatDictationFailed(): boolean {
|
||||
return useContext(MobileDictationLiveContext)?.failureReason != null
|
||||
}
|
||||
|
||||
@@ -166,7 +166,12 @@ export function useMobileSessionNativeChatDictation(
|
||||
}
|
||||
})().catch(() => keepUndeliveredDictation(text))
|
||||
},
|
||||
onError: (err) => {
|
||||
onError: (err, { kept }) => {
|
||||
// A kept failure shows in place with Retry, which still needs the dictation's target.
|
||||
if (kept) {
|
||||
triggerError()
|
||||
return
|
||||
}
|
||||
dictationRouteContextRef.current = null
|
||||
reportDictationFailure(err)
|
||||
},
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { RefObject } from 'react'
|
||||
import { useContext, type RefObject } from 'react'
|
||||
import {
|
||||
ActivityIndicator,
|
||||
Pressable,
|
||||
@@ -17,7 +17,11 @@ import { useMobileColorScheme } from '../../theme/use-mobile-color-scheme'
|
||||
import { keepHeldPressThroughLongPress } from '../../platform/held-press-long-press'
|
||||
import { dictationControlsCopy } from '../../dictation/mobile-dictation-controls-copy'
|
||||
import { useMobileDictationCancelDrop } from '../../dictation/use-mobile-dictation-cancel-drop'
|
||||
import { MobileDictationMicIcon } from '../../dictation/MobileDictationLiveControls'
|
||||
import {
|
||||
MobileDictationMicIcon,
|
||||
MobileDictationRetryButton
|
||||
} from '../../dictation/MobileDictationLiveControls'
|
||||
import { MobileDictationLiveContext } from '../../dictation/mobile-dictation-live-context'
|
||||
|
||||
type DictationState = {
|
||||
readonly isStarting: boolean
|
||||
@@ -54,11 +58,20 @@ type MobileTerminalInputActionsProps = MobileTerminalDictationButtonProps &
|
||||
|
||||
export function MobileTerminalInputActions(props: MobileTerminalInputActionsProps) {
|
||||
const { isStarting, isRecording, isProcessing } = props.dictation
|
||||
// Attach steps aside while dictating so the words get the field's room.
|
||||
const live = useContext(MobileDictationLiveContext)
|
||||
// A kept failure is cleared by every start and Retry, so it never shows while dictating.
|
||||
const failureReason = live?.failureReason ?? null
|
||||
// Attach steps aside while dictating or holding a failed recording, so the field gets the room.
|
||||
return (
|
||||
<>
|
||||
{isStarting || isRecording || isProcessing ? null : <MobileTerminalAttachButton {...props} />}
|
||||
<MobileTerminalDictationButton {...props} />
|
||||
{isStarting || isRecording || isProcessing || failureReason !== null ? null : (
|
||||
<MobileTerminalAttachButton {...props} />
|
||||
)}
|
||||
{failureReason !== null && live ? (
|
||||
<MobileDictationRetryButton onRetry={live.retry} labelled={false} />
|
||||
) : (
|
||||
<MobileTerminalDictationButton {...props} />
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
import { MobileDictationLiveContext } from '../../dictation/mobile-dictation-live-context'
|
||||
import {
|
||||
MobileDictationCancelButton,
|
||||
MobileDictationFailureNote,
|
||||
MobileDictationFinishingNote,
|
||||
MobileDictationRecordingMeter
|
||||
} from '../../dictation/MobileDictationLiveControls'
|
||||
@@ -37,6 +38,16 @@ export function MobileTerminalLiveInputStatus({
|
||||
const t = useMobileTranslation()
|
||||
const live = useContext(MobileDictationLiveContext)
|
||||
const dictationTitle = mobileDictationStatusTitle(dictation)
|
||||
if (live?.failureReason != null) {
|
||||
return (
|
||||
<View style={[styles.status, styles.dictating]} pointerEvents="box-none">
|
||||
<MobileDictationCancelButton onCancel={live.cancel} compact discards />
|
||||
<View style={styles.dictationLine} pointerEvents="none">
|
||||
<MobileDictationFailureNote reason={live.failureReason} compact />
|
||||
</View>
|
||||
</View>
|
||||
)
|
||||
}
|
||||
if (!dictationTitle) {
|
||||
return isAttaching ? (
|
||||
<View style={styles.status} pointerEvents="none">
|
||||
|
||||
Reference in New Issue
Block a user