diff --git a/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs b/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs index 4a87546e66a..1e391c41ccc 100644 --- a/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs +++ b/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs @@ -102,12 +102,13 @@ export const MOBILE_WEB_PAGE_ROUTES = [ 'native.audio.read', 'native.audio.stop' ], - // Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act. + // Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act; dictation uses the computer without native.speech.act. optionalGrants: [ 'externalNavigation', 'native.media.name', 'native.pdf.view', 'native.image.act', + 'native.speech.act', 'terminalKeyRail', DEVICE_STORE_GRANT ] diff --git a/mobile/app.json b/mobile/app.json index 61e54d2421c..8158d4193f5 100644 --- a/mobile/app.json +++ b/mobile/app.json @@ -33,7 +33,7 @@ "infoPlist": { "CFBundleAllowMixedLocalizations": true, "NSLocalNetworkUsageDescription": "Orca connects to the desktop app on your LAN.", - "NSMicrophoneUsageDescription": "Allow Orca to record voice dictation and transcribe it on your paired desktop.", + "NSMicrophoneUsageDescription": "Allow Orca to use your microphone for voice dictation, transcribed on this phone or on your paired computer.", "NSPhotoLibraryUsageDescription": "Allow Orca to attach photos from your library to a terminal session on your paired desktop.", "NSPhotoLibraryAddUsageDescription": "Allow Orca to save images from your sessions to Photos.", "NSAppTransportSecurity": { diff --git a/mobile/locales/de.json b/mobile/locales/de.json index 903b85c5d3c..6116f94c3f0 100644 --- a/mobile/locales/de.json +++ b/mobile/locales/de.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Orca erlauben, die Kamera zum Scannen von QR-Codes zu verwenden.", "NSLocalNetworkUsageDescription": "Orca verbindet sich über das LAN mit der Desktop-App.", - "NSMicrophoneUsageDescription": "Orca erlauben, das Mikrofon für Diktate zu verwenden, die zur Transkription an den gekoppelten Desktop gesendet werden.", + "NSMicrophoneUsageDescription": "Erlaube Orca, dein Mikrofon für Sprachdiktat zu verwenden. Es wird auf diesem Telefon oder auf deinem gekoppelten Computer transkribiert.", "NSPhotoLibraryAddUsageDescription": "Orca erlauben, Bilder aus deinen Sitzungen in Fotos zu sichern.", "NSPhotoLibraryUsageDescription": "Orca erlauben, Fotos aus der Mediathek an eine Terminal-Sitzung auf dem gekoppelten Desktop anzuhängen." } diff --git a/mobile/locales/en.json b/mobile/locales/en.json index 00960fe3901..e14752e5d8a 100644 --- a/mobile/locales/en.json +++ b/mobile/locales/en.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Allow Orca to use the camera for QR code scanning.", "NSLocalNetworkUsageDescription": "Orca connects to the desktop app on your LAN.", - "NSMicrophoneUsageDescription": "Allow Orca to use your microphone for voice dictation sent to your paired desktop for transcription.", + "NSMicrophoneUsageDescription": "Allow Orca to use your microphone for voice dictation, transcribed on this phone or on your paired computer.", "NSPhotoLibraryAddUsageDescription": "Allow Orca to save images from your sessions to Photos.", "NSPhotoLibraryUsageDescription": "Allow Orca to attach photos from your library to a terminal session on your paired desktop." } diff --git a/mobile/locales/es.json b/mobile/locales/es.json index f421bf50e34..f06582fa503 100644 --- a/mobile/locales/es.json +++ b/mobile/locales/es.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Permite que Orca use la cámara para escanear códigos QR.", "NSLocalNetworkUsageDescription": "Orca se conecta a la aplicación de escritorio en tu LAN.", - "NSMicrophoneUsageDescription": "Permite que Orca use tu micrófono para el dictado de voz y lo envíe a tu escritorio emparejado para transcribirlo.", + "NSMicrophoneUsageDescription": "Permite que Orca use tu micrófono para el dictado por voz, transcrito en este teléfono o en tu ordenador vinculado.", "NSPhotoLibraryAddUsageDescription": "Permite que Orca guarde imágenes de tus sesiones en Fotos.", "NSPhotoLibraryUsageDescription": "Permite que Orca adjunte fotos de tu fototeca a una sesión de terminal en tu escritorio emparejado." } diff --git a/mobile/locales/fr.json b/mobile/locales/fr.json index 9718e893edd..bf92174a56e 100644 --- a/mobile/locales/fr.json +++ b/mobile/locales/fr.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Autorisez Orca à utiliser l'appareil photo pour scanner des QR codes.", "NSLocalNetworkUsageDescription": "Orca se connecte à l'application sur votre ordinateur via votre LAN.", - "NSMicrophoneUsageDescription": "Autorisez Orca à utiliser votre microphone pour la dictée vocale transmise à votre ordinateur appairé pour transcription.", + "NSMicrophoneUsageDescription": "Autorisez Orca à utiliser votre micro pour la dictée vocale, transcrite sur ce téléphone ou sur votre ordinateur associé.", "NSPhotoLibraryAddUsageDescription": "Autorisez Orca à enregistrer des images de vos sessions dans Photos.", "NSPhotoLibraryUsageDescription": "Autorisez Orca à joindre des photos de votre photothèque à une session de terminal sur votre ordinateur appairé." } diff --git a/mobile/locales/ja.json b/mobile/locales/ja.json index b6e2424fce9..f94a1c34ad0 100644 --- a/mobile/locales/ja.json +++ b/mobile/locales/ja.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "QR コードをスキャンするため、Orca にカメラの使用を許可してください。", "NSLocalNetworkUsageDescription": "Orca は LAN 上のデスクトップアプリに接続します。", - "NSMicrophoneUsageDescription": "音声ディクテーションのため、Orca にマイクの使用を許可してください。音声はペアリング済みのデスクトップに送信され、文字起こしされます。", + "NSMicrophoneUsageDescription": "音声入力のためにマイクを使用します。文字起こしはこの端末またはペアリングしたコンピュータで行われます。", "NSPhotoLibraryAddUsageDescription": "セッションの画像を写真に保存するため、Orca に写真への追加を許可してください。", "NSPhotoLibraryUsageDescription": "ペアリング済みのデスクトップのターミナルセッションに写真を添付するため、Orca に写真ライブラリへのアクセスを許可してください。" } diff --git a/mobile/locales/ko.json b/mobile/locales/ko.json index c55673bfdb7..2e2e6252cf8 100644 --- a/mobile/locales/ko.json +++ b/mobile/locales/ko.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Orca가 QR 코드를 스캔할 수 있도록 카메라 사용을 허용하세요.", "NSLocalNetworkUsageDescription": "Orca는 LAN을 통해 데스크톱 앱에 연결합니다.", - "NSMicrophoneUsageDescription": "Orca가 음성 받아쓰기에 마이크를 사용할 수 있도록 허용하세요. 음성은 텍스트로 변환하기 위해 페어링된 데스크톱으로 전송됩니다.", + "NSMicrophoneUsageDescription": "음성 받아쓰기에 마이크를 사용하도록 Orca를 허용합니다. 받아쓰기는 이 휴대폰 또는 페어링된 컴퓨터에서 이루어집니다.", "NSPhotoLibraryAddUsageDescription": "세션의 이미지를 사진 앱에 저장할 수 있도록 Orca를 허용하세요.", "NSPhotoLibraryUsageDescription": "Orca가 사진 보관함의 사진을 페어링된 데스크톱의 터미널 세션에 첨부할 수 있도록 허용하세요." } diff --git a/mobile/locales/pt-BR.json b/mobile/locales/pt-BR.json index fcd1e66028f..bb22de0ddf1 100644 --- a/mobile/locales/pt-BR.json +++ b/mobile/locales/pt-BR.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Permita que o Orca use a câmera para ler códigos QR.", "NSLocalNetworkUsageDescription": "O Orca se conecta ao aplicativo para desktop na sua rede local (LAN).", - "NSMicrophoneUsageDescription": "Permita que o Orca use seu microfone para ditado por voz, que será enviado ao seu desktop pareado para transcrição.", + "NSMicrophoneUsageDescription": "Permita que o Orca use seu microfone para ditado por voz, transcrito neste celular ou no seu computador pareado.", "NSPhotoLibraryAddUsageDescription": "Permita que o Orca salve imagens das suas sessões no app Fotos.", "NSPhotoLibraryUsageDescription": "Permita que o Orca anexe fotos da sua biblioteca a uma sessão de terminal no seu desktop pareado." } diff --git a/mobile/locales/ru.json b/mobile/locales/ru.json index d241bb112e4..541440aceab 100644 --- a/mobile/locales/ru.json +++ b/mobile/locales/ru.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Разрешите Orca использовать камеру для сканирования QR-кодов.", "NSLocalNetworkUsageDescription": "Orca подключается к настольному приложению в вашей локальной сети.", - "NSMicrophoneUsageDescription": "Разрешите Orca использовать микрофон для диктовки. Запись отправляется на связанный компьютер для расшифровки.", + "NSMicrophoneUsageDescription": "Разрешите Orca использовать микрофон для голосовой диктовки. Речь распознаётся на этом телефоне или на связанном компьютере.", "NSPhotoLibraryAddUsageDescription": "Разрешите Orca сохранять изображения из ваших сессий в Фото.", "NSPhotoLibraryUsageDescription": "Разрешите Orca прикреплять фотографии из медиатеки к сессии терминала на связанном компьютере." } diff --git a/mobile/locales/tr.json b/mobile/locales/tr.json index 29dfde3e5de..b94e6fcf0a0 100644 --- a/mobile/locales/tr.json +++ b/mobile/locales/tr.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "Orca'nın QR kodlarını taramak için kamerayı kullanmasına izin verin.", "NSLocalNetworkUsageDescription": "Orca, yerel ağınızdaki masaüstü uygulamasına bağlanır.", - "NSMicrophoneUsageDescription": "Orca'nın sesli dikte için mikrofonunuzu kullanmasına izin verin. Ses kaydı, yazıya dönüştürülmek üzere eşleştirdiğiniz masaüstü bilgisayara gönderilir.", + "NSMicrophoneUsageDescription": "Orca’nın sesli dikte için mikrofonunuzu kullanmasına izin verin; konuşma bu telefonda veya eşlenmiş bilgisayarınızda yazıya dökülür.", "NSPhotoLibraryAddUsageDescription": "Orca'nın oturumlarınızdaki görselleri Fotoğraflar'a kaydetmesine izin verin.", "NSPhotoLibraryUsageDescription": "Orca'nın arşivinizdeki fotoğrafları eşleştirdiğiniz masaüstü bilgisayardaki bir terminal oturumuna eklemesine izin verin." } diff --git a/mobile/locales/zh-Hans.json b/mobile/locales/zh-Hans.json index 81af2a01ced..e903a484e15 100644 --- a/mobile/locales/zh-Hans.json +++ b/mobile/locales/zh-Hans.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "允许 Orca 使用相机扫描二维码。", "NSLocalNetworkUsageDescription": "Orca 会连接到局域网内的桌面应用。", - "NSMicrophoneUsageDescription": "允许 Orca 使用麦克风进行语音听写，录音将发送到已配对的桌面进行转写。", + "NSMicrophoneUsageDescription": "允许 Orca 使用麦克风进行语音听写，在此手机或已配对的电脑上转写。", "NSPhotoLibraryAddUsageDescription": "允许 Orca 将会话中的图片存储到“照片”。", "NSPhotoLibraryUsageDescription": "允许 Orca 将图库中的照片附加到已配对桌面的终端会话。" } diff --git a/mobile/locales/zh-Hant.json b/mobile/locales/zh-Hant.json index 58c688aa575..5086ed934a7 100644 --- a/mobile/locales/zh-Hant.json +++ b/mobile/locales/zh-Hant.json @@ -2,7 +2,7 @@ "ios": { "NSCameraUsageDescription": "允許 Orca 使用相機掃描 QR 碼。", "NSLocalNetworkUsageDescription": "Orca 會透過區域網路連線至桌面版應用程式。", - "NSMicrophoneUsageDescription": "允許 Orca 使用麥克風進行語音聽寫，並將語音傳送至已配對的電腦轉錄為文字。", + "NSMicrophoneUsageDescription": "允許 Orca 使用麥克風進行語音聽寫，在此手機或已配對的電腦上轉寫。", "NSPhotoLibraryAddUsageDescription": "允許 Orca 將工作階段中的影像儲存到「照片」。", "NSPhotoLibraryUsageDescription": "允許 Orca 將圖庫中的照片附加至已配對電腦上的終端機工作階段。" } diff --git a/mobile/packages/expo-two-way-audio/src/events.ts b/mobile/packages/expo-two-way-audio/src/events.ts index d98d023aa98..71ec54bb8a2 100644 --- a/mobile/packages/expo-two-way-audio/src/events.ts +++ b/mobile/packages/expo-two-way-audio/src/events.ts @@ -1,4 +1,5 @@ import ExpoTwoWayAudioModule from './ExpoTwoWayAudioModule' +import type { SpeechDownloadProgressEvent, SpeechErrorEvent, SpeechTextEvent } from './speech' export interface MicrophoneDataEvent { data: Uint8Array @@ -21,6 +22,9 @@ export interface ExpoTwoWayAudioEventMap { onInputVolumeLevelData: VolumeLevelEvent onRecordingChange: RecordingChangeEvent onAudioInterruption: AudioInterruptionEvent + onSpeechText: SpeechTextEvent + onSpeechError: SpeechErrorEvent + onSpeechDownloadProgress: SpeechDownloadProgressEvent } export type MicrophoneDataCallback = (event: MicrophoneDataEvent) => void diff --git a/mobile/packages/expo-two-way-audio/src/index.ts b/mobile/packages/expo-two-way-audio/src/index.ts index 2e2da015f14..f9891a3e3ad 100644 --- a/mobile/packages/expo-two-way-audio/src/index.ts +++ b/mobile/packages/expo-two-way-audio/src/index.ts @@ -1,3 +1,4 @@ export * from './core' export * from './events' export * from './hooks' +export * from './speech' diff --git a/mobile/packages/expo-two-way-audio/src/speech.ts b/mobile/packages/expo-two-way-audio/src/speech.ts new file mode 100644 index 00000000000..36730a01e0a --- /dev/null +++ b/mobile/packages/expo-two-way-audio/src/speech.ts @@ -0,0 +1,108 @@ +import ExpoTwoWayAudioModule from './ExpoTwoWayAudioModule' + +// On-device speech recognition contract. iOS and Android implement these functions and events on the +// ExpoTwoWayAudio module; JS feature-detects them because a shell built before them answers 'unsupported'. +// +// Audio: the shell feeds the recognizer the same 16-bit little-endian mono PCM the microphone delivers in +// onMicrophoneData, through speechFeed. The recognizer never opens the microphone itself, so capture, +// interruptions and the screen lock stay where they are. +// +// One session at a time. speechStart while a session is live cancels the old one first. + +export const SPEECH_AVAILABILITY_STATUSES = ['ready', 'needs-download', 'unsupported'] as const + +export type SpeechAvailabilityStatus = (typeof SPEECH_AVAILABILITY_STATUSES)[number] + +export type SpeechAvailability = { + status: SpeechAvailabilityStatus + // The supported locale the recognizer would use for the requested one (e.g. 'en' -> 'en-US'); null when unsupported. + locale: string | null +} + +// Why a start or a live session failed. Every code sends the page to the computer, so keep them coarse. +export const SPEECH_FAILURE_CODES = [ + 'unsupported', + 'needs-download', + 'permission-denied', + 'busy', + 'failed' +] as const + +export type SpeechFailureCode = (typeof SPEECH_FAILURE_CODES)[number] + +export type SpeechStartResult = + | { started: true; locale: string } + | { started: false; reason: SpeechFailureCode } + +// committed is the whole final text so far, never a delta; tentative is the current unstable tail after it. +// revision rises by one per event, so a reader can drop a stale one. +export type SpeechTextEvent = { revision: number; committed: string; tentative: string } + +export type SpeechErrorEvent = { code: SpeechFailureCode } + +// fraction is 0..1 for the locale's assets; a download that ends emits no final event, its promise settles instead. +export type SpeechDownloadProgressEvent = { locale: string; fraction: number } + +type SpeechNativeModule = { + // Never prompts and never downloads. Must be cheap: called before every dictation. + speechAvailability?: (locale: string) => Promise + // Installs the assets for locale; resolves true when ready, false when it failed or isn't possible. + speechDownload?: (locale: string) => Promise + // Arms a session. Prompts for any speech permission the OS needs (the microphone prompt is not its job). + // Resolve quickly: audio fed before the recognizer is warm must be queued natively, never dropped. + speechStart?: (locale: string, sampleRate: number) => Promise + // Synchronous and never throws. Ignored with no live session. + speechFeed?: (pcm: Uint8Array) => void + // Ends input, waits for every fed sample to be recognized, resolves the whole final text and ends the session. + // Rejects (or emits onSpeechError) when recognition failed; the page then sends the kept audio to the computer. + speechFinish?: () => Promise + // Ends the session and drops its text. Synchronous, never throws, safe with no session. + speechCancel?: () => void +} + +const speech: SpeechNativeModule = ExpoTwoWayAudioModule + +// A shell from before this contract lacks every function; partial builds are treated the same way. +export function hasNativeSpeech(): boolean { + return ( + typeof speech.speechAvailability === 'function' && + typeof speech.speechDownload === 'function' && + typeof speech.speechStart === 'function' && + typeof speech.speechFeed === 'function' && + typeof speech.speechFinish === 'function' && + typeof speech.speechCancel === 'function' + ) +} + +export async function speechAvailability(locale: string): Promise { + if (!hasNativeSpeech() || !speech.speechAvailability) { + return { status: 'unsupported', locale: null } + } + return speech.speechAvailability(locale) +} + +export async function speechDownload(locale: string): Promise { + return hasNativeSpeech() && speech.speechDownload ? speech.speechDownload(locale) : false +} + +export async function speechStart(locale: string, sampleRate: number): Promise { + if (!hasNativeSpeech() || !speech.speechStart) { + return { started: false, reason: 'unsupported' } + } + return speech.speechStart(locale, sampleRate) +} + +export function speechFeed(pcm: Uint8Array): void { + speech.speechFeed?.(pcm) +} + +export async function speechFinish(): Promise { + if (!speech.speechFinish) { + throw new Error('on-device speech is not available in this build') + } + return speech.speechFinish() +} + +export function speechCancel(): void { + speech.speechCancel?.() +} diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts index a364cdebf29..4323767e973 100644 --- a/mobile/src/hooks/mobile-dictation-audio-chunk.ts +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -30,7 +30,7 @@ type MobileDictationAudioChunkQueue = { transcriptSoFar: MobileDictationTranscriptSoFar } -// Holding: the microphone runs from the tap while the host's start reply is pending. +// Holding: the microphone runs from the tap while the host's start reply is pending, or for a whole on-device dictation. export type MobileDictationAudioPhase = 'closed' | 'holding' | 'sending' // Newer hosts report the text so far: shown while speaking, and handed over by a failure instead of dropped. @@ -126,32 +126,40 @@ export function subscribeMobileDictationAudioChunks( enqueueMobileDictationAudioChunk(client, dictationId, chunk, chunkQueue) } const subscription = capture.onChunk((chunk) => { - const client = source.getClient() const dictationId = source.getActiveId() const phase = source.audioPhase() - if (!client || !dictationId || phase === 'closed') { + if (!dictationId || phase === 'closed') { return } + // An on-device dictation holds its audio with no computer at all. if (phase === 'holding') { queue.heldAudio.hold(chunk) return } - send(client, dictationId, chunk) + const client = source.getClient() + if (client) { + send(client, dictationId, chunk) + } }) return { remove: () => subscription.remove(), // Call once the phase turns to sending, before any later chunk, so the seq order matches speech order. + // Held audio is sent first through the normal slots; it may be a long on-device dictation's replay, so it never counts as falling behind. sendHeld: () => { - const held = queue.heldAudio.take() const client = source.getClient() const dictationId = source.getActiveId() - for (const chunk of held) { - // A failed send ends the dictation and clears the active id, so stop there. - if (!client || !dictationId || source.getActiveId() !== dictationId) { - return - } - send(client, dictationId, chunk) + if (!client || !dictationId) { + return } + if (queue.heldAudio.droppedBytes > 0) { + queue.failActiveDictation( + dictationId, + new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE) + ) + return + } + queue.pendingAudioBudget.reserveBacklog(queue.heldAudio.bytes) + fillSendSlots(client, dictationId, chunkQueue) } } } @@ -173,13 +181,23 @@ export function enqueueMobileDictationAudioChunk( return } queue.heldAudio.hold(chunk) - if (queue.pendingChunks.size >= MOBILE_DICTATION_MAX_CHUNK_SENDS_IN_FLIGHT) { - return + fillSendSlots(client, dictationId, queue) +} + +function fillSendSlots( + client: RpcClient, + dictationId: string, + queue: MobileDictationAudioChunkQueue +): void { + while ( + queue.pendingChunks.size < MOBILE_DICTATION_MAX_CHUNK_SENDS_IN_FLIGHT && + queue.heldAudio.bytes > 0 + ) { + const slot = sendHeldAudio(client, dictationId, queue).finally(() => { + queue.pendingChunks.delete(slot) + }) + queue.pendingChunks.add(slot) } - const slot = sendHeldAudio(client, dictationId, queue).finally(() => { - queue.pendingChunks.delete(slot) - }) - queue.pendingChunks.add(slot) } // Finish awaits these slots, so a slot keeps sending until nothing is held. @@ -194,7 +212,8 @@ async function sendHeldAudio( if (!chunk) { return } - await sendAudioChunk(client, dictationId, chunk.data, queue) + const backlogShare = queue.pendingAudioBudget.takeBacklog(chunk.data.byteLength) + await sendAudioChunk(client, dictationId, chunk.data, backlogShare, queue) } } @@ -202,6 +221,7 @@ async function sendAudioChunk( client: RpcClient, dictationId: string, bytes: Uint8Array, + backlogShare: number, queue: MobileDictationAudioChunkQueue ): Promise { const params = { @@ -243,7 +263,7 @@ async function sendAudioChunk( } } finally { if (queue.isActive(dictationId)) { - queue.pendingAudioBudget.release(bytes.byteLength) + queue.pendingAudioBudget.release(bytes.byteLength, backlogShare) } } } diff --git a/mobile/src/hooks/mobile-dictation-device-engine.ts b/mobile/src/hooks/mobile-dictation-device-engine.ts new file mode 100644 index 00000000000..6501be3ca4c --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-device-engine.ts @@ -0,0 +1,117 @@ +import type { + BridgeSpeechFailure, + BridgeSpeechRead +} from '../mobile-web-shell/bridge/bridge-speech-verbs' +import type { OnDeviceSpeech } from '../platform/on-device-speech-contract' +import { DICTATION_CAPTURE_DRAIN_INTERVAL_MS } from '../platform/dictation-capture-contract' +import { translate } from '../i18n/mobile-i18n' + +// Which engine hears this dictation. fallback: a failed phone session may replay its audio to the computer (Automatic only). +export type MobileDictationEngine = + | { kind: 'host' } + | { kind: 'device'; fallback: boolean; stopReading: () => void } + +export type MobileDictationRoute = + | { kind: 'host' } + | { kind: 'device'; fallback: boolean } + | { kind: 'phone-unavailable' } + +// Automatic uses the phone when it is ready; This phone never sends audio to the computer; Computer is today's path. +export async function chooseMobileDictationRoute( + speech: OnDeviceSpeech | null +): Promise { + if (speech === null) { + return { kind: 'host' } + } + let status + try { + status = await speech.status() + } catch { + return { kind: 'host' } + } + if (status.preference === 'computer') { + return { kind: 'host' } + } + const fallback = status.preference === 'automatic' + if (status.status === 'ready') { + return { kind: 'device', fallback } + } + return fallback ? { kind: 'host' } : { kind: 'phone-unavailable' } +} + +// Arms the phone's recognizer; null means it refused or failed, and the caller decides between the computer and an error. +export async function startOnDeviceSpeech( + speech: OnDeviceSpeech +): Promise { + try { + const started = await speech.start() + return started.started ? null : (started.reason ?? 'failed') + } catch { + return 'failed' + } +} + +function onDeviceTranscriptSoFar(read: Pick) { + const committed = read.committed.trim() + const tentative = read.tentative.trim() + return committed && tentative ? `${committed} ${tentative}` : committed || tentative +} + +// The bridge has no push lane, so poll at the audio drain's pace, one read in flight at a time. A failure stops polling and is reported once. +export function readOnDeviceSpeech( + speech: OnDeviceSpeech, + onText: (text: string) => void, + onFailure: (failure: BridgeSpeechFailure) => void, + intervalMs: number = DICTATION_CAPTURE_DRAIN_INTERVAL_MS +): () => void { + let revision = -1 + let stopped = false + let timer: ReturnType | null = null + const stop = () => { + stopped = true + if (timer !== null) { + clearTimeout(timer) + } + } + const fail = (failure: BridgeSpeechFailure) => { + if (!stopped) { + stop() + onFailure(failure) + } + } + const tick = () => { + void speech.read().then( + (read) => { + if (stopped) { + return + } + if (read.failure !== null) { + fail(read.failure) + return + } + if (read.revision > revision) { + revision = read.revision + onText(onDeviceTranscriptSoFar(read)) + } + timer = setTimeout(tick, intervalMs) + }, + () => fail('failed') + ) + } + timer = setTimeout(tick, intervalMs) + return stop +} + +export function phoneDictationUnavailableMessage(): string { + return translate( + 'hooks.useMobileDictation.phoneUnavailable', + 'Dictation on this phone isn’t ready. Download it in Voice settings, or choose Automatic.' + ) +} + +export function phoneDictationFailedMessage(): string { + return translate( + 'hooks.useMobileDictation.phoneFailed', + 'Dictation on this phone stopped working. Try again.' + ) +} diff --git a/mobile/src/hooks/mobile-dictation-engine-run.ts b/mobile/src/hooks/mobile-dictation-engine-run.ts new file mode 100644 index 00000000000..e067ad3f7ab --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-engine-run.ts @@ -0,0 +1,313 @@ +import type { RefObject } from 'react' +import { dictationSessionErrorMessage } from '../dictation/mobile-dictation-setup' +import type { DictationCapture } from '../platform/dictation-capture-contract' +import type { OnDeviceSpeech } from '../platform/on-device-speech-contract' +import { openMobileDictationCapture } from './mobile-dictation-capture-open' +import { createMobileDictationId, type DictationStatus } from './mobile-dictation-session-state' +import type { RpcClient } from '../transport/rpc-client' +import type { + MobileDictationAudioPhase, + MobileDictationTranscriptSoFar +} from './mobile-dictation-audio-chunk' +import { + cancelDesktopDictation, + settledDesktopDictationWork, + startMobileDictationDesktopSession +} from './mobile-dictation-desktop-start' +import { + chooseMobileDictationRoute, + phoneDictationFailedMessage, + phoneDictationUnavailableMessage, + readOnDeviceSpeech, + startOnDeviceSpeech, + type MobileDictationEngine, + type MobileDictationRoute +} from './mobile-dictation-device-engine' +import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' +import type { MobileDictationHeldAudio } from './mobile-dictation-held-audio' + +// What the engines share with useMobileDictation: its refs (read at call time) and its state transitions. +export type MobileDictationRun = { + readonly status: RefObject + readonly activeId: RefObject + readonly client: RefObject + readonly speech: RefObject + readonly engine: RefObject + readonly enabled: RefObject + readonly generation: RefObject + readonly stopWhenStarted: RefObject + readonly audioPhase: RefObject + readonly pendingChunks: RefObject>> + readonly pendingAudioBudget: RefObject + readonly heldAudio: RefObject + readonly sendHeldAudio: RefObject<() => void> + readonly finish: RefObject<(emptyTranscriptMessage: string) => Promise> + readonly onTranscript: RefObject<(text: string) => void> + readonly keptTranscript: MobileDictationTranscriptSoFar + readonly applyStatus: (next: DictationStatus) => void + readonly closeDictationAudio: () => void + readonly reportError: (err: unknown) => void + readonly failActiveDictation: (dictationId: string, err: unknown) => void +} + +// Stop whichever engine heard this dictation; only a computer session needs a cancel on the wire. +export function releaseMobileDictationEngine( + run: Pick, + client: RpcClient | null, + dictationId: string | null +): Promise { + const engine = run.engine.current + run.engine.current = { kind: 'host' } + if (engine.kind === 'device') { + engine.stopReading() + run.speech.current?.cancel() + return Promise.resolve() + } + return client && dictationId ? cancelDesktopDictation(client, dictationId) : Promise.resolve() +} + +// Close the dictation with its final text; both engines end here. +export function settleMobileDictation( + run: MobileDictationRun, + text: string, + emptyTranscriptMessage: string +): void { + run.activeId.current = null + run.engine.current = { kind: 'host' } + run.audioPhase.current = 'closed' + run.heldAudio.current.clear() + run.pendingChunks.current.clear() + run.pendingAudioBudget.current.reset() + run.applyStatus('idle') + if (text) { + run.onTranscript.current(text) + } else { + run.reportError(new Error(emptyTranscriptMessage)) + } +} + +export async function startMobileDictationOnHost( + run: MobileDictationRun, + client: RpcClient, + dictationId: string, + generation: number +): Promise { + await settledDesktopDictationWork(client) + if (run.generation.current !== generation) { + return + } + await startMobileDictationDesktopSession({ + client, + dictationId, + generation, + getCurrentGeneration: () => run.generation.current, + getEnabled: () => run.enabled.current, + getActiveId: () => run.activeId.current, + clearActiveId: (id) => { + if (run.activeId.current === id) { + run.activeId.current = null + } + }, + setIdle: () => run.applyStatus('idle'), + commitRecordingStart: (hostDedupesResends) => { + run.audioPhase.current = 'sending' + run.pendingChunks.current.clear() + run.pendingAudioBudget.current.reset(hostDedupesResends) + // A phone dictation stopped or failed at finish is already processing; it never records again. + if (run.status.current !== 'processing') { + run.applyStatus('recording') + } + run.sendHeldAudio.current() + if (run.stopWhenStarted.current) { + void run.finish.current(dictationSessionErrorMessage('no-speech')) + } + }, + rollbackRecordingStart: run.closeDictationAudio + }) +} + +// The phone's recognizer failed: replay the audio it kept to the computer, as a start's held audio (seq from 0), and carry on there. +export async function fallBackMobileDictationToHost( + run: MobileDictationRun, + dictationId: string, + finishing: boolean +): Promise { + const engine = run.engine.current + if (engine.kind !== 'device' || run.activeId.current !== dictationId) { + return + } + engine.stopReading() + run.speech.current?.cancel() + const client = run.client.current + // Offline there is no computer to replay to; fail now with the phone's text rather than wait out the reconnect grace. + if ( + !engine.fallback || + !client || + client.getState() !== 'connected' || + !run.heldAudio.current.complete + ) { + run.failActiveDictation(dictationId, new Error(phoneDictationFailedMessage())) + return + } + run.engine.current = { kind: 'host' } + // A stop that began while the phone was still live is already processing; the computer must finish it too. + run.stopWhenStarted.current = finishing || run.status.current === 'processing' + try { + await startMobileDictationOnHost(run, client, dictationId, run.generation.current) + } catch (err) { + // The host start already released the mic and the dictation; keep what the phone heard. + const kept = run.keptTranscript.take(dictationId) + if (kept) { + run.onTranscript.current(kept) + } + run.reportError(err) + } +} + +// Pick the engine before the mic opens, and arm the phone's recognizer so it hears the first word. +// Null: stale, or the phone failed with no fallback allowed (thrown). Device: armed. Host: use the computer. +export async function armMobileDictationEngine( + run: MobileDictationRun, + isCurrent: () => boolean +): Promise | null> { + const speech = run.speech.current + const route = await chooseMobileDictationRoute(speech) + if (!isCurrent()) { + return null + } + if (route.kind === 'phone-unavailable') { + throw new Error(phoneDictationUnavailableMessage()) + } + if (route.kind === 'host' || speech === null) { + return { kind: 'host' } + } + const refused = await startOnDeviceSpeech(speech) + if (!isCurrent()) { + if (refused === null) { + speech.cancel() + } + return null + } + if (refused === null) { + return route + } + if (!route.fallback) { + throw new Error(phoneDictationFailedMessage()) + } + return { kind: 'host' } +} + +// Recording started on the phone: no computer round trip, text read on the drain tick, audio held for a replay. +export function commitMobileDictationOnDevice( + run: MobileDictationRun, + dictationId: string, + fallback: boolean +): void { + const speech = run.speech.current + if (speech === null) { + return + } + run.engine.current = { + kind: 'device', + fallback, + stopReading: readOnDeviceSpeech( + speech, + (text) => { + if (run.activeId.current === dictationId) { + // Lowest rank: once a fallback replays the audio, the computer's numbered text replaces the phone's. + run.keptTranscript.note(dictationId, text, Number.NEGATIVE_INFINITY) + } + }, + () => void fallBackMobileDictationToHost(run, dictationId, false) + ) + } + // This phone never replays to the computer, so it keeps no audio. + if (!fallback) { + run.audioPhase.current = 'closed' + } + run.applyStatus('recording') + if (run.stopWhenStarted.current) { + void run.finish.current(dictationSessionErrorMessage('no-speech')) + } +} + +// The phone's final text, or a replay to the computer when the phone could not give one. +export async function finishMobileDictationOnDevice( + run: MobileDictationRun, + engine: Extract, + dictationId: string, + emptyTranscriptMessage: string, + isCurrent: () => boolean +): Promise { + engine.stopReading() + const speech = run.speech.current + const finished = speech === null ? null : await speech.finish().catch(() => null) + if (!isCurrent()) { + return + } + if (finished === null || finished.failure !== null || finished.text === null) { + await fallBackMobileDictationToHost(run, dictationId, true) + return + } + settleMobileDictation(run, finished.text.trim(), emptyTranscriptMessage) +} + +const HOST_ENGINE = { kind: 'host' } as const + +// Pick the engine, open the mic from the tap, then start the phone's recognizer or the computer's session. +export async function startMobileDictation( + run: MobileDictationRun, + capture: DictationCapture +): Promise { + const client = run.client.current + const generation = run.generation.current + 1 + run.generation.current = generation + run.stopWhenStarted.current = false + run.applyStatus('starting') + const isCurrentGeneration = () => run.generation.current === generation + // Without a phone recognizer, start exactly as before: no extra turn before the mic opens. + const engine = + run.speech.current === null + ? HOST_ENGINE + : await armMobileDictationEngine(run, isCurrentGeneration).catch((err: unknown) => { + run.applyStatus('idle') + throw err + }) + if (engine === null || (engine.kind === 'host' && !client)) { + if (engine !== null) { + run.applyStatus('idle') + } + return + } + // A phone recognizer armed for a mic that never opened must not keep listening. + const disarm = () => { + if (engine.kind === 'device') { + run.speech.current?.cancel() + } + } + const recording = await openMobileDictationCapture({ + capture, + isCurrent: () => isCurrentGeneration() && run.enabled.current, + isCurrentGeneration, + setIdle: () => run.applyStatus('idle'), + // Record from the tap: speech during the host's engine build and round trip is held, not lost; a phone dictation holds it all for a replay. + hold: () => { + run.audioPhase.current = 'holding' + }, + close: run.closeDictationAudio + }).catch((err: unknown) => { + disarm() + throw err + }) + if (!recording) { + disarm() + return + } + const dictationId = createMobileDictationId() + run.activeId.current = dictationId + if (engine.kind === 'device') { + commitMobileDictationOnDevice(run, dictationId, engine.fallback) + } else if (client) { + await startMobileDictationOnHost(run, client, dictationId, generation) + } +} diff --git a/mobile/src/hooks/mobile-dictation-held-audio.ts b/mobile/src/hooks/mobile-dictation-held-audio.ts index c80eaf3edde..3e57dfad7d7 100644 --- a/mobile/src/hooks/mobile-dictation-held-audio.ts +++ b/mobile/src/hooks/mobile-dictation-held-audio.ts @@ -1,12 +1,47 @@ -import { MAX_DICTATION_AUDIO_CHUNK_BYTES } from '../../../src/shared/rpc-contract/speech-params' +import { + DICTATION_SAMPLE_RATE, + MAX_DICTATION_AUDIO_CHUNK_BYTES, + PCM_BYTES_PER_SAMPLE +} from '../../../src/shared/rpc-contract/speech-params' import type { DictationCaptureChunk } from '../platform/dictation-capture-contract' -// Speech not sent yet, in order: held between the tap and the host's start reply, and while every send slot is busy. +// Five minutes of 16 kHz PCM (9.6 MB): what an on-device dictation keeps to replay to the computer if the phone fails. +const MAX_HELD_AUDIO_BYTES = DICTATION_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * 300 + +// Speech not sent yet, in order: held between the tap and the host's start reply, while every send slot is busy, and for a whole on-device dictation. export class MobileDictationHeldAudio { private held: DictationCaptureChunk[] = [] + private heldBytes = 0 + private heldDroppedBytes = 0 + private overflowed = false hold(chunk: DictationCaptureChunk): void { + if (this.overflowed) { + return + } + // Past the cap a replay would lose speech, so keep nothing and let the caller refuse it. + if (this.heldBytes + chunk.data.byteLength > MAX_HELD_AUDIO_BYTES) { + this.clear() + this.overflowed = true + return + } this.held.push(chunk) + this.heldBytes += chunk.data.byteLength + this.heldDroppedBytes += chunk.droppedBytes + } + + get bytes(): number { + return this.heldBytes + } + + // Capture lost audio while it was held: the page fell behind the microphone. + get droppedBytes(): number { + return this.heldDroppedBytes + } + + // False once speech was dropped past the cap; the held audio no longer covers the whole dictation. + get complete(): boolean { + return !this.overflowed } // Merge into one wire-sized chunk so a backlog costs a few requests, not one per capture event. @@ -26,6 +61,8 @@ export class MobileDictationHeldAudio { droppedBytes += chunk.droppedBytes } const parts = this.held.splice(0, count) + this.heldBytes -= bytes + this.heldDroppedBytes -= droppedBytes if (parts.length === 1) { return parts[0] } @@ -48,5 +85,8 @@ export class MobileDictationHeldAudio { clear(): void { this.held = [] + this.heldBytes = 0 + this.heldDroppedBytes = 0 + this.overflowed = false } } diff --git a/mobile/src/hooks/mobile-dictation-on-device.test.tsx b/mobile/src/hooks/mobile-dictation-on-device.test.tsx new file mode 100644 index 00000000000..d7f9c020804 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-on-device.test.tsx @@ -0,0 +1,298 @@ +// The phone's recognizer hears the dictation first; when it fails, the audio the page kept replays to the computer. +import { createElement } from 'react' +import { act, create } from 'react-test-renderer' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as z from 'zod' +import { + createFakeRpcClient, + type FakeRpcClient, + type SentRequest +} from '../mobile-web-shell/bridge-host-test-fakes' +import type { + DictationCapture, + DictationCaptureChunk +} from '../platform/dictation-capture-contract' +import type { OnDeviceSpeech } from '../platform/on-device-speech-contract' +import type { + BridgeSpeechFinish, + BridgeSpeechRead, + DictationEnginePreference +} from '../mobile-web-shell/bridge/bridge-speech-verbs' + +type Seam = { + chunkHandlers: Set<(chunk: DictationCaptureChunk) => void> + preference: DictationEnginePreference + read: BridgeSpeechRead + finish: BridgeSpeechFinish + calls: string[] +} + +const seam = vi.hoisted((): Seam => ({ + chunkHandlers: new Set(), + preference: 'automatic', + read: { revision: 0, committed: '', tentative: '', failure: null }, + finish: { text: '', failure: null }, + calls: [] +})) + +vi.mock('react-native', () => ({ + AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) }, + Platform: { OS: 'ios' } +})) + +vi.mock('../platform/dictation-capture', () => { + const capture: DictationCapture = { + open: async () => ({ ok: true }), + begin: () => true, + end: async () => {}, + release: () => {}, + onChunk: (handler) => { + seam.chunkHandlers.add(handler) + return { remove: () => seam.chunkHandlers.delete(handler) } + }, + onInterruption: () => ({ remove: () => {} }) + } + return { useDictationCapture: () => capture } +}) + +vi.mock('../platform/on-device-speech', () => { + const speech: OnDeviceSpeech = { + status: async () => ({ + status: 'ready', + locale: 'en-US', + preference: seam.preference + }), + start: async () => { + seam.calls.push('start') + return { started: true, locale: 'en-US', reason: null } + }, + read: async () => seam.read, + finish: async () => { + seam.calls.push('finish') + return seam.finish + }, + cancel: () => { + seam.calls.push('cancel') + } + } + return { useOnDeviceSpeech: () => speech } +}) + +import { useMobileDictation, type UseMobileDictationResult } from './use-mobile-dictation' + +const held: { dictation: UseMobileDictationResult | null; errors: string[] } = { + dictation: null, + errors: [] +} + +function mount(client: FakeRpcClient, onTranscript: (text: string) => void): void { + function Probe(): null { + held.dictation = useMobileDictation({ + client, + enabled: true, + onTranscript, + onError: (error) => held.errors.push(error.message) + }) + return null + } + act(() => { + create(createElement(Probe)) + }) +} + +function dictation(): UseMobileDictationResult { + if (held.dictation === null) { + throw new Error('nothing mounted') + } + return held.dictation +} + +function speak(bytes: number[]): void { + for (const handler of seam.chunkHandlers) { + handler({ data: Uint8Array.from(bytes), droppedBytes: 0 }) + } +} + +async function pump(rpc: FakeRpcClient, sent: SentRequest[]): Promise { + for (let round = 0; round < 40; round += 1) { + for (const request of rpc.requests.splice(0)) { + sent.push(request) + request.resolve({ + id: 'desktop', + ok: true, + result: request.method === 'speech.dictation.finish' ? { text: 'from the computer' } : {} + }) + } + await Promise.resolve() + await Promise.resolve() + } +} + +const chunkParamsSchema = z.object({ seq: z.number(), audioBase64: z.string() }) + +function chunkAudio(sent: SentRequest[]): [number, string][] { + return sent + .filter((request) => request.method === 'speech.dictation.chunk') + .map((request): [number, string] => { + const params = chunkParamsSchema.parse(request.args[1]) + return [params.seq, params.audioBase64] + }) +} + +beforeEach(() => { + seam.chunkHandlers.clear() + seam.preference = 'automatic' + seam.read = { revision: 0, committed: '', tentative: '', failure: null } + seam.finish = { text: '', failure: null } + seam.calls = [] + held.dictation = null + held.errors = [] +}) + +describe('a dictation the phone can hear', () => { + it('records without the computer and inserts the phone’s text', async () => { + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const transcripts: string[] = [] + mount(rpc, (text) => transcripts.push(text)) + await act(async () => { + await dictation().start() + }) + expect(dictation().status).toBe('recording') + speak([1, 2, 3, 4]) + seam.finish = { text: ' hello phone ', failure: null } + await act(async () => { + const stopped = dictation().stop() + await pump(rpc, sent) + await stopped + }) + expect(transcripts).toEqual(['hello phone']) + expect(sent).toEqual([]) + expect(dictation().status).toBe('idle') + }) +}) + +describe('a phone recognizer that fails at the finish', () => { + it('replays what was said to the computer and inserts its text', async () => { + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const transcripts: string[] = [] + mount(rpc, (text) => transcripts.push(text)) + await act(async () => { + await dictation().start() + }) + speak([1, 2, 3]) + speak([4, 5, 6]) + seam.finish = { text: null, failure: 'failed' } + await act(async () => { + const stopped = dictation().stop() + await pump(rpc, sent) + await stopped + }) + const methods = sent.map((request) => request.method) + expect(methods).toEqual([ + 'speech.dictation.start', + 'speech.dictation.chunk', + 'speech.dictation.finish' + ]) + // Both utterances, merged in order, starting the computer's sequence at zero. + expect(chunkAudio(sent)).toEqual([[0, 'AQIDBAUG']]) + expect(transcripts).toEqual(['from the computer']) + expect(held.errors).toEqual([]) + }) + + it('never sends the audio anywhere when the user chose This phone', async () => { + seam.preference = 'phone' + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const transcripts: string[] = [] + mount(rpc, (text) => transcripts.push(text)) + await act(async () => { + await dictation().start() + }) + speak([1, 2, 3]) + seam.finish = { text: null, failure: 'failed' } + await act(async () => { + const stopped = dictation().stop() + await pump(rpc, sent) + await stopped + }) + expect(sent).toEqual([]) + expect(transcripts).toEqual([]) + expect(held.errors).toHaveLength(1) + expect(dictation().status).toBe('error') + }) +}) + +describe('a phone recognizer that fails after a long dictation', () => { + it('replays all of it to the computer instead of calling the connection slow', async () => { + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const transcripts: string[] = [] + mount(rpc, (text) => transcripts.push(text)) + await act(async () => { + await dictation().start() + }) + // 40 s of 16 kHz PCM: more than the 30 s the host may have pending at once. + for (let second = 0; second < 40; second += 1) { + speak(Array.from({ length: 32_000 }, () => second)) + } + seam.finish = { text: null, failure: 'failed' } + await act(async () => { + const stopped = dictation().stop() + await pump(rpc, sent) + await stopped + }) + expect(held.errors).toEqual([]) + expect(chunkAudio(sent).map(([seq]) => seq)).toEqual([0, 1, 2, 3, 4, 5, 6, 7]) + expect(transcripts).toEqual(['from the computer']) + }) +}) + +describe('a stop while a failed phone dictation is reaching the computer', () => { + it('finishes on the computer once its start lands, with the audio kept so far', async () => { + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + const transcripts: string[] = [] + mount(rpc, (text) => transcripts.push(text)) + await act(async () => { + await dictation().start() + }) + speak([1, 2, 3]) + seam.read = { revision: 1, committed: '', tentative: '', failure: 'failed' } + // The drain tick reads the failure and starts the computer's session. + await act(async () => { + await vi.waitFor(() => + expect(rpc.requests.map((request) => request.method)).toEqual(['speech.dictation.start']) + ) + }) + await act(async () => { + const stopped = dictation().stop() + await pump(rpc, sent) + await stopped + }) + expect(sent.map((request) => request.method)).toEqual([ + 'speech.dictation.start', + 'speech.dictation.chunk', + 'speech.dictation.finish' + ]) + expect(transcripts).toEqual(['from the computer']) + expect(dictation().status).toBe('idle') + }) +}) + +describe('a dictation when the user chose Computer', () => { + it('goes to the computer and never arms the phone', async () => { + seam.preference = 'computer' + const rpc = createFakeRpcClient() + const sent: SentRequest[] = [] + mount(rpc, () => {}) + await act(async () => { + const started = dictation().start() + await pump(rpc, sent) + await started + }) + expect(sent.map((request) => request.method)).toEqual(['speech.dictation.start']) + expect(seam.calls).toEqual([]) + }) +}) diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts index bd6564258fc..77190320f52 100644 --- a/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts @@ -24,4 +24,16 @@ describe('MobileDictationPendingAudioBudget', () => { expect(budget.pendingAudioBytes).toBe(0) expect(budget.tryReserve(10)).toBe(true) }) + + it('keeps a replayed backlog out of the live limit while live requests settle first', () => { + const budget = new MobileDictationPendingAudioBudget(100) + budget.reserveBacklog(150) + expect(budget.takeBacklog(150)).toBe(150) + expect(budget.tryReserve(60)).toBe(true) + expect(budget.takeBacklog(60)).toBe(0) + // The live request answers before the older backlog request does. + budget.release(60, 0) + expect(budget.tryReserve(90)).toBe(true) + expect(budget.tryReserve(11)).toBe(false) + }) }) diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts index 507f47c1609..6f59df36b63 100644 --- a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts @@ -10,6 +10,10 @@ export const MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE = export class MobileDictationPendingAudioBudget { private pendingBytes = 0 + // Audio held before the host committed (a whole on-device dictation, for a replay) is sent first and may exceed the limit; only audio after it counts. + private backlogBytes = 0 + // Backlog not yet taken into a request; it sits at the front of the held audio. + private untakenBacklogBytes = 0 private sentChunks = 0 // Set at each start: a host that feeds each chunk seq once can take audio a drop left unanswered again. hostDedupesResends = false @@ -24,21 +28,39 @@ export class MobileDictationPendingAudioBudget { return this.sentChunks++ } + reserveBacklog(byteLength: number): void { + const normalizedByteLength = normalizeByteLength(byteLength) + this.pendingBytes += normalizedByteLength + this.backlogBytes += normalizedByteLength + this.untakenBacklogBytes += normalizedByteLength + } + + // The backlog share of a request just taken from the front of the held audio; release it with that request. + takeBacklog(byteLength: number): number { + const share = Math.min(this.untakenBacklogBytes, normalizeByteLength(byteLength)) + this.untakenBacklogBytes -= share + return share + } + tryReserve(byteLength: number): boolean { const normalizedByteLength = normalizeByteLength(byteLength) - if (this.pendingBytes + normalizedByteLength > this.maxPendingBytes) { + if (this.pendingBytes + normalizedByteLength > this.maxPendingBytes + this.backlogBytes) { return false } this.pendingBytes += normalizedByteLength return true } - release(byteLength: number): void { + // Requests settle out of order, so each releases only its own backlog share. + release(byteLength: number, backlogShare = 0): void { this.pendingBytes = Math.max(0, this.pendingBytes - normalizeByteLength(byteLength)) + this.backlogBytes = Math.max(0, this.backlogBytes - normalizeByteLength(backlogShare)) } reset(hostDedupesResends = false): void { this.pendingBytes = 0 + this.backlogBytes = 0 + this.untakenBacklogBytes = 0 this.sentChunks = 0 this.hostDedupesResends = hostDedupesResends } diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index b9d098f1d62..4810642c4e7 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -1,5 +1,6 @@ -import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { useDictationCapture } from '../platform/dictation-capture' +import { useOnDeviceSpeech } from '../platform/on-device-speech' import { dictationSessionErrorMessage } from '../dictation/mobile-dictation-setup' import { useMobileTranslation } from '../i18n/use-mobile-translation' import { useMobileDictationMeter } from './mobile-dictation-meter' @@ -10,17 +11,16 @@ import { type MobileDictationAudioPhase } from './mobile-dictation-audio-chunk' import { MobileDictationHeldAudio } from './mobile-dictation-held-audio' -import { openMobileDictationCapture } from './mobile-dictation-capture-open' +import { isCurrentMobileDictationFinish } from './mobile-dictation-session-state' +import { finishDesktopDictation } from './mobile-dictation-desktop-start' +import type { MobileDictationEngine } from './mobile-dictation-device-engine' import { - createMobileDictationId, - isCurrentMobileDictationFinish -} from './mobile-dictation-session-state' -import { - cancelDesktopDictation, - finishDesktopDictation, - settledDesktopDictationWork, - startMobileDictationDesktopSession -} from './mobile-dictation-desktop-start' + finishMobileDictationOnDevice, + releaseMobileDictationEngine, + settleMobileDictation, + startMobileDictation, + type MobileDictationRun +} from './mobile-dictation-engine-run' import { rpcPayloadMember } from '../transport/rpc-reader-payload' import type { DictationStatus, @@ -35,6 +35,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const { client, enabled, onTranscript, onError } = options // Both capture seams hold the screen while the microphone is open; native uses the device directly and page uses shell audio verbs. const capture = useDictationCapture() + const speech = useOnDeviceSpeech() const [status, setStatus] = useState('idle') const [error, setError] = useState(null) const [transcriptSoFar, setTranscriptSoFar, keptTranscript] = useMobileDictationTranscriptSoFar() @@ -42,6 +43,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const statusRef = useRef('idle') const activeIdRef = useRef(null) const clientRef = useRef(client) + const speechRef = useRef(speech) + const engineRef = useRef({ kind: 'host' }) + const finishRef = useRef<(emptyTranscriptMessage: string) => Promise>(async () => {}) const enabledRef = useRef(enabled) const onTranscriptRef = useRef(onTranscript) const onErrorRef = useRef(onError) @@ -57,10 +61,11 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil // Native audio events can arrive before passive Effects flush, but refs // should only expose options from a committed render. clientRef.current = client + speechRef.current = speech enabledRef.current = enabled onTranscriptRef.current = onTranscript onErrorRef.current = onError - }, [client, enabled, onTranscript, onError]) + }, [client, speech, enabled, onTranscript, onError]) const applyStatus = useCallback((next: DictationStatus) => { statusRef.current = next @@ -94,15 +99,16 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const failActiveDictation = useCallback( (dictationId: string, err: unknown) => { - const client = clientRef.current if (activeIdRef.current !== dictationId) { return } activeIdRef.current = null closeDictationAudio() - if (client && dictationId) { - void cancelDesktopDictation(client, dictationId) - } + void releaseMobileDictationEngine( + { engine: engineRef, speech: speechRef }, + clientRef.current, + dictationId + ) const kept = keptTranscript.take(dictationId) if (kept) { onTranscriptRef.current(kept) @@ -112,6 +118,32 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil [closeDictationAudio, reportError] ) + const run = useMemo( + () => ({ + status: statusRef, + activeId: activeIdRef, + client: clientRef, + speech: speechRef, + engine: engineRef, + enabled: enabledRef, + generation: generationRef, + stopWhenStarted: stopWhenStartedRef, + audioPhase: audioPhaseRef, + pendingChunks: pendingChunksRef, + pendingAudioBudget: pendingAudioBudgetRef, + heldAudio: heldAudioRef, + sendHeldAudio: sendHeldAudioRef, + finish: finishRef, + onTranscript: onTranscriptRef, + keptTranscript, + applyStatus, + closeDictationAudio, + reportError, + failActiveDictation + }), + [applyStatus, closeDictationAudio, failActiveDictation, keptTranscript, reportError] + ) + useEffect(() => { const source = { getClient: () => clientRef.current, @@ -133,7 +165,18 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil async (emptyTranscriptMessage: string) => { const client = clientRef.current const dictationId = activeIdRef.current - if (!client || !dictationId) { + const engine = engineRef.current + if (!dictationId || (engine.kind === 'host' && !client)) { + return + } + // A fallback's host start is still pending: finish once it commits and the kept audio has gone. + if (engine.kind === 'host' && audioPhaseRef.current === 'holding') { + stopWhenStartedRef.current = true + if (statusRef.current === 'recording') { + applyStatus('processing') + // Stop listening now; the tail joins the held audio the replay sends. + void capture.end() + } return } @@ -151,9 +194,19 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil try { // Keep shutdown inside try for finally cleanup; accept its tail chunks and snapshot pending sends only after end settles, or finish can overtake the last audio. await capture.end() + if (engine.kind === 'device') { + await finishMobileDictationOnDevice( + run, + engine, + dictationId, + emptyTranscriptMessage, + isCurrent + ) + return + } audioPhaseRef.current = 'closed' await Promise.allSettled(Array.from(pendingChunksRef.current)) - if (!isCurrent()) { + if (!isCurrent() || !client) { return } const finished = await finishDesktopDictation(client, dictationId) @@ -161,22 +214,20 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil return } const transcript = rpcPayloadMember(finished, 'text') - const text = typeof transcript === 'string' ? transcript.trim() : '' - activeIdRef.current = null - pendingChunksRef.current.clear() - pendingAudioBudgetRef.current.reset() - applyStatus('idle') - if (text) { - onTranscriptRef.current(text) - } else { - reportError(new Error(emptyTranscriptMessage)) - } + settleMobileDictation( + run, + typeof transcript === 'string' ? transcript.trim() : '', + emptyTranscriptMessage + ) } catch (err) { failActiveDictation(dictationId, err) } }, - [applyStatus, capture, failActiveDictation] + [applyStatus, capture, failActiveDictation, run] ) + useLayoutEffect(() => { + finishRef.current = finishDictation + }, [finishDictation]) const stop = useCallback(async () => { // Speech is held from the tap, so a stop before the host answers finishes once it does. @@ -188,64 +239,12 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil }, [finishDictation]) const start = useCallback(async () => { - const client = clientRef.current - if (!client || !enabledRef.current || activeIdRef.current) { + if ((!clientRef.current && !speechRef.current) || !enabledRef.current || activeIdRef.current) { return } - - const generation = generationRef.current + 1 - generationRef.current = generation - stopWhenStartedRef.current = false setError(null) - applyStatus('starting') - const isCurrentGeneration = () => generationRef.current === generation - const recording = await openMobileDictationCapture({ - capture, - isCurrent: () => isCurrentGeneration() && enabledRef.current, - isCurrentGeneration, - setIdle: () => applyStatus('idle'), - // Record from the tap: speech during the host's engine build and the round trip is held, not lost. - hold: () => { - audioPhaseRef.current = 'holding' - }, - close: closeDictationAudio - }) - if (!recording) { - return - } - const dictationId = createMobileDictationId() - activeIdRef.current = dictationId - await settledDesktopDictationWork(client) - if (!isCurrentGeneration()) { - return - } - - await startMobileDictationDesktopSession({ - client, - dictationId, - generation, - getCurrentGeneration: () => generationRef.current, - getEnabled: () => enabledRef.current, - getActiveId: () => activeIdRef.current, - clearActiveId: (id) => { - if (activeIdRef.current === id) { - activeIdRef.current = null - } - }, - setIdle: () => applyStatus('idle'), - commitRecordingStart: (hostDedupesResends) => { - audioPhaseRef.current = 'sending' - pendingChunksRef.current.clear() - pendingAudioBudgetRef.current.reset(hostDedupesResends) - applyStatus('recording') - sendHeldAudioRef.current() - if (stopWhenStartedRef.current) { - void finishDictation(dictationSessionErrorMessage('no-speech')) - } - }, - rollbackRecordingStart: closeDictationAudio - }) - }, [applyStatus, capture, closeDictationAudio, finishDictation]) + await startMobileDictation(run, capture) + }, [capture, run]) // Only the user’s own cancel (reason:null) ends silently; other interruptions call onError so the tap has a visible outcome. const abandonDictation = useCallback( @@ -263,11 +262,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil applyStatus('idle') setError(null) } - if (client && dictationId) { - await cancelDesktopDictation(client, dictationId) - } + await releaseMobileDictationEngine(run, client, dictationId) }, - [applyStatus, closeDictationAudio, reportError] + [applyStatus, closeDictationAudio, reportError, run] ) const cancel = useCallback(() => abandonDictation(null), [abandonDictation]) @@ -297,11 +294,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil activeIdRef.current = null closeDictationAudio() capture.release() - if (clientRef.current && dictationId) { - void cancelDesktopDictation(clientRef.current, dictationId) - } + void releaseMobileDictationEngine(run, clientRef.current, dictationId) } - }, [capture, closeDictationAudio]) + }, [capture, closeDictationAudio, run]) return { status, diff --git a/mobile/src/hooks/use-phone-dictation-ready.ts b/mobile/src/hooks/use-phone-dictation-ready.ts new file mode 100644 index 00000000000..214e659455d --- /dev/null +++ b/mobile/src/hooks/use-phone-dictation-ready.ts @@ -0,0 +1,20 @@ +import { useCallback, useRef, useState } from 'react' +import { useOnDeviceSpeech } from '../platform/on-device-speech' +import { chooseMobileDictationRoute } from './mobile-dictation-device-engine' + +// Whether a tap now would dictate on the phone itself. The ref answers inside a tap; the state re-renders the mic. +export function usePhoneDictationReady() { + const speech = useOnDeviceSpeech() + const [ready, setReady] = useState(false) + const readyRef = useRef(false) + const readRef = useRef(0) + const refresh = useCallback(async () => { + const read = ++readRef.current + const route = await chooseMobileDictationRoute(speech) + if (read === readRef.current) { + readyRef.current = route.kind === 'device' + setReady(readyRef.current) + } + }, [speech]) + return { ready, readyRef, refresh } +} diff --git a/mobile/src/i18n/locales/de.json b/mobile/src/i18n/locales/de.json index d304c03ff89..e71885ca619 100644 --- a/mobile/src/i18n/locales/de.json +++ b/mobile/src/i18n/locales/de.json @@ -1471,7 +1471,9 @@ "microphoneInitFailed": "Mikrofon konnte nicht initialisiert werden.", "microphoneInterrupted": "Das Diktat wurde beendet, weil die Mikrofonaufnahme unterbrochen wurde. Erneut versuchen.", "microphonePermissionDenied": "Mikrofonzugriff verweigert", - "noSpeech": "Keine Sprache erkannt." + "noSpeech": "Keine Sprache erkannt.", + "phoneUnavailable": "Diktieren auf diesem Telefon ist nicht bereit. Lade es in den Spracheinstellungen herunter oder wähle Automatisch.", + "phoneFailed": "Diktieren auf diesem Telefon funktioniert nicht mehr. Versuche es erneut." } }, "host": { @@ -2826,6 +2828,18 @@ }, "agents": { "permissionOwnAll": "Eigene Einstellungen der Agenten" + }, + "voiceTranscribeOn": { + "heading": "TRANSKRIBIEREN AUF", + "description": "Was dieses Telefon hört, bleibt auf ihm. Automatisch nutzt deinen Computer, wenn dieses Telefon nicht transkribieren kann.", + "automatic": "Automatisch", + "phone": "Dieses Telefon", + "computer": "Computer", + "phoneReady": "Auf diesem Telefon bereit", + "phoneNeedsDownload": "Sprachdownload erforderlich", + "phoneDownloading": "Wird geladen… {{progress}} %", + "phoneUnsupported": "Für die Sprache dieses Telefons nicht verfügbar", + "download": "Für dieses Telefon herunterladen" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/en.json b/mobile/src/i18n/locales/en.json index 52f750987fd..71d7c30f417 100644 --- a/mobile/src/i18n/locales/en.json +++ b/mobile/src/i18n/locales/en.json @@ -148,6 +148,18 @@ }, "agents": { "permissionOwnAll": "Agents' own settings" + }, + "voiceTranscribeOn": { + "heading": "TRANSCRIBE ON", + "description": "Speech heard on this phone stays on it. Automatic uses your computer when this phone can’t transcribe.", + "automatic": "Automatic", + "phone": "This phone", + "computer": "Computer", + "phoneReady": "Ready on this phone", + "phoneNeedsDownload": "Needs a language download", + "phoneDownloading": "Downloading… {{progress}}%", + "phoneUnsupported": "Not available for this phone’s language", + "download": "Download for this phone" } }, "common": { @@ -608,7 +620,9 @@ "microphoneInitFailed": "Failed to initialize microphone", "noSpeech": "No speech detected.", "microphoneInterrupted": "Voice dictation stopped because the microphone was interrupted. Try again.", - "inputClosed": "Voice dictation stopped because this session is no longer accepting input. Try again." + "inputClosed": "Voice dictation stopped because this session is no longer accepting input. Try again.", + "phoneUnavailable": "Dictation on this phone isn’t ready. Download it in Voice settings, or choose Automatic.", + "phoneFailed": "Dictation on this phone stopped working. Try again." } }, "hostScreen": { diff --git a/mobile/src/i18n/locales/es.json b/mobile/src/i18n/locales/es.json index 2eb0e09e51f..18c9779dbf8 100644 --- a/mobile/src/i18n/locales/es.json +++ b/mobile/src/i18n/locales/es.json @@ -1510,7 +1510,9 @@ "microphoneInitFailed": "No se pudo inicializar el micrófono", "microphoneInterrupted": "El dictado de voz se detuvo porque se interrumpió el uso del micrófono. Vuelve a intentarlo.", "microphonePermissionDenied": "Permiso de acceso al micrófono denegado", - "noSpeech": "No se detectó ninguna voz." + "noSpeech": "No se detectó ninguna voz.", + "phoneUnavailable": "El dictado en este teléfono no está listo. Descárgalo en Ajustes de voz o elige Automático.", + "phoneFailed": "El dictado en este teléfono dejó de funcionar. Inténtalo de nuevo." } }, "host": { @@ -2898,6 +2900,18 @@ }, "agents": { "permissionOwnAll": "Configuración propia de los agentes" + }, + "voiceTranscribeOn": { + "heading": "TRANSCRIBIR EN", + "description": "Lo que escucha este teléfono se queda en él. Automático usa tu ordenador cuando este teléfono no puede transcribir.", + "automatic": "Automático", + "phone": "Este teléfono", + "computer": "Ordenador", + "phoneReady": "Listo en este teléfono", + "phoneNeedsDownload": "Necesita descargar el idioma", + "phoneDownloading": "Descargando… {{progress}} %", + "phoneUnsupported": "No disponible para el idioma de este teléfono", + "download": "Descargar para este teléfono" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/fr.json b/mobile/src/i18n/locales/fr.json index f12acb08574..c08432fe6d6 100644 --- a/mobile/src/i18n/locales/fr.json +++ b/mobile/src/i18n/locales/fr.json @@ -1510,7 +1510,9 @@ "microphoneInitFailed": "Impossible d'initialiser le microphone", "microphoneInterrupted": "La dictée vocale s'est arrêtée car l'accès au microphone a été interrompu. Réessayez.", "microphonePermissionDenied": "Accès au microphone refusé", - "noSpeech": "Aucune parole détectée." + "noSpeech": "Aucune parole détectée.", + "phoneUnavailable": "La dictée sur ce téléphone n’est pas prête. Téléchargez-la dans les réglages Voix ou choisissez Automatique.", + "phoneFailed": "La dictée sur ce téléphone a cessé de fonctionner. Réessayez." } }, "host": { @@ -2898,6 +2900,18 @@ }, "agents": { "permissionOwnAll": "Paramètres propres aux agents" + }, + "voiceTranscribeOn": { + "heading": "TRANSCRIRE SUR", + "description": "Ce que ce téléphone entend reste sur ce téléphone. Automatique utilise votre ordinateur quand ce téléphone ne peut pas transcrire.", + "automatic": "Automatique", + "phone": "Ce téléphone", + "computer": "Ordinateur", + "phoneReady": "Prêt sur ce téléphone", + "phoneNeedsDownload": "Téléchargement de la langue requis", + "phoneDownloading": "Téléchargement… {{progress}} %", + "phoneUnsupported": "Indisponible pour la langue de ce téléphone", + "download": "Télécharger pour ce téléphone" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/ja.json b/mobile/src/i18n/locales/ja.json index aa46e7c187d..17c41e7887a 100644 --- a/mobile/src/i18n/locales/ja.json +++ b/mobile/src/i18n/locales/ja.json @@ -1467,7 +1467,9 @@ "microphoneInitFailed": "マイクの初期化に失敗しました", "microphoneInterrupted": "マイクの使用が中断されたため、音声ディクテーションを停止しました。再試行してください。", "microphonePermissionDenied": "マイクの使用が許可されていません", - "noSpeech": "音声が検出されませんでした。" + "noSpeech": "音声が検出されませんでした。", + "phoneUnavailable": "この端末での音声入力は準備ができていません。音声設定でダウンロードするか、「自動」を選んでください。", + "phoneFailed": "この端末での音声入力が停止しました。もう一度お試しください。" } }, "host": { @@ -2822,6 +2824,18 @@ }, "agents": { "permissionOwnAll": "Agent 独自の設定" + }, + "voiceTranscribeOn": { + "heading": "文字起こしの場所", + "description": "この端末で聞き取った音声は端末の外に出ません。「自動」では、この端末で文字起こしできないときにコンピュータを使います。", + "automatic": "自動", + "phone": "この端末", + "computer": "コンピュータ", + "phoneReady": "この端末で使用可能", + "phoneNeedsDownload": "言語のダウンロードが必要", + "phoneDownloading": "ダウンロード中… {{progress}}%", + "phoneUnsupported": "この端末の言語では使用できません", + "download": "この端末にダウンロード" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/ko.json b/mobile/src/i18n/locales/ko.json index aa20a35889b..6096ff45edf 100644 --- a/mobile/src/i18n/locales/ko.json +++ b/mobile/src/i18n/locales/ko.json @@ -1467,7 +1467,9 @@ "microphoneInitFailed": "마이크를 초기화하지 못했습니다.", "microphoneInterrupted": "마이크 사용이 중단되어 음성 받아쓰기가 멈췄습니다. 다시 시도하세요.", "microphonePermissionDenied": "마이크 권한이 거부되었습니다.", - "noSpeech": "음성이 감지되지 않았습니다." + "noSpeech": "음성이 감지되지 않았습니다.", + "phoneUnavailable": "이 휴대폰의 받아쓰기가 준비되지 않았습니다. 음성 설정에서 다운로드하거나 자동을 선택하세요.", + "phoneFailed": "이 휴대폰의 받아쓰기가 중단되었습니다. 다시 시도하세요." } }, "host": { @@ -2822,6 +2824,18 @@ }, "agents": { "permissionOwnAll": "에이전트 자체 설정" + }, + "voiceTranscribeOn": { + "heading": "받아쓰기 위치", + "description": "이 휴대폰이 들은 음성은 휴대폰 밖으로 나가지 않습니다. 자동은 이 휴대폰이 받아쓸 수 없을 때 컴퓨터를 사용합니다.", + "automatic": "자동", + "phone": "이 휴대폰", + "computer": "컴퓨터", + "phoneReady": "이 휴대폰에서 사용 가능", + "phoneNeedsDownload": "언어 다운로드 필요", + "phoneDownloading": "다운로드 중… {{progress}}%", + "phoneUnsupported": "이 휴대폰의 언어에서는 사용할 수 없음", + "download": "이 휴대폰에 다운로드" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/pt.json b/mobile/src/i18n/locales/pt.json index 4c279a73b47..b1e227bd590 100644 --- a/mobile/src/i18n/locales/pt.json +++ b/mobile/src/i18n/locales/pt.json @@ -1510,7 +1510,9 @@ "microphoneInitFailed": "Falha ao inicializar o microfone", "microphoneInterrupted": "O ditado por voz parou porque o uso do microfone foi interrompido. Tente novamente.", "microphonePermissionDenied": "Permissão de acesso ao microfone negada", - "noSpeech": "Nenhuma fala detectada." + "noSpeech": "Nenhuma fala detectada.", + "phoneUnavailable": "O ditado neste celular não está pronto. Baixe-o nos ajustes de Voz ou escolha Automático.", + "phoneFailed": "O ditado neste celular parou de funcionar. Tente novamente." } }, "host": { @@ -2898,6 +2900,18 @@ }, "agents": { "permissionOwnAll": "Configurações próprias dos agentes" + }, + "voiceTranscribeOn": { + "heading": "TRANSCREVER EM", + "description": "O que este celular ouve fica nele. Automático usa seu computador quando este celular não consegue transcrever.", + "automatic": "Automático", + "phone": "Este celular", + "computer": "Computador", + "phoneReady": "Pronto neste celular", + "phoneNeedsDownload": "Precisa baixar o idioma", + "phoneDownloading": "Baixando… {{progress}}%", + "phoneUnsupported": "Indisponível para o idioma deste celular", + "download": "Baixar para este celular" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/ru.json b/mobile/src/i18n/locales/ru.json index 6dc0b555004..a4bb02c619d 100644 --- a/mobile/src/i18n/locales/ru.json +++ b/mobile/src/i18n/locales/ru.json @@ -1549,7 +1549,9 @@ "microphoneInitFailed": "Не удалось инициализировать микрофон", "microphoneInterrupted": "Диктовка остановлена из-за прерывания работы микрофона. Попробуйте ещё раз.", "microphonePermissionDenied": "Доступ к микрофону запрещён", - "noSpeech": "Речь не обнаружена." + "noSpeech": "Речь не обнаружена.", + "phoneUnavailable": "Диктовка на этом телефоне не готова. Скачайте её в настройках голоса или выберите «Автоматически».", + "phoneFailed": "Диктовка на этом телефоне перестала работать. Попробуйте ещё раз." } }, "host": { @@ -2970,6 +2972,18 @@ }, "agents": { "permissionOwnAll": "Собственные настройки агентов" + }, + "voiceTranscribeOn": { + "heading": "ГДЕ РАСПОЗНАВАТЬ", + "description": "То, что слышит этот телефон, остаётся на нём. «Автоматически» использует компьютер, когда телефон не может распознать речь.", + "automatic": "Автоматически", + "phone": "Этот телефон", + "computer": "Компьютер", + "phoneReady": "Готово на этом телефоне", + "phoneNeedsDownload": "Нужно скачать язык", + "phoneDownloading": "Загрузка… {{progress}}%", + "phoneUnsupported": "Недоступно для языка этого телефона", + "download": "Скачать на этот телефон" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/tr.json b/mobile/src/i18n/locales/tr.json index ff77fa8169e..24ced71a142 100644 --- a/mobile/src/i18n/locales/tr.json +++ b/mobile/src/i18n/locales/tr.json @@ -1471,7 +1471,9 @@ "microphoneInitFailed": "Mikrofon başlatılamadı", "microphoneInterrupted": "Mikrofon kesintisi nedeniyle sesli dikte durdu. Tekrar deneyin.", "microphonePermissionDenied": "Mikrofon izni reddedildi", - "noSpeech": "Konuşma algılanmadı." + "noSpeech": "Konuşma algılanmadı.", + "phoneUnavailable": "Bu telefonda dikte hazır değil. Ses ayarlarından indirin veya Otomatik’i seçin.", + "phoneFailed": "Bu telefonda dikte çalışmayı durdurdu. Tekrar deneyin." } }, "host": { @@ -2826,6 +2828,18 @@ }, "agents": { "permissionOwnAll": "Ajanların kendi ayarları" + }, + "voiceTranscribeOn": { + "heading": "YAZIYA DÖKME YERİ", + "description": "Bu telefonun duyduğu konuşma telefonda kalır. Otomatik, bu telefon yazıya dökemediğinde bilgisayarınızı kullanır.", + "automatic": "Otomatik", + "phone": "Bu telefon", + "computer": "Bilgisayar", + "phoneReady": "Bu telefonda hazır", + "phoneNeedsDownload": "Dil indirmesi gerekiyor", + "phoneDownloading": "İndiriliyor… %{{progress}}", + "phoneUnsupported": "Bu telefonun dilinde kullanılamıyor", + "download": "Bu telefona indir" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/zh-Hant.json b/mobile/src/i18n/locales/zh-Hant.json index 102e2041d5a..627f9675115 100644 --- a/mobile/src/i18n/locales/zh-Hant.json +++ b/mobile/src/i18n/locales/zh-Hant.json @@ -1467,7 +1467,9 @@ "microphoneInitFailed": "無法初始化麥克風", "microphoneInterrupted": "麥克風遭中斷，語音聽寫已停止。請再試一次。", "microphonePermissionDenied": "麥克風權限遭拒", - "noSpeech": "未偵測到語音。" + "noSpeech": "未偵測到語音。", + "phoneUnavailable": "此手機上的聽寫尚未就緒。請在語音設定中下載，或選擇「自動」。", + "phoneFailed": "此手機上的聽寫已停止運作。請再試一次。" } }, "host": { @@ -2822,6 +2824,18 @@ }, "agents": { "permissionOwnAll": "各代理程式自身的設定" + }, + "voiceTranscribeOn": { + "heading": "轉寫位置", + "description": "此手機聽到的語音只會留在手機上。選擇「自動」時，若此手機無法轉寫，會改用你的電腦。", + "automatic": "自動", + "phone": "此手機", + "computer": "電腦", + "phoneReady": "已可在此手機上使用", + "phoneNeedsDownload": "需要下載語言", + "phoneDownloading": "正在下載… {{progress}}%", + "phoneUnsupported": "不支援此手機的語言", + "download": "下載到此手機" } }, "sourceControl": { diff --git a/mobile/src/i18n/locales/zh.json b/mobile/src/i18n/locales/zh.json index 0944b3af5f1..e4596b506e8 100644 --- a/mobile/src/i18n/locales/zh.json +++ b/mobile/src/i18n/locales/zh.json @@ -1467,7 +1467,9 @@ "microphoneInitFailed": "麦克风初始化失败", "microphoneInterrupted": "麦克风录音中断，语音听写已停止。请重试。", "microphonePermissionDenied": "麦克风权限被拒绝", - "noSpeech": "未检测到语音。" + "noSpeech": "未检测到语音。", + "phoneUnavailable": "此手机上的听写尚未就绪。请在语音设置中下载，或选择“自动”。", + "phoneFailed": "此手机上的听写已停止工作。请重试。" } }, "host": { @@ -2822,6 +2824,18 @@ }, "agents": { "permissionOwnAll": "各智能体自己的设置" + }, + "voiceTranscribeOn": { + "heading": "转写位置", + "description": "此手机听到的语音只留在手机上。选择“自动”时，如果此手机无法转写，会改用你的电脑。", + "automatic": "自动", + "phone": "此手机", + "computer": "电脑", + "phoneReady": "已可在此手机上使用", + "phoneNeedsDownload": "需要下载语言", + "phoneDownloading": "正在下载… {{progress}}%", + "phoneUnsupported": "不支持此手机的语言", + "download": "下载到此手机" } }, "sourceControl": { diff --git a/mobile/src/mobile-web-shell/bridge-host-verdict-hold.ts b/mobile/src/mobile-web-shell/bridge-host-verdict-hold.ts index c15808f98fa..67897d907e9 100644 --- a/mobile/src/mobile-web-shell/bridge-host-verdict-hold.ts +++ b/mobile/src/mobile-web-shell/bridge-host-verdict-hold.ts @@ -28,7 +28,8 @@ const PHONE_ONLY_NATIVE_VERBS: ReadonlySet = new Set([ 'native.audio.read', 'native.audio.stop', 'native.storage.read', - 'native.image.act' + 'native.image.act', + 'native.speech.act' ]) export type BridgeVerdictHold = { diff --git a/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.test.ts b/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.test.ts index 5d070e9dd91..5c89a0ddf2e 100644 --- a/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.test.ts +++ b/mobile/src/mobile-web-shell/bridge/bridge-audio-verbs.test.ts @@ -62,6 +62,7 @@ function createTestEngine( hold: () => screen.push('+'), release: () => screen.push('-') }, + feedSpeech: () => {}, onMicrophoneData: (handler) => { microphone.push(handler) return { diff --git a/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts b/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts index 346a214ac3a..82cead71335 100644 --- a/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts +++ b/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts @@ -18,6 +18,7 @@ import { mediaNameResultSchema } from './bridge-media-verbs' import { imageParamsSchema, imageResultSchema } from './bridge-image-verbs' +import { BRIDGE_SPEECH_VERB, speechParamsSchema, speechResultSchema } from './bridge-speech-verbs' import { BRIDGE_STORE_VERB, storeParamsSchema, storeResultSchema } from './bridge-store-verb' import { PAGE_STORAGE_MAX_KEY_CHARS } from '../page-storage-keys' import { mobileFileMediaMime, PDF_MIME_TYPE } from '../../files/mobile-file-media' @@ -39,6 +40,7 @@ export const BRIDGE_NATIVE_VERB_NAMES = [ 'native.storage.read', 'native.pdf.view', 'native.image.act', + BRIDGE_SPEECH_VERB, BRIDGE_STORE_VERB ] as const @@ -138,6 +140,10 @@ export const BRIDGE_NATIVE_VERBS: Readonly +export type BridgeSpeechStartResult = z.infer +export type BridgeSpeechRead = z.infer +export type BridgeSpeechFinish = z.infer + +// The verb table validates the shell's reply against any action's shape; the page re-parses with the action's own. +export const speechResultSchema = z.union([ + speechStatusResultSchema, + speechStartResultSchema, + speechReadResultSchema, + speechFinishResultSchema, + speechCancelResultSchema +]) diff --git a/mobile/src/mobile-web-shell/bridge/use-native-verbs.ts b/mobile/src/mobile-web-shell/bridge/use-native-verbs.ts index f86c6ea51a8..4383c0ef8e7 100644 --- a/mobile/src/mobile-web-shell/bridge/use-native-verbs.ts +++ b/mobile/src/mobile-web-shell/bridge/use-native-verbs.ts @@ -25,6 +25,18 @@ import { type BridgePdfOpenParams } from './bridge-native-verbs' import { imageResultSchema, type BridgeImageRunParams } from './bridge-image-verbs' +import { + BRIDGE_SPEECH_VERB, + speechCancelResultSchema, + speechFinishResultSchema, + speechReadResultSchema, + speechStartResultSchema, + speechStatusResultSchema, + type BridgeSpeechFinish, + type BridgeSpeechRead, + type BridgeSpeechStartResult, + type BridgeSpeechStatusResult +} from './bridge-speech-verbs' // Use ordinary request frames/caps, served only on-device. Refuse ungranted verbs before posting and parse results because the installed shell can differ from this page build. export type NativeVerbs = { @@ -51,6 +63,13 @@ export type NativeVerbs = { canActOnImages: boolean stageImage: (offset: number, base64: string) => Promise runImageAction: (params: BridgeImageRunParams) => Promise + // Without it dictation goes to the computer exactly as before. + canRecognizeSpeech: boolean + speechStatus: (locale: string) => Promise + speechStart: (locale: string) => Promise + speechRead: () => Promise + speechFinish: () => Promise + speechCancel: () => Promise } // Expose stable reason codes, including local ungranted and frame refusals. Unknown newer-shell codes floor to unreported so caller switches remain exhaustive. @@ -175,6 +194,16 @@ export function useNativeVerbs(): NativeVerbs { }, runImageAction: async (params) => { await call('native.image.act', { action: 'run', ...params }, imageResultSchema) + }, + canRecognizeSpeech: has(BRIDGE_SPEECH_VERB), + speechStatus: (locale) => + call(BRIDGE_SPEECH_VERB, { action: 'status', locale }, speechStatusResultSchema), + speechStart: (locale) => + call(BRIDGE_SPEECH_VERB, { action: 'start', locale }, speechStartResultSchema), + speechRead: () => call(BRIDGE_SPEECH_VERB, { action: 'read' }, speechReadResultSchema), + speechFinish: () => call(BRIDGE_SPEECH_VERB, { action: 'finish' }, speechFinishResultSchema), + speechCancel: async () => { + await call(BRIDGE_SPEECH_VERB, { action: 'cancel' }, speechCancelResultSchema) } } }, [client]) diff --git a/mobile/src/platform/dictation-capture.web.test.tsx b/mobile/src/platform/dictation-capture.web.test.tsx index 2d6bc3daa31..e8d35854e27 100644 --- a/mobile/src/platform/dictation-capture.web.test.tsx +++ b/mobile/src/platform/dictation-capture.web.test.tsx @@ -42,6 +42,7 @@ function createAudioShell( open: async (sampleRate) => ({ opened: options.opens !== false, sampleRate }), begin: () => true, end: () => {}, + feedSpeech: () => {}, onMicrophoneData: (handler) => { microphone = handler return { diff --git a/mobile/src/platform/native-audio-device.ts b/mobile/src/platform/native-audio-device.ts index 78a6a975c99..7ace5684478 100644 --- a/mobile/src/platform/native-audio-device.ts +++ b/mobile/src/platform/native-audio-device.ts @@ -2,6 +2,7 @@ import { addExpoTwoWayAudioEventListener, initialize, requestMicrophonePermissionsAsync, + speechFeed, tearDown, toggleRecording } from '@orca/expo-two-way-audio' @@ -48,6 +49,7 @@ export const nativeAudioDeviceEngine: NativeAudioEngine = { const raw = event.data handler(raw instanceof Uint8Array ? raw : new Uint8Array(raw)) }), + feedSpeech: (bytes) => speechFeed(bytes), onInterruption: (handler) => addExpoTwoWayAudioEventListener('onAudioInterruption', (event) => { // Ignore unknown engine interruption kinds instead of forwarding a value the page cannot describe. diff --git a/mobile/src/platform/native-audio-session-ownership.test.ts b/mobile/src/platform/native-audio-session-ownership.test.ts index 991426d2bfe..60717ebcc53 100644 --- a/mobile/src/platform/native-audio-session-ownership.test.ts +++ b/mobile/src/platform/native-audio-session-ownership.test.ts @@ -50,6 +50,7 @@ function device(blockAt?: 'permission' | 'open') { holds-- } }, + feedSpeech: () => {}, onMicrophoneData: (listener) => { listeners.add(listener) return { diff --git a/mobile/src/platform/native-audio.ts b/mobile/src/platform/native-audio.ts index ca47291624d..472f13de5bc 100644 --- a/mobile/src/platform/native-audio.ts +++ b/mobile/src/platform/native-audio.ts @@ -22,6 +22,8 @@ export type NativeAudioEngine = { // Hold the screen with an open mic because locking suspends capture; inject the device wake API for tests. readonly screenLock: MicrophoneScreenLock readonly onMicrophoneData: (handler: (bytes: Uint8Array) => void) => { remove: () => void } + // The on-device recognizer hears the same samples the page drains; it ignores them with no session. + readonly feedSpeech: (bytes: Uint8Array) => void readonly onInterruption: (handler: (kind: BridgeAudioInterruption) => void) => { remove: () => void } @@ -130,6 +132,7 @@ export function createNativeAudioCapture(engine: NativeAudioEngine): NativeAudio const ring = new NativeAudioRing() const microphone = engine.onMicrophoneData((bytes) => { ring.append(bytes) + engine.feedSpeech(bytes) }) const interruptions = engine.onInterruption((kind) => { if (capture === null) { diff --git a/mobile/src/platform/native-speech-device.ts b/mobile/src/platform/native-speech-device.ts new file mode 100644 index 00000000000..d2e46302426 --- /dev/null +++ b/mobile/src/platform/native-speech-device.ts @@ -0,0 +1,25 @@ +import { + addExpoTwoWayAudioEventListener, + speechAvailability, + speechCancel, + speechFinish, + speechStart +} from '@orca/expo-two-way-audio' +import { loadDictationEnginePreference } from '../storage/dictation-engine-preference' +import { NATIVE_AUDIO_DEVICE_SAMPLE_RATE } from './native-audio-device' +import type { NativeSpeechEngine } from './native-speech' + +// Isolate the device module so tests drive the server with a fake engine. +export const nativeSpeechDeviceEngine: NativeSpeechEngine = { + availability: (locale) => speechAvailability(locale), + start: (locale) => speechStart(locale, NATIVE_AUDIO_DEVICE_SAMPLE_RATE), + finish: () => speechFinish(), + cancel: () => speechCancel(), + readPreference: () => loadDictationEnginePreference(), + onText: (handler) => { + addExpoTwoWayAudioEventListener('onSpeechText', handler) + }, + onError: (handler) => { + addExpoTwoWayAudioEventListener('onSpeechError', (event) => handler(event.code)) + } +} diff --git a/mobile/src/platform/native-speech.test.ts b/mobile/src/platform/native-speech.test.ts new file mode 100644 index 00000000000..9d8ee96ceb1 --- /dev/null +++ b/mobile/src/platform/native-speech.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import type { DictationEnginePreference } from '../mobile-web-shell/bridge/bridge-speech-verbs' +import { createNativeSpeechServer, type NativeSpeechEngine } from './native-speech' + +function engineWith(preference: DictationEnginePreference): NativeSpeechEngine { + return { + availability: async () => ({ status: 'ready', locale: 'en-US' }), + start: async (locale) => ({ started: true, locale }), + finish: async () => '', + cancel: () => {}, + readPreference: async () => preference, + onText: () => {}, + onError: () => {} + } +} + +describe('a page asking for the microphone when the user chose This phone', () => { + it('is refused unless its own recognizer is armed', async () => { + const server = createNativeSpeechServer(engineWith('phone')) + // An older page streams the mic to the computer without arming anything. + expect(await server.mayCapture()).toBe(false) + await server.serve({ action: 'start', locale: 'en-US' }) + expect(await server.mayCapture()).toBe(true) + await server.serve({ action: 'cancel' }) + expect(await server.mayCapture()).toBe(false) + }) + + it('is allowed under Automatic, where the computer is a fallback', async () => { + const server = createNativeSpeechServer(engineWith('automatic')) + expect(await server.mayCapture()).toBe(true) + }) +}) + +describe('a recognizer another page took over', () => { + it('reports the first page’s session as failed so it falls back', async () => { + const engine = engineWith('automatic') + const first = createNativeSpeechServer(engine) + const second = createNativeSpeechServer(engine) + await first.serve({ action: 'start', locale: 'en-US' }) + await second.serve({ action: 'start', locale: 'en-US' }) + expect(await first.serve({ action: 'read' })).toEqual({ + revision: 0, + committed: '', + tentative: '', + failure: 'failed' + }) + expect(await second.serve({ action: 'read' })).toMatchObject({ failure: null }) + }) +}) diff --git a/mobile/src/platform/native-speech.ts b/mobile/src/platform/native-speech.ts new file mode 100644 index 00000000000..8743986d678 --- /dev/null +++ b/mobile/src/platform/native-speech.ts @@ -0,0 +1,172 @@ +import { + BRIDGE_SPEECH_FAILURES, + speechParamsSchema, + type BridgeSpeechFailure, + type BridgeSpeechRead, + type DictationEnginePreference +} from '../mobile-web-shell/bridge/bridge-speech-verbs' + +// The shell side of native.speech.act. The recognizer itself lives in the native audio module; this keeps its events between the page's polls. + +export type NativeSpeechEngine = { + readonly availability: ( + locale: string + ) => Promise<{ status: 'ready' | 'needs-download' | 'unsupported'; locale: string | null }> + readonly start: ( + locale: string + ) => Promise<{ started: true; locale: string } | { started: false; reason: string }> + readonly finish: () => Promise + readonly cancel: () => void + readonly readPreference: () => Promise + readonly onText: (handler: (text: Omit) => void) => void + readonly onError: (handler: (code: string) => void) => void +} + +export type NativeSpeechServer = { + readonly serve: (params: unknown) => Promise + // This phone promises audio never reaches the computer; only a page whose recognizer is armed may open the mic. Older pages stream to the computer and are refused. + readonly mayCapture: () => Promise + readonly dispose: () => void +} + +type Session = { text: Omit; failure: BridgeSpeechFailure | null } + +// The recognizer is one per phone, so its live session is per engine; each server holds only its own. +type EngineState = { live: Session | null } + +const engineStates = new WeakMap() + +// A newer engine's code the shipped page does not know still means "use the computer". +function failureOf(code: string): BridgeSpeechFailure { + return BRIDGE_SPEECH_FAILURES.find((known) => known === code) ?? 'failed' +} + +function engineState(engine: NativeSpeechEngine): EngineState { + const known = engineStates.get(engine) + if (known !== undefined) { + return known + } + const state: EngineState = { live: null } + engineStates.set(engine, state) + // Subscribed once per engine for the app's life, so no page session can miss or double an event. + engine.onText((text) => { + if (state.live !== null && text.revision > state.live.text.revision) { + state.live.text = text + } + }) + engine.onError((code) => { + if (state.live !== null) { + state.live.failure ??= failureOf(code) + } + }) + return state +} + +export function createNativeSpeechServer(engine: NativeSpeechEngine): NativeSpeechServer { + const shared = engineState(engine) + let session: Session | null = null + + // Another page's start takes the recognizer; this session is then over. + function owned(): Session | null { + return session !== null && session === shared.live ? session : null + } + + function endSession(): void { + if (owned() !== null) { + shared.live = null + engine.cancel() + } + session = null + } + + async function status(locale: string): Promise { + // Answer the preference even when the recognizer can't say, so This phone never falls to the computer. + const [preference, available] = await Promise.all([ + engine.readPreference(), + engine.availability(locale).catch(() => null) + ]) + return available === null + ? { status: 'unsupported', locale: null, preference } + : { status: available.status, locale: available.locale, preference } + } + + async function start(locale: string): Promise { + endSession() + // Claim before the await so events from the moment native arms land here; the engine's own start replaces any other page's session. + const claimed: Session = { text: { revision: 0, committed: '', tentative: '' }, failure: null } + session = claimed + shared.live = claimed + const started = await engine + .start(locale) + .catch(() => ({ started: false as const, reason: 'failed' })) + const kept = session === claimed && shared.live === claimed + // Cancelled while arming: stop what native armed, unless another page's start already replaced it. + if (started.started && !kept && (shared.live === null || shared.live === claimed)) { + engine.cancel() + } + if (!started.started || !kept) { + if (shared.live === claimed) { + shared.live = null + } + if (session === claimed) { + session = null + } + return { + started: false, + locale: null, + reason: started.started ? 'busy' : failureOf(started.reason) + } + } + return { started: true, locale: started.locale, reason: null } + } + + function read(): unknown { + const live = owned() + // No session here means it was lost or taken; the page falls back to the computer. + if (live === null) { + return { revision: 0, committed: '', tentative: '', failure: 'failed' } + } + return { ...live.text, failure: live.failure } + } + + async function finish(): Promise { + const live = owned() + if (live === null) { + return { text: null, failure: 'failed' } + } + try { + const text = await engine.finish() + return live.failure === null ? { text, failure: null } : { text: null, failure: live.failure } + } catch { + return { text: null, failure: live.failure ?? 'failed' } + } finally { + if (shared.live === live) { + shared.live = null + } + if (session === live) { + session = null + } + } + } + + return { + serve: async (params) => { + const call = speechParamsSchema.parse(params) + switch (call.action) { + case 'status': + return status(call.locale) + case 'start': + return start(call.locale) + case 'read': + return read() + case 'finish': + return finish() + case 'cancel': + endSession() + return {} + } + }, + mayCapture: async () => owned() !== null || (await engine.readPreference()) !== 'phone', + dispose: endSession + } +} diff --git a/mobile/src/platform/on-device-speech-contract.ts b/mobile/src/platform/on-device-speech-contract.ts new file mode 100644 index 00000000000..3fc61546b91 --- /dev/null +++ b/mobile/src/platform/on-device-speech-contract.ts @@ -0,0 +1,15 @@ +import type { + BridgeSpeechFinish, + BridgeSpeechRead, + BridgeSpeechStartResult, + BridgeSpeechStatusResult +} from '../mobile-web-shell/bridge/bridge-speech-verbs' + +// The phone's own recognizer as dictation sees it, in the phone's language. Any rejection means "use the computer". +export type OnDeviceSpeech = { + readonly status: () => Promise + readonly start: () => Promise + readonly read: () => Promise + readonly finish: () => Promise + readonly cancel: () => void +} diff --git a/mobile/src/platform/on-device-speech-locale.ts b/mobile/src/platform/on-device-speech-locale.ts new file mode 100644 index 00000000000..cf5edb0684e --- /dev/null +++ b/mobile/src/platform/on-device-speech-locale.ts @@ -0,0 +1,6 @@ +import { readDeviceUiLocaleTags } from '../i18n/system-ui-locale' + +// The phone's first language, read the same way in the shell's settings and the page, so both ask the recognizer about one locale. +export function onDeviceSpeechLocale(): string { + return readDeviceUiLocaleTags()[0] ?? 'en-US' +} diff --git a/mobile/src/platform/on-device-speech-locale.web.ts b/mobile/src/platform/on-device-speech-locale.web.ts new file mode 100644 index 00000000000..3bb07703f29 --- /dev/null +++ b/mobile/src/platform/on-device-speech-locale.web.ts @@ -0,0 +1,4 @@ +// The page asks the WebView, which reports the phone's own language list; expo-localization stays out of the page bundle. +export function onDeviceSpeechLocale(): string { + return globalThis.navigator?.languages?.[0] ?? globalThis.navigator?.language ?? 'en-US' +} diff --git a/mobile/src/platform/on-device-speech.ts b/mobile/src/platform/on-device-speech.ts new file mode 100644 index 00000000000..06c08ca49ce --- /dev/null +++ b/mobile/src/platform/on-device-speech.ts @@ -0,0 +1,6 @@ +import type { OnDeviceSpeech } from './on-device-speech-contract' + +// The session screen is a page in the shell (on-device-speech.web.ts); native screens dictate through the computer. +export function useOnDeviceSpeech(): OnDeviceSpeech | null { + return null +} diff --git a/mobile/src/platform/on-device-speech.web.ts b/mobile/src/platform/on-device-speech.web.ts new file mode 100644 index 00000000000..b11eed5ddda --- /dev/null +++ b/mobile/src/platform/on-device-speech.web.ts @@ -0,0 +1,23 @@ +import { useMemo } from 'react' +import { useNativeVerbs, type NativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs' +import type { OnDeviceSpeech } from './on-device-speech-contract' +import { onDeviceSpeechLocale } from './on-device-speech-locale' + +function createPageOnDeviceSpeech(verbs: NativeVerbs): OnDeviceSpeech { + return { + status: () => verbs.speechStatus(onDeviceSpeechLocale()), + start: () => verbs.speechStart(onDeviceSpeechLocale()), + read: verbs.speechRead, + finish: verbs.speechFinish, + // Cancel has no reply anyone waits on; a shell that lost the session has nothing to stop. + cancel: () => { + void verbs.speechCancel().catch(() => undefined) + } + } +} + +// Null on a shell without the verb, so dictation stays on the computer exactly as before. +export function useOnDeviceSpeech(): OnDeviceSpeech | null { + const verbs = useNativeVerbs() + return useMemo(() => (verbs.canRecognizeSpeech ? createPageOnDeviceSpeech(verbs) : null), [verbs]) +} diff --git a/mobile/src/platform/use-native-device-verbs.ts b/mobile/src/platform/use-native-device-verbs.ts index 3a368541327..c26f8ecee89 100644 --- a/mobile/src/platform/use-native-device-verbs.ts +++ b/mobile/src/platform/use-native-device-verbs.ts @@ -2,7 +2,9 @@ import { useEffect, useMemo } from 'react' import type { BridgeNativeVerb } from '../mobile-web-shell/bridge/bridge-native-verbs' import { useMediaHandleRegistry } from '../mobile-web-shell/use-media-handle-registry' import { createNativeAudioCapture } from './native-audio' -import { nativeAudioDeviceEngine } from './native-audio-device' +import { NATIVE_AUDIO_DEVICE_SAMPLE_RATE, nativeAudioDeviceEngine } from './native-audio-device' +import { createNativeSpeechServer } from './native-speech' +import { nativeSpeechDeviceEngine } from './native-speech-device' import { serveNativeClipboardVerb } from './native-clipboard' import { createNativeMediaVerbServer } from './native-media' import { discardStagedMedia, nativeMediaDeviceDeps } from './native-media-device' @@ -24,13 +26,29 @@ export function useNativeDeviceVerbs( // A new session is a new document; close any mic its predecessor left open. const audio = useMemo(() => createNativeAudioCapture(nativeAudioDeviceEngine), [sessionId]) useEffect(() => () => audio.dispose(), [audio]) + const speech = useMemo(() => createNativeSpeechServer(nativeSpeechDeviceEngine), [sessionId]) + useEffect(() => () => speech.dispose(), [speech]) return useMemo( () => (verb, params) => { if (verb === 'native.clipboard.write' || verb === 'native.clipboard.read') { return serveNativeClipboardVerb(verb, params) } + if (verb === 'native.speech.act') { + return speech.serve(params) + } + if (verb === 'native.audio.start') { + return speech.mayCapture().then((may) => + may + ? audio.serve(verb, params) + : { + started: false, + sampleRate: NATIVE_AUDIO_DEVICE_SAMPLE_RATE, + permission: 'granted' + } + ) + } return verb.startsWith('native.audio.') ? audio.serve(verb, params) : serveMedia(verb, params) }, - [audio, serveMedia] + [audio, serveMedia, speech] ) } diff --git a/mobile/src/session/use-mobile-session-native-chat-dictation.ts b/mobile/src/session/use-mobile-session-native-chat-dictation.ts index a74110519c7..e23c7a6d904 100644 --- a/mobile/src/session/use-mobile-session-native-chat-dictation.ts +++ b/mobile/src/session/use-mobile-session-native-chat-dictation.ts @@ -2,6 +2,7 @@ import { useEffect, useCallback, useRef } from 'react' import { AppState } from 'react-native' import { useFocusEffect } from 'expo-router' import { useMobileDictation } from '../hooks/use-mobile-dictation' +import { usePhoneDictationReady } from '../hooks/use-phone-dictation-ready' import { triggerError } from '../platform/haptics' import { useClipboardWriter } from '../platform/clipboard' import { @@ -129,8 +130,12 @@ export function useMobileSessionNativeChatDictation( } const destination = useMobileDictationDestination(nativeChatController) - // Starting needs the host, but a reconnect must not end a recording: the host keeps the session through it. - const dictationCanStart = showNativeChat ? connState === 'connected' : canSend + // The phone's own recognizer needs no computer setup, and chat inserts its words on the phone. + const phoneDictation = usePhoneDictationReady() + // Starting needs the host (unless the phone hears chat itself), but a reconnect must not end a recording: the host keeps the session through it. + const dictationCanStart = showNativeChat + ? connState === 'connected' || phoneDictation.ready + : canSend const dictation = useMobileDictation({ client, enabled: (showNativeChat || canCompose) && dictationEnabled, @@ -174,7 +179,7 @@ export function useMobileSessionNativeChatDictation( if (!dictationCanStart) { return } - if (dictationNeedsSetupRef.current) { + if (dictationNeedsSetupRef.current && !phoneDictation.readyRef.current) { // Explain setup before the OS microphone prompt: mic access cannot help until the host can transcribe. setShowDictationSetup(true) return @@ -193,6 +198,7 @@ export function useMobileSessionNativeChatDictation( destination.captureDestination, dictation, dictationCanStart, + phoneDictation.readyRef, reportDictationFailure, setShowDictationSetup, showNativeChat @@ -232,6 +238,7 @@ export function useMobileSessionNativeChatDictation( const dictationSettingsReadRef = useRef(0) const refreshDictationMode = useCallback(async () => { const read = ++dictationSettingsReadRef.current + void phoneDictation.refresh() if (!client || connState !== 'connected') { return } @@ -245,7 +252,7 @@ export function useMobileSessionNativeChatDictation( } catch { // Keep the last settings when an older host cannot report them. } - }, [client, connState, setDictationEnabled, setDictationMode]) + }, [client, connState, phoneDictation.refresh, setDictationEnabled, setDictationMode]) // The sheet may have changed host setup: let the host judge taps until the fresh read lands. const closeDictationSetup = useCallback(() => { diff --git a/mobile/src/settings/settings-screen-state.test.tsx b/mobile/src/settings/settings-screen-state.test.tsx index f690d01702a..6bd21b6b58c 100644 --- a/mobile/src/settings/settings-screen-state.test.tsx +++ b/mobile/src/settings/settings-screen-state.test.tsx @@ -37,6 +37,10 @@ vi.mock('../components/bottom-drawer/BottomDrawer', () => ({ BottomDrawer: ({ children }: { children: ReactNode }) => children })) vi.mock('../components/VoiceModelList', () => ({ VoiceModelList: () => null })) +vi.mock('./voice-transcribe-on-section', () => ({ + VoiceTranscribeOnSection: () => null, + voiceTranscribeOnShown: () => false +})) vi.mock('../dictation/use-dictation-setup-poller', () => ({ useDictationSetupPoller: ({ refresh }: { refresh: () => Promise }) => { useEffect(() => { diff --git a/mobile/src/settings/voice-settings-screen.tsx b/mobile/src/settings/voice-settings-screen.tsx index 3bfc37cc988..1766b7512d5 100644 --- a/mobile/src/settings/voice-settings-screen.tsx +++ b/mobile/src/settings/voice-settings-screen.tsx @@ -18,6 +18,7 @@ import { } from '../dictation/mobile-dictation-setup' import { MobileSwitch } from '../components/MobileSwitch' import { SettingsNote, SettingsScreen, SettingsSection } from './settings-screen-frame' +import { VoiceTranscribeOnSection, voiceTranscribeOnShown } from './voice-transcribe-on-section' const POLL_INTERVAL_MS = 1500 @@ -274,6 +275,7 @@ export default function VoiceSettingsScreen({ } > + {showLoading ? ( ( + DEFAULT_DICTATION_ENGINE_PREFERENCE + ) + const [phone, setPhone] = useState(null) + // Progress events and the download's promise arrive apart; a late event must not reopen a finished download. + const downloadingRef = useRef(false) + const supported = voiceTranscribeOnShown() + + const readPhone = useCallback(async () => { + const available = await speechAvailability(onDeviceSpeechLocale()).catch(() => null) + setPhone(available === null ? { status: 'unsupported' } : { status: available.status }) + }, []) + + useEffect(() => { + if (!supported) { + return + } + void loadDictationEnginePreference().then(setPreference) + void readPhone() + const progress = addExpoTwoWayAudioEventListener('onSpeechDownloadProgress', (event) => { + if (!downloadingRef.current) { + return + } + setPhone({ status: 'downloading', progress: Math.min(1, Math.max(0, event.fraction)) }) + }) + return () => progress.remove() + }, [readPhone, supported]) + + if (!supported) { + return null + } + + const choose = (next: DictationEnginePreference) => { + setPreference(next) + void saveDictationEnginePreference(next).catch(() => {}) + } + + const download = () => { + if (downloadingRef.current) { + return + } + downloadingRef.current = true + setPhone({ status: 'downloading', progress: 0 }) + void speechDownload(onDeviceSpeechLocale()) + .catch(() => false) + .then(() => { + downloadingRef.current = false + return readPhone() + }) + } + + return ( + + choose('automatic')} + /> + choose('phone')} + /> + choose('computer')} + /> + {phone?.status === 'needs-download' || phone?.status === 'downloading' ? ( + + ) : null} + + ) +} diff --git a/mobile/src/storage/dictation-engine-preference.ts b/mobile/src/storage/dictation-engine-preference.ts new file mode 100644 index 00000000000..8716b673621 --- /dev/null +++ b/mobile/src/storage/dictation-engine-preference.ts @@ -0,0 +1,36 @@ +import AsyncStorage from '@react-native-async-storage/async-storage' +import { + DICTATION_ENGINE_PREFERENCES, + type DictationEnginePreference +} from '../mobile-web-shell/bridge/bridge-speech-verbs' + +// Where the phone transcribes dictation. Phone-only: the shell answers it to the page with the speech status. +const DICTATION_ENGINE_PREFERENCE_KEY = 'orca:dictationEngine' + +export const DEFAULT_DICTATION_ENGINE_PREFERENCE: DictationEnginePreference = 'automatic' + +// The shell reads it before every dictation's status and mic start; only this module writes it, so keep it in memory once read. +let cached: DictationEnginePreference | null = null + +export async function loadDictationEnginePreference(): Promise { + if (cached !== null) { + return cached + } + try { + const raw = await AsyncStorage.getItem(DICTATION_ENGINE_PREFERENCE_KEY) + // A save during the read wins over what the read found. + cached ??= + DICTATION_ENGINE_PREFERENCES.find((known) => known === raw) ?? + DEFAULT_DICTATION_ENGINE_PREFERENCE + return cached + } catch { + return DEFAULT_DICTATION_ENGINE_PREFERENCE + } +} + +export async function saveDictationEnginePreference( + preference: DictationEnginePreference +): Promise { + cached = preference + await AsyncStorage.setItem(DICTATION_ENGINE_PREFERENCE_KEY, preference) +} diff --git a/mobile/web-entry/web-overrides.json b/mobile/web-entry/web-overrides.json index 880961111cd..fd3060a3cc1 100644 --- a/mobile/web-entry/web-overrides.json +++ b/mobile/web-entry/web-overrides.json @@ -145,6 +145,14 @@ "file": "src/platform/dictation-capture.web.ts", "reason": "Dictation's capture seam. The native file holds the microphone through @orca/expo-two-way-audio, which a browser does not have; this one asks the shell for it over native.audio.start|read|stop, draining the shell's ring on a timer and raising each reply as the events the engine emits directly, and taking the utterance's tail off the stop's own reply. The screen an open microphone holds awake is not in this seam at all: the device module that opens the mic takes it and gives it back on both hosts. The flow above the seam is one file on both hosts." }, + { + "file": "src/platform/on-device-speech.web.ts", + "reason": "Dictation's on-device recognizer. Native screens never dictate on the phone, so the native file answers null; the session page asks the shell over native.speech.act, whose recognizer hears the microphone the shell already captures, and answers null on shells that do not grant the verb so dictation goes to the computer as before." + }, + { + "file": "src/platform/on-device-speech-locale.web.ts", + "reason": "The language the phone's recognizer is asked about. The native file reads the device's language list through expo-localization, which the page does not load; the WebView's navigator.languages is that same list, so the shell's Voice settings and the session page ask about one locale." + }, { "file": "src/platform/pdf-opener.web.ts", "reason": "Native file routes never render a preview themselves: they mount this page, which cannot show a PDF because the shell's CSP refuses frames and objects. This one asks the shell through native.pdf.view, which downloads the file and shows it in Quick Look or the Android PDF viewer, cancels that download when the preview is left, and returns null on shells that do not grant the verb so the preview asks for an app update."