From a5dbd1cb2ff9907615f4444eb4763bb7cbc13eb7 Mon Sep 17 00:00:00 2001
From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com>
Date: Mon, 9 Nov 2026 22:52:59 +0000
Subject: [PATCH] feat(mobile): read dictation progress to VoiceOver and
TalkBack (#40504)
* feat(mobile): announce dictation to VoiceOver through the shell
* feat(mobile): speak dictation from the host, so leaving the session and the pill's toasts are announced too
* fix(mobile): say dictation outcomes where they are known, and re-read a repeated live message
* fix(mobile): leave a setup refusal to its setup sheet instead of reading the raw code
* fix(mobile): say cancelled when a kept recording is discarded
---
.../mobile-web-app/mobile-web-page-routes.mjs | 3 +-
mobile/src/dictation/MobileDictationPill.tsx | 10 +-
.../MobileDictationStatusAnnouncement.tsx | 12 +-
.../mobile-dictation-announce-copy.ts | 39 ++++++
.../mobile-dictation-host-context.ts | 7 ++
.../src/dictation/mobile-dictation-host.tsx | 61 ++++++++--
.../use-mobile-dictation-announcements.ts | 113 ++++++++++++++++++
.../dictation/use-mobile-dictation-live.ts | 12 +-
mobile/src/i18n/locales/de.json | 12 +-
mobile/src/i18n/locales/en.json | 12 +-
mobile/src/i18n/locales/es.json | 12 +-
mobile/src/i18n/locales/fr.json | 12 +-
mobile/src/i18n/locales/ja.json | 12 +-
mobile/src/i18n/locales/ko.json | 12 +-
mobile/src/i18n/locales/pt.json | 12 +-
mobile/src/i18n/locales/ru.json | 12 +-
mobile/src/i18n/locales/tr.json | 12 +-
mobile/src/i18n/locales/zh-Hant.json | 12 +-
mobile/src/i18n/locales/zh.json | 12 +-
.../bridge/bridge-native-verbs.ts | 12 +-
.../bridge/use-native-verbs.ts | 10 +-
.../src/platform/accessibility-announcer.ts | 10 ++
.../platform/accessibility-announcer.web.ts | 16 +++
.../platform/native-accessibility-announce.ts | 12 ++
.../platform/use-native-device-verbs.test.tsx | 5 +
.../src/platform/use-native-device-verbs.ts | 4 +
.../src/session/MobileSessionContentRow.tsx | 2 -
...mobile-dictation-session-outcomes.test.tsx | 3 +
.../session/use-mobile-session-dictation.ts | 22 +++-
mobile/web-entry/web-overrides.json | 4 +
30 files changed, 443 insertions(+), 46 deletions(-)
create mode 100644 mobile/src/dictation/mobile-dictation-announce-copy.ts
create mode 100644 mobile/src/dictation/use-mobile-dictation-announcements.ts
create mode 100644 mobile/src/platform/accessibility-announcer.ts
create mode 100644 mobile/src/platform/accessibility-announcer.web.ts
create mode 100644 mobile/src/platform/native-accessibility-announce.ts
diff --git a/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs b/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs
index 1e391c41ccc..79d0482fb59 100644
--- a/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs
+++ b/config/scripts/build/mobile-web-app/mobile-web-page-routes.mjs
@@ -102,13 +102,14 @@ export const MOBILE_WEB_PAGE_ROUTES = [
'native.audio.read',
'native.audio.stop'
],
- // Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act; dictation uses the computer without native.speech.act.
+ // Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act; dictation uses the computer without native.speech.act; without native.a11y.announce only the page's live region speaks.
optionalGrants: [
'externalNavigation',
'native.media.name',
'native.pdf.view',
'native.image.act',
'native.speech.act',
+ 'native.a11y.announce',
'terminalKeyRail',
DEVICE_STORE_GRANT
]
diff --git a/mobile/src/dictation/MobileDictationPill.tsx b/mobile/src/dictation/MobileDictationPill.tsx
index 56f33ae8c50..e257844d486 100644
--- a/mobile/src/dictation/MobileDictationPill.tsx
+++ b/mobile/src/dictation/MobileDictationPill.tsx
@@ -24,6 +24,7 @@ import {
} from './mobile-dictation-controls-copy'
import { dictationAwayCopy } from './mobile-dictation-away-copy'
import { composerInputStyles } from '../components/composer-input/composer-input-styles'
+import { MobileDictationStatusAnnouncement } from './MobileDictationStatusAnnouncement'
import {
useMobileDictationHost,
useMobileDictationHostLive,
@@ -163,13 +164,9 @@ function MobileDictationToast({
const timer = setTimeout(onDone, toast.open ? TOAST_WITH_OPEN_MS : TOAST_MS)
return () => clearTimeout(timer)
}, [onDone, toast])
- // The live region stays mounted so a screen reader hears the message appear.
+ // The host's announcer reads the message (shell or live region), so the toast itself stays quiet.
return (
-
+
{toast ? (
@@ -212,6 +209,7 @@ export function MobileDictationPillLayer(): React.JSX.Element | null {
) : null}
+
- {mobileDictationStatusTitle(dictation) ?? ''}
+ {liveText.text}
)
}
diff --git a/mobile/src/dictation/mobile-dictation-announce-copy.ts b/mobile/src/dictation/mobile-dictation-announce-copy.ts
new file mode 100644
index 00000000000..5a7d3bdab18
--- /dev/null
+++ b/mobile/src/dictation/mobile-dictation-announce-copy.ts
@@ -0,0 +1,39 @@
+import { translate } from '../i18n/mobile-i18n'
+
+// What a screen reader hears as a dictation moves along; nothing while you talk, so it never talks over you.
+export const dictationAnnounceCopy = {
+ listening: (engine: 'phone' | 'computer', computer: string | null) =>
+ engine === 'phone'
+ ? translate('dictation.announce.listeningOnPhone', 'Listening. Transcribing on this phone.')
+ : computer
+ ? translate('dictation.announce.listeningOn', 'Listening. Transcribing on {{computer}}.', {
+ computer
+ })
+ : translate(
+ 'dictation.announce.listeningOnComputer',
+ 'Listening. Transcribing on the computer.'
+ ),
+ stillListening: () => translate('dictation.announce.stillListening', 'Still listening.'),
+ finishing: () => translate('dictation.announce.finishing', 'Finishing.'),
+ inserted: () => translate('dictation.announce.inserted', 'Inserted.'),
+ cancelled: () => translate('dictation.announce.cancelled', 'Dictation cancelled.'),
+ movedToComputer: (computer: string | null) =>
+ computer
+ ? translate(
+ 'dictation.announce.movedTo',
+ 'This phone stopped transcribing. Continuing on {{computer}}.',
+ { computer }
+ )
+ : translate(
+ 'dictation.announce.movedToComputer',
+ 'This phone stopped transcribing. Continuing on the computer.'
+ ),
+ stoppedKept: (reason: string) =>
+ translate(
+ 'dictation.announce.stoppedKept',
+ 'Dictation stopped. {{reason}}. Your recording is kept.',
+ {
+ reason
+ }
+ )
+}
diff --git a/mobile/src/dictation/mobile-dictation-host-context.ts b/mobile/src/dictation/mobile-dictation-host-context.ts
index 240a31f50b2..9f4d3c23798 100644
--- a/mobile/src/dictation/mobile-dictation-host-context.ts
+++ b/mobile/src/dictation/mobile-dictation-host-context.ts
@@ -2,6 +2,7 @@ import { createContext, useContext, useEffect, useState, useSyncExternalStore }
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
import type { MobileDictationAwayTarget } from './mobile-dictation-away-insert'
+import type { MobileDictationLiveText } from './use-mobile-dictation-announcements'
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
export type MobileDictationHostToast = { message: string; open?: () => void }
@@ -25,6 +26,8 @@ export type MobileDictationOwner = {
readonly dictationTerms: readonly DictationTerm[] | undefined
// The computer spells its own words with this worktree's repo terms.
readonly worktreeId: string
+ // The computer that would transcribe it, for what a screen reader hears; null when the phone lacks its name.
+ readonly computerName: string | null
}
// A mounted session screen, as the dictation host sees it.
@@ -48,6 +51,8 @@ export type MobileDictationHost = {
readonly start: (owner: MobileDictationOwner) => Promise
// Stop with the selection tick, as every stop button does.
readonly stop: () => void
+ // Say something to the screen reader; a screen's own toast is not read on the page.
+ readonly announce: (message: string) => void
readonly attach: (screen: MobileDictationScreen) => () => void
// A screen's shown() answer changed; the pill reads it on render.
readonly refresh: () => void
@@ -69,6 +74,8 @@ export type MobileDictationPill = {
readonly clearToast: () => void
// The tallest bottom bar a screen reported, so the pill floats above it.
readonly clearance: number
+ // Announcements the shell could not speak, for the page's live region (TalkBack, older shells).
+ readonly liveText: MobileDictationLiveText
}
export const MobileDictationHostContext = createContext(null)
diff --git a/mobile/src/dictation/mobile-dictation-host.tsx b/mobile/src/dictation/mobile-dictation-host.tsx
index c4960704a7a..b2276df26aa 100644
--- a/mobile/src/dictation/mobile-dictation-host.tsx
+++ b/mobile/src/dictation/mobile-dictation-host.tsx
@@ -4,12 +4,16 @@ import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session
import { useRouteHandoff } from '../navigation/route-handoff'
import { triggerError } from '../platform/haptics'
import { useClipboardWriter } from '../platform/clipboard'
-import { dictationFailureMessage } from './mobile-dictation-setup'
+import { dictationFailureMessage, isDictationSetupRequiredError } from './mobile-dictation-setup'
import { useMobileDictationHaptics } from './use-mobile-dictation-haptics'
import { useMobileDictationSilenceStop } from './use-mobile-dictation-silence-stop'
import { insertMobileDictationAway } from './mobile-dictation-away-insert'
import { nativeChatDictationCopy } from './mobile-session-native-chat-dictation-copy'
import { dictationAwayCopy } from './mobile-dictation-away-copy'
+import {
+ useMobileDictationAnnouncements,
+ useMobileDictationSpeech
+} from './use-mobile-dictation-announcements'
import {
MobileDictationClearanceContext,
MobileDictationHostContext,
@@ -58,6 +62,8 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
const clearToast = useCallback(() => setToast(null), [])
const [clearances, setClearances] = useState>(new Map())
const clipboard = useClipboardWriter()
+ const speech = useMobileDictationSpeech()
+ const { say: speak } = speech
// Bring a dictation's target back: its screen if one is still mounted, otherwise open the session again.
const openOwner = useCallback(
@@ -78,10 +84,12 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
}, [openOwner])
// Say it on the screen that shows the dictation, else in the host's toast (with Open for the dictation it is about).
+ // A screen's toast is not read by VoiceOver on the page, so it is spoken too; the host's toast is spoken when it shows.
const say = (message: string, about?: MobileDictationOwner) => {
const screen = screenFor(screensRef.current, ownerRef.current, false)
if (screen?.shown()) {
screen.toast(message)
+ speak(message)
} else {
setToast(about ? { message, open: () => openOwner(about) } : { message })
}
@@ -90,6 +98,10 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
const screen = screenFor(screensRef.current, ownerRef.current, false)
if (screen?.shown()) {
screen.reportFailure(err)
+ // A setup refusal opens the setup sheet, which is read on its own.
+ if (!isDictationSetupRequiredError(err.message)) {
+ speak(dictationFailureMessage(err.message))
+ }
return
}
triggerError()
@@ -136,9 +148,30 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
useMobileDictationSilenceStop(dictation, owner?.dictationMode, stop, (message) => say(message))
// Screens subscribe to the live dictation; the host's context only changes when its owner or pill does.
- const liveRef = useRef(dictation)
+ const liveRef = useRef(dictation)
const listenersRef = useRef(new Set<() => void>())
- const { meter, start: startDictation, stop: stopDictation, cancel, retry } = dictation
+ const {
+ meter,
+ start: startDictation,
+ stop: stopDictation,
+ cancel: cancelDictation,
+ retry
+ } = dictation
+ const away =
+ owner !== null &&
+ isUnderway(dictation) &&
+ screenFor(screensRef.current, owner, false)?.shown() !== true
+ const { noteCancel } = useMobileDictationAnnouncements(
+ dictation,
+ { computerName: owner?.computerName ?? null, away, toast: toast?.message ?? null },
+ speak
+ )
+ // Every ✕ (in place, the pill, the terminal's long press) ends here, so a cancel is never read as "Inserted."
+ const cancel = useCallback(() => {
+ noteCancel()
+ return cancelDictation()
+ }, [cancelDictation, noteCancel])
+ const published = useMemo(() => ({ ...dictation, cancel }), [cancel, dictation])
const idle = useMemo(
() => ({
status: 'idle',
@@ -159,7 +192,7 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
)
const idleRef = useRef(idle)
useLayoutEffect(() => {
- liveRef.current = dictation
+ liveRef.current = published
idleRef.current = idle
for (const listener of listenersRef.current) {
listener()
@@ -215,19 +248,18 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
})
}, [])
- const away =
- owner !== null && isUnderway(dictation) && !screenFor(screensRef.current, owner, false)?.shown()
const title = owner?.title ?? ''
const clearance = Math.max(0, ...clearances.values())
const pill = useMemo(
- () => ({ away, title, open, toast, clearToast, clearance }),
- [away, clearToast, clearance, open, title, toast]
+ () => ({ away, title, open, toast, clearToast, clearance, liveText: speech.liveText }),
+ [speech.liveText, away, clearToast, clearance, open, title, toast]
)
const value = useMemo(
() => ({
owner,
start,
stop,
+ announce: speak,
attach,
refresh,
subscribe,
@@ -235,7 +267,18 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
getIdleDictation,
getUnderway
}),
- [attach, getDictation, getIdleDictation, getUnderway, owner, refresh, start, stop, subscribe]
+ [
+ attach,
+ getDictation,
+ getIdleDictation,
+ getUnderway,
+ owner,
+ refresh,
+ speak,
+ start,
+ stop,
+ subscribe
+ ]
)
return (
diff --git a/mobile/src/dictation/use-mobile-dictation-announcements.ts b/mobile/src/dictation/use-mobile-dictation-announcements.ts
new file mode 100644
index 00000000000..c06933349b9
--- /dev/null
+++ b/mobile/src/dictation/use-mobile-dictation-announcements.ts
@@ -0,0 +1,113 @@
+import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
+import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
+import { useAccessibilityAnnouncer } from '../platform/accessibility-announcer'
+import { dictationAnnounceCopy } from './mobile-dictation-announce-copy'
+import { dictationFailureReason } from './mobile-dictation-controls-copy'
+
+// Finishing is said only when it is slow.
+const FINISHING_ANNOUNCE_DELAY_MS = 1000
+
+// What the shell could not speak, for the page's live region; the id changes on every message so a repeat is read again.
+export type MobileDictationLiveText = { readonly text: string; readonly id: number }
+
+// Speak to the screen reader through the shell where it can, else into the live region. say keeps its identity.
+export function useMobileDictationSpeech(): {
+ say: (message: string) => void
+ liveText: MobileDictationLiveText
+} {
+ const announce = useAccessibilityAnnouncer()
+ const announceRef = useRef(announce)
+ useLayoutEffect(() => {
+ announceRef.current = announce
+ })
+ const [liveText, setLiveText] = useState({ text: '', id: 0 })
+ const say = useCallback((message: string) => {
+ void announceRef.current(message).then((spoken) => {
+ if (!spoken) {
+ setLiveText((current) => ({ text: message, id: current.id + 1 }))
+ }
+ })
+ }, [])
+ return { say, liveText }
+}
+
+// The dictation's own spoken moments: start, leaving its screen, a move to the computer, a slow finish, a cancel, a kept failure, the host's toasts.
+// Outcomes (inserted, failed) are said by whoever knows them: the screen that inserted, or the host's failure report.
+export function useMobileDictationAnnouncements(
+ dictation: Pick,
+ {
+ computerName,
+ away,
+ toast
+ }: { computerName: string | null; away: boolean; toast: string | null },
+ say: (message: string) => void
+): { noteCancel: () => void } {
+ const { status, engine, failure } = dictation
+ const previousStatus = useRef(status)
+ const cancelled = useRef(false)
+ const listeningSaid = useRef(false)
+ const movedAtSaid = useRef(null)
+ const statusRef = useRef(status)
+
+ useEffect(() => {
+ const previous = previousStatus.current
+ previousStatus.current = status
+ statusRef.current = status
+ if (status === 'starting') {
+ cancelled.current = false
+ listeningSaid.current = false
+ }
+ // Only a ✕ (including Discard on a kept failure); a refused start's reason is said where it is reported.
+ if (status === 'idle' && previous !== 'idle' && cancelled.current) {
+ say(dictationAnnounceCopy.cancelled())
+ }
+ if (status === 'idle') {
+ cancelled.current = false
+ }
+ }, [say, status])
+
+ // Leaving the screen keeps it going; say so once, then the next screen reads as usual.
+ useEffect(() => {
+ if (away && statusRef.current === 'recording') {
+ say(dictationAnnounceCopy.stillListening())
+ }
+ }, [away, say])
+ useEffect(() => {
+ if (toast) {
+ say(toast)
+ }
+ }, [say, toast])
+
+ useEffect(() => {
+ if (status === 'recording' && engine && !listeningSaid.current) {
+ listeningSaid.current = true
+ say(dictationAnnounceCopy.listening(engine.kind, computerName))
+ }
+ if (engine?.movedAt != null && engine.movedAt !== movedAtSaid.current) {
+ movedAtSaid.current = engine.movedAt
+ say(dictationAnnounceCopy.movedToComputer(computerName))
+ }
+ }, [computerName, engine, say, status])
+
+ useEffect(() => {
+ if (status !== 'processing') {
+ return
+ }
+ const timer = setTimeout(
+ () => say(dictationAnnounceCopy.finishing()),
+ FINISHING_ANNOUNCE_DELAY_MS
+ )
+ return () => clearTimeout(timer)
+ }, [say, status])
+
+ useEffect(() => {
+ if (failure) {
+ say(dictationAnnounceCopy.stoppedKept(dictationFailureReason(failure)))
+ }
+ }, [failure, say])
+
+ const noteCancel = useCallback(() => {
+ cancelled.current = true
+ }, [])
+ return { noteCancel }
+}
diff --git a/mobile/src/dictation/use-mobile-dictation-live.ts b/mobile/src/dictation/use-mobile-dictation-live.ts
index 37528abb0cb..13aa5f5081a 100644
--- a/mobile/src/dictation/use-mobile-dictation-live.ts
+++ b/mobile/src/dictation/use-mobile-dictation-live.ts
@@ -17,10 +17,7 @@ export function useMobileDictationLive(
cancelLatest.current = cancel
}, [cancel])
const stableCancel = useCallback(() => cancelLatest.current(), [])
- const hostProfile = useHostProfile(hostId)
- const hostTitle = useHostDisplay(hostProfile).title
- // A workspace routed to another server transcribes there, and the phone lacks that server's name.
- const computerName = useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
+ const computerName = useDictationComputerName(hostId)
const { meter, engine, failure, retry } = dictation
const failureReason = failure === null ? null : dictationFailureReason(failure)
return useMemo(
@@ -35,3 +32,10 @@ export function useMobileDictationLive(
[meter, engine, computerName, stableCancel, failureReason, retry]
)
}
+
+// The computer that transcribes a session's dictation; null when another server runs the workspace and the phone lacks its name.
+export function useDictationComputerName(hostId: string): string | null {
+ const hostProfile = useHostProfile(hostId)
+ const hostTitle = useHostDisplay(hostProfile).title
+ return useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
+}
diff --git a/mobile/src/i18n/locales/de.json b/mobile/src/i18n/locales/de.json
index 93c90b1859d..6ac9c9e827d 100644
--- a/mobile/src/i18n/locales/de.json
+++ b/mobile/src/i18n/locales/de.json
@@ -1339,7 +1339,17 @@
},
"announce": {
"typedInto": "Eingetippt in {{target}}",
- "insertedIn": "Eingefügt in {{target}}"
+ "insertedIn": "Eingefügt in {{target}}",
+ "listeningOnPhone": "Hört zu. Transkription auf diesem Telefon.",
+ "listeningOn": "Hört zu. Transkription auf {{computer}}.",
+ "listeningOnComputer": "Hört zu. Transkription auf dem Computer.",
+ "finishing": "Wird abgeschlossen.",
+ "inserted": "Eingefügt.",
+ "cancelled": "Diktat abgebrochen.",
+ "movedTo": "Dieses Telefon hat die Transkription beendet. Weiter auf {{computer}}.",
+ "movedToComputer": "Dieses Telefon hat die Transkription beendet. Weiter auf dem Computer.",
+ "stoppedKept": "Diktat gestoppt. {{reason}}. Deine Aufnahme bleibt erhalten.",
+ "stillListening": "Hört weiter zu."
},
"pill": {
"open": "Öffnen",
diff --git a/mobile/src/i18n/locales/en.json b/mobile/src/i18n/locales/en.json
index a17ea4122a0..2d9ce8ba6cf 100644
--- a/mobile/src/i18n/locales/en.json
+++ b/mobile/src/i18n/locales/en.json
@@ -554,7 +554,17 @@
},
"announce": {
"typedInto": "Typed into {{target}}",
- "insertedIn": "Inserted in {{target}}"
+ "insertedIn": "Inserted in {{target}}",
+ "listeningOnPhone": "Listening. Transcribing on this phone.",
+ "listeningOn": "Listening. Transcribing on {{computer}}.",
+ "listeningOnComputer": "Listening. Transcribing on the computer.",
+ "finishing": "Finishing.",
+ "inserted": "Inserted.",
+ "cancelled": "Dictation cancelled.",
+ "movedTo": "This phone stopped transcribing. Continuing on {{computer}}.",
+ "movedToComputer": "This phone stopped transcribing. Continuing on the computer.",
+ "stoppedKept": "Dictation stopped. {{reason}}. Your recording is kept.",
+ "stillListening": "Still listening."
},
"pill": {
"open": "Open",
diff --git a/mobile/src/i18n/locales/es.json b/mobile/src/i18n/locales/es.json
index 23bea482416..06a433dc383 100644
--- a/mobile/src/i18n/locales/es.json
+++ b/mobile/src/i18n/locales/es.json
@@ -1380,7 +1380,17 @@
},
"announce": {
"typedInto": "Escrito en {{target}}",
- "insertedIn": "Insertado en {{target}}"
+ "insertedIn": "Insertado en {{target}}",
+ "listeningOnPhone": "Escuchando. Transcribiendo en este teléfono.",
+ "listeningOn": "Escuchando. Transcribiendo en {{computer}}.",
+ "listeningOnComputer": "Escuchando. Transcribiendo en el ordenador.",
+ "finishing": "Terminando.",
+ "inserted": "Insertado.",
+ "cancelled": "Dictado cancelado.",
+ "movedTo": "Este teléfono dejó de transcribir. Continúa en {{computer}}.",
+ "movedToComputer": "Este teléfono dejó de transcribir. Continúa en el ordenador.",
+ "stoppedKept": "Dictado detenido. {{reason}}. Tu grabación se conserva.",
+ "stillListening": "Sigue escuchando."
},
"pill": {
"open": "Abrir",
diff --git a/mobile/src/i18n/locales/fr.json b/mobile/src/i18n/locales/fr.json
index 030a9037c45..af77cbdb538 100644
--- a/mobile/src/i18n/locales/fr.json
+++ b/mobile/src/i18n/locales/fr.json
@@ -1380,7 +1380,17 @@
},
"announce": {
"typedInto": "Saisi dans {{target}}",
- "insertedIn": "Inséré dans {{target}}"
+ "insertedIn": "Inséré dans {{target}}",
+ "listeningOnPhone": "Écoute. Transcription sur ce téléphone.",
+ "listeningOn": "Écoute. Transcription sur {{computer}}.",
+ "listeningOnComputer": "Écoute. Transcription sur l’ordinateur.",
+ "finishing": "Finalisation.",
+ "inserted": "Inséré.",
+ "cancelled": "Dictée annulée.",
+ "movedTo": "Ce téléphone a arrêté de transcrire. Suite sur {{computer}}.",
+ "movedToComputer": "Ce téléphone a arrêté de transcrire. Suite sur l’ordinateur.",
+ "stoppedKept": "Dictée arrêtée. {{reason}}. Votre enregistrement est conservé.",
+ "stillListening": "Toujours à l’écoute."
},
"pill": {
"open": "Ouvrir",
diff --git a/mobile/src/i18n/locales/ja.json b/mobile/src/i18n/locales/ja.json
index c6410cd1e67..9b2aa411b82 100644
--- a/mobile/src/i18n/locales/ja.json
+++ b/mobile/src/i18n/locales/ja.json
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "{{target}} に入力しました",
- "insertedIn": "{{target}} に挿入しました"
+ "insertedIn": "{{target}} に挿入しました",
+ "listeningOnPhone": "聞き取り中。このスマートフォンで文字起こししています。",
+ "listeningOn": "聞き取り中。{{computer}} で文字起こししています。",
+ "listeningOnComputer": "聞き取り中。コンピュータで文字起こししています。",
+ "finishing": "仕上げ中。",
+ "inserted": "挿入しました。",
+ "cancelled": "音声入力をキャンセルしました。",
+ "movedTo": "このスマートフォンでの文字起こしが止まりました。{{computer}} で続けます。",
+ "movedToComputer": "このスマートフォンでの文字起こしが止まりました。コンピュータで続けます。",
+ "stoppedKept": "音声入力が停止しました。{{reason}}。録音は保持されています。",
+ "stillListening": "聞き取りを続けています。"
},
"pill": {
"open": "開く",
diff --git a/mobile/src/i18n/locales/ko.json b/mobile/src/i18n/locales/ko.json
index 2b166821479..c6246089d50 100644
--- a/mobile/src/i18n/locales/ko.json
+++ b/mobile/src/i18n/locales/ko.json
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "{{target}}에 입력됨",
- "insertedIn": "{{target}}에 삽입됨"
+ "insertedIn": "{{target}}에 삽입됨",
+ "listeningOnPhone": "듣는 중. 이 휴대폰에서 받아쓰는 중입니다.",
+ "listeningOn": "듣는 중. {{computer}}에서 받아쓰는 중입니다.",
+ "listeningOnComputer": "듣는 중. 컴퓨터에서 받아쓰는 중입니다.",
+ "finishing": "마무리 중.",
+ "inserted": "삽입됨.",
+ "cancelled": "받아쓰기가 취소되었습니다.",
+ "movedTo": "이 휴대폰의 받아쓰기가 멈췄습니다. {{computer}}에서 계속합니다.",
+ "movedToComputer": "이 휴대폰의 받아쓰기가 멈췄습니다. 컴퓨터에서 계속합니다.",
+ "stoppedKept": "받아쓰기가 중지되었습니다. {{reason}}. 녹음은 보관됩니다.",
+ "stillListening": "계속 듣는 중입니다."
},
"pill": {
"open": "열기",
diff --git a/mobile/src/i18n/locales/pt.json b/mobile/src/i18n/locales/pt.json
index ea08a696d58..e563922fc2e 100644
--- a/mobile/src/i18n/locales/pt.json
+++ b/mobile/src/i18n/locales/pt.json
@@ -1380,7 +1380,17 @@
},
"announce": {
"typedInto": "Digitado em {{target}}",
- "insertedIn": "Inserido em {{target}}"
+ "insertedIn": "Inserido em {{target}}",
+ "listeningOnPhone": "Ouvindo. Transcrevendo neste telefone.",
+ "listeningOn": "Ouvindo. Transcrevendo em {{computer}}.",
+ "listeningOnComputer": "Ouvindo. Transcrevendo no computador.",
+ "finishing": "Finalizando.",
+ "inserted": "Inserido.",
+ "cancelled": "Ditado cancelado.",
+ "movedTo": "Este telefone parou de transcrever. Continuando em {{computer}}.",
+ "movedToComputer": "Este telefone parou de transcrever. Continuando no computador.",
+ "stoppedKept": "Ditado interrompido. {{reason}}. Sua gravação foi mantida.",
+ "stillListening": "Ainda ouvindo."
},
"pill": {
"open": "Abrir",
diff --git a/mobile/src/i18n/locales/ru.json b/mobile/src/i18n/locales/ru.json
index b3753c99614..06a53c287b8 100644
--- a/mobile/src/i18n/locales/ru.json
+++ b/mobile/src/i18n/locales/ru.json
@@ -1421,7 +1421,17 @@
},
"announce": {
"typedInto": "Введено в {{target}}",
- "insertedIn": "Вставлено в {{target}}"
+ "insertedIn": "Вставлено в {{target}}",
+ "listeningOnPhone": "Слушаю. Распознавание на этом телефоне.",
+ "listeningOn": "Слушаю. Распознавание на {{computer}}.",
+ "listeningOnComputer": "Слушаю. Распознавание на компьютере.",
+ "finishing": "Завершение.",
+ "inserted": "Вставлено.",
+ "cancelled": "Диктовка отменена.",
+ "movedTo": "Этот телефон перестал распознавать. Продолжение на {{computer}}.",
+ "movedToComputer": "Этот телефон перестал распознавать. Продолжение на компьютере.",
+ "stoppedKept": "Диктовка остановлена. {{reason}}. Запись сохранена.",
+ "stillListening": "Продолжаю слушать."
},
"pill": {
"open": "Открыть",
diff --git a/mobile/src/i18n/locales/tr.json b/mobile/src/i18n/locales/tr.json
index 69e6eb64810..03704efc034 100644
--- a/mobile/src/i18n/locales/tr.json
+++ b/mobile/src/i18n/locales/tr.json
@@ -1339,7 +1339,17 @@
},
"announce": {
"typedInto": "{{target}} içine yazıldı",
- "insertedIn": "{{target}} içine eklendi"
+ "insertedIn": "{{target}} içine eklendi",
+ "listeningOnPhone": "Dinleniyor. Bu telefonda yazıya dökülüyor.",
+ "listeningOn": "Dinleniyor. {{computer}} üzerinde yazıya dökülüyor.",
+ "listeningOnComputer": "Dinleniyor. Bilgisayarda yazıya dökülüyor.",
+ "finishing": "Tamamlanıyor.",
+ "inserted": "Eklendi.",
+ "cancelled": "Dikte iptal edildi.",
+ "movedTo": "Bu telefon yazıya dökmeyi bıraktı. {{computer}} üzerinde devam ediliyor.",
+ "movedToComputer": "Bu telefon yazıya dökmeyi bıraktı. Bilgisayarda devam ediliyor.",
+ "stoppedKept": "Dikte durdu. {{reason}}. Kaydınız saklandı.",
+ "stillListening": "Dinlemeye devam ediliyor."
},
"pill": {
"open": "Aç",
diff --git a/mobile/src/i18n/locales/zh-Hant.json b/mobile/src/i18n/locales/zh-Hant.json
index 7fad8bd6227..a3c8197a0f6 100644
--- a/mobile/src/i18n/locales/zh-Hant.json
+++ b/mobile/src/i18n/locales/zh-Hant.json
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "已輸入到 {{target}}",
- "insertedIn": "已插入到 {{target}}"
+ "insertedIn": "已插入到 {{target}}",
+ "listeningOnPhone": "正在聽。正在此手機上轉寫。",
+ "listeningOn": "正在聽。正在 {{computer}} 上轉寫。",
+ "listeningOnComputer": "正在聽。正在電腦上轉寫。",
+ "finishing": "正在完成。",
+ "inserted": "已插入。",
+ "cancelled": "已取消聽寫。",
+ "movedTo": "此手機已停止轉寫。在 {{computer}} 上繼續。",
+ "movedToComputer": "此手機已停止轉寫。在電腦上繼續。",
+ "stoppedKept": "聽寫已停止。{{reason}}。錄音已保留。",
+ "stillListening": "仍在聽。"
},
"pill": {
"open": "開啟",
diff --git a/mobile/src/i18n/locales/zh.json b/mobile/src/i18n/locales/zh.json
index 1d9441b494c..459357aa1e1 100644
--- a/mobile/src/i18n/locales/zh.json
+++ b/mobile/src/i18n/locales/zh.json
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "已输入到 {{target}}",
- "insertedIn": "已插入到 {{target}}"
+ "insertedIn": "已插入到 {{target}}",
+ "listeningOnPhone": "正在听。正在此手机上转写。",
+ "listeningOn": "正在听。正在 {{computer}} 上转写。",
+ "listeningOnComputer": "正在听。正在电脑上转写。",
+ "finishing": "正在完成。",
+ "inserted": "已插入。",
+ "cancelled": "已取消听写。",
+ "movedTo": "此手机已停止转写。在 {{computer}} 上继续。",
+ "movedToComputer": "此手机已停止转写。在电脑上继续。",
+ "stoppedKept": "听写已停止。{{reason}}。录音已保留。",
+ "stillListening": "仍在听。"
},
"pill": {
"open": "打开",
diff --git a/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts b/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts
index ef1f628473e..7dfff72f157 100644
--- a/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts
+++ b/mobile/src/mobile-web-shell/bridge/bridge-native-verbs.ts
@@ -41,7 +41,8 @@ export const BRIDGE_NATIVE_VERB_NAMES = [
'native.pdf.view',
'native.image.act',
BRIDGE_SPEECH_VERB,
- BRIDGE_STORE_VERB
+ BRIDGE_STORE_VERB,
+ 'native.a11y.announce'
] as const
export type BridgeNativeVerb = (typeof BRIDGE_NATIVE_VERB_NAMES)[number]
@@ -93,6 +94,11 @@ export type BridgePdfOpenParams = Omit<
'action'
>
+// The page has no announce call of its own (RNW's is a no-op); the shell speaks for it. Long text is a page bug, so it is refused.
+export const a11yAnnounceParamsSchema = z.strictObject({ message: z.string().min(1).max(500) })
+// announced:false when the shell leaves it to the page's live region (Android, where TalkBack reads it).
+export const a11yAnnounceResultSchema = z.strictObject({ announced: z.boolean() })
+
export const BRIDGE_NATIVE_VERBS: Readonly> = {
'native.clipboard.write': {
params: clipboardWriteParamsSchema,
@@ -150,6 +156,10 @@ export const BRIDGE_NATIVE_VERBS: Readonly Promise
speechFinish: () => Promise
speechCancel: () => Promise
+ // Older shells lack it; the page then keeps its live region, which VoiceOver may not read.
+ canAnnounce: boolean
+ // True when the shell spoke it.
+ announce: (message: string) => Promise
}
// Expose stable reason codes, including local ungranted and frame refusals. Unknown newer-shell codes floor to unreported so caller switches remain exhaustive.
@@ -204,7 +209,10 @@ export function useNativeVerbs(): NativeVerbs {
speechFinish: () => call(BRIDGE_SPEECH_VERB, { action: 'finish' }, speechFinishResultSchema),
speechCancel: async () => {
await call(BRIDGE_SPEECH_VERB, { action: 'cancel' }, speechCancelResultSchema)
- }
+ },
+ canAnnounce: has('native.a11y.announce'),
+ announce: async (message) =>
+ (await call('native.a11y.announce', { message }, a11yAnnounceResultSchema)).announced
}
}, [client])
}
diff --git a/mobile/src/platform/accessibility-announcer.ts b/mobile/src/platform/accessibility-announcer.ts
new file mode 100644
index 00000000000..3f98cfba33b
--- /dev/null
+++ b/mobile/src/platform/accessibility-announcer.ts
@@ -0,0 +1,10 @@
+import { useCallback } from 'react'
+import { AccessibilityInfo } from 'react-native'
+
+// Speak a message to the screen reader; false when the caller should put it in a live region instead.
+export function useAccessibilityAnnouncer(): (message: string) => Promise {
+ return useCallback(async (message: string) => {
+ AccessibilityInfo.announceForAccessibility(message)
+ return true
+ }, [])
+}
diff --git a/mobile/src/platform/accessibility-announcer.web.ts b/mobile/src/platform/accessibility-announcer.web.ts
new file mode 100644
index 00000000000..647555ab841
--- /dev/null
+++ b/mobile/src/platform/accessibility-announcer.web.ts
@@ -0,0 +1,16 @@
+import { useCallback } from 'react'
+import { useNativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
+
+// RNW's announceForAccessibility does nothing, so the shell announces; older shells and Android leave it to the page's live region.
+export function useAccessibilityAnnouncer(): (message: string) => Promise {
+ const verbs = useNativeVerbs()
+ return useCallback(
+ async (message: string) => {
+ if (!verbs.canAnnounce) {
+ return false
+ }
+ return verbs.announce(message).catch(() => false)
+ },
+ [verbs]
+ )
+}
diff --git a/mobile/src/platform/native-accessibility-announce.ts b/mobile/src/platform/native-accessibility-announce.ts
new file mode 100644
index 00000000000..b171ff3948e
--- /dev/null
+++ b/mobile/src/platform/native-accessibility-announce.ts
@@ -0,0 +1,12 @@
+import { AccessibilityInfo, Platform } from 'react-native'
+import { a11yAnnounceParamsSchema } from '../mobile-web-shell/bridge/bridge-native-verbs'
+
+// VoiceOver ignores the page's live regions inside the shell's WebView, so the shell announces for it; TalkBack reads them, so Android leaves it to the page.
+export function serveNativeAnnounceVerb(params: unknown): { announced: boolean } {
+ const { message } = a11yAnnounceParamsSchema.parse(params)
+ if (Platform.OS !== 'ios') {
+ return { announced: false }
+ }
+ AccessibilityInfo.announceForAccessibilityWithOptions(message, { queue: true })
+ return { announced: true }
+}
diff --git a/mobile/src/platform/use-native-device-verbs.test.tsx b/mobile/src/platform/use-native-device-verbs.test.tsx
index da46626ef78..757d8cb3805 100644
--- a/mobile/src/platform/use-native-device-verbs.test.tsx
+++ b/mobile/src/platform/use-native-device-verbs.test.tsx
@@ -17,6 +17,11 @@ const device = vi.hoisted(() => ({
wakeTags: new Array()
}))
+// The shell's announce verb speaks through React Native's AccessibilityInfo.
+vi.mock('react-native', () => ({
+ AccessibilityInfo: { announceForAccessibilityWithOptions: () => {} },
+ Platform: { OS: 'ios' }
+}))
vi.mock('expo-clipboard', () => ({
setStringAsync: () => Promise.resolve(true),
getStringAsync: device.clipboard.getStringAsync,
diff --git a/mobile/src/platform/use-native-device-verbs.ts b/mobile/src/platform/use-native-device-verbs.ts
index 3e7579ecd34..647133dea35 100644
--- a/mobile/src/platform/use-native-device-verbs.ts
+++ b/mobile/src/platform/use-native-device-verbs.ts
@@ -7,6 +7,7 @@ import { NATIVE_AUDIO_DEVICE_SAMPLE_RATE, nativeAudioDeviceEngine } from './nati
import { createNativeSpeechServer } from './native-speech'
import { nativeSpeechDeviceEngine } from './native-speech-device'
import { serveNativeClipboardVerb } from './native-clipboard'
+import { serveNativeAnnounceVerb } from './native-accessibility-announce'
import { createNativeMediaVerbServer } from './native-media'
import { discardStagedMedia, nativeMediaDeviceDeps } from './native-media-device'
@@ -37,6 +38,9 @@ export function useNativeDeviceVerbs(
if (verb === 'native.speech.act') {
return speech.serve(params)
}
+ if (verb === 'native.a11y.announce') {
+ return Promise.resolve(serveNativeAnnounceVerb(params))
+ }
if (verb === 'native.audio.start') {
return speech.mayCapture().then((may) =>
may
diff --git a/mobile/src/session/MobileSessionContentRow.tsx b/mobile/src/session/MobileSessionContentRow.tsx
index 9a7cea2d3f1..474f974e441 100644
--- a/mobile/src/session/MobileSessionContentRow.tsx
+++ b/mobile/src/session/MobileSessionContentRow.tsx
@@ -9,7 +9,6 @@ import { styles } from './mobile-session-styles'
import type { MobileSessionController } from './use-mobile-session-controller'
import { MobileSessionActiveContent } from './MobileSessionActiveContent'
import { MobileSessionCommandDock } from './MobileSessionCommandDock'
-import { MobileDictationStatusAnnouncement } from '../dictation/MobileDictationStatusAnnouncement'
import { MobileDictationLiveContext } from '../dictation/mobile-dictation-live-context'
import { useMobileDictationLive } from '../dictation/use-mobile-dictation-live'
@@ -56,7 +55,6 @@ export function MobileSessionContentRow({ controller }: { controller: MobileSess
{/* Why: translate instead of resize so keyboard toggles don't trigger a server-side PTY viewport change. */}
-
{canDockPanel && activePanel !== null && (
{
vi.mock('react-native', () => ({
AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) },
+ AccessibilityInfo: { announceForAccessibility: () => {} },
Platform: { OS: 'ios' }
}))
+// The computer's name only words announcements; the stored host profile pulls in secure storage.
+vi.mock('../transport/hosts/use-host-profile', () => ({ useHostProfile: () => null }))
vi.mock('@orca/expo-two-way-audio', () => ({
addExpoTwoWayAudioEventListener: (name: string, listener: (event: { data: string }) => void) => {
seen.audioListeners.set(name, listener)
diff --git a/mobile/src/session/use-mobile-session-dictation.ts b/mobile/src/session/use-mobile-session-dictation.ts
index 9212ed7d75d..32159402a3c 100644
--- a/mobile/src/session/use-mobile-session-dictation.ts
+++ b/mobile/src/session/use-mobile-session-dictation.ts
@@ -22,7 +22,9 @@ import {
import { useMobileDictationLeaveHold } from '../dictation/mobile-dictation-leave-hold'
import { useMobileDictationHost } from '../dictation/mobile-dictation-host'
import { mobileDictationShownOnScreen } from '../dictation/mobile-dictation-shown'
+import { useDictationComputerName } from '../dictation/use-mobile-dictation-live'
import { nativeChatDictationCopy } from '../dictation/mobile-session-native-chat-dictation-copy'
+import { dictationAnnounceCopy } from '../dictation/mobile-dictation-announce-copy'
import {
useWorkspaceExecutionHost,
workspaceRouteHref
@@ -54,6 +56,7 @@ export function useMobileSessionDictation(
} = scope
const { showNativeChat, showNativeChatRef } = nativeChatController
const host = useMobileDictationHost()
+ const computerName = useDictationComputerName(hostId)
const { start: startHost } = host
const executionHost = useWorkspaceExecutionHost()
const sessionKey = JSON.stringify([hostId, executionHost ?? null, worktreeId])
@@ -93,12 +96,17 @@ export function useMobileSessionDictation(
)
const clipboard = useClipboardWriter()
+ // The screen's toast is not read on the page, so where the words went is also spoken.
+ const tell = (message: string) => {
+ showToast(message)
+ host.announce(message)
+ }
// A transcript its target refused must not vanish: keep it on the clipboard and say so.
const keepUndeliveredDictation = (text: string) => {
triggerError()
void clipboard.writeText(text).then(
- () => showToast(nativeChatDictationCopy.insertFailedCopied()),
- () => showToast(nativeChatDictationCopy.insertFailed())
+ () => tell(nativeChatDictationCopy.insertFailedCopied()),
+ () => tell(nativeChatDictationCopy.insertFailed())
)
}
@@ -106,7 +114,9 @@ export function useMobileSessionDictation(
// Chat dictation is inserted locally; a terminal's words are typed into it live, while it is the active one.
const insertHere = (text: string): boolean => {
if (destination.landsInChat(showNativeChatRef.current)) {
- if (!destination.insertInChat(text)) {
+ if (destination.insertInChat(text)) {
+ host.announce(dictationAnnounceCopy.inserted())
+ } else {
keepUndeliveredDictation(text)
}
return true
@@ -124,7 +134,7 @@ export function useMobileSessionDictation(
(await sendLiveTerminalInput(insertHandle, text))
// Live input lands in a terminal the keyboard may cover, so say where the words went.
if (sent) {
- showToast(nativeChatDictationCopy.inserted())
+ tell(nativeChatDictationCopy.inserted())
} else {
keepUndeliveredDictation(text)
}
@@ -237,9 +247,11 @@ export function useMobileSessionDictation(
dictationMode: scope.dictationMode,
removeFillerWords,
dictationTerms,
- worktreeId
+ worktreeId,
+ computerName
})
}, [
+ computerName,
activeHandle,
activeSessionTab,
busyElsewhere,
diff --git a/mobile/web-entry/web-overrides.json b/mobile/web-entry/web-overrides.json
index 453b856218e..954c12892fb 100644
--- a/mobile/web-entry/web-overrides.json
+++ b/mobile/web-entry/web-overrides.json
@@ -244,6 +244,10 @@
{
"file": "src/platform/page-box.web.ts",
"reason": "RNW's measure() is asynchronous and drops its callback for a detached node, so a held mic released over the dictation cancel button would never stop. The page reads the DOM box synchronously; native keeps measure()."
+ },
+ {
+ "file": "src/platform/accessibility-announcer.web.ts",
+ "reason": "React Native Web's announceForAccessibility is a no-op and VoiceOver in the shell's WebView doesn't read the page's live regions, so the page asks the shell to announce through native.a11y.announce; without it, the caller keeps its live region."
}
]
}