feat(mobile): read dictation progress to VoiceOver and TalkBack (#40504)

* feat(mobile): announce dictation to VoiceOver through the shell

* feat(mobile): speak dictation from the host, so leaving the session and the pill's toasts are announced too

* fix(mobile): say dictation outcomes where they are known, and re-read a repeated live message

* fix(mobile): leave a setup refusal to its setup sheet instead of reading the raw code

* fix(mobile): say cancelled when a kept recording is discarded
This commit is contained in:
Kelvin Amoaba
2026-11-09 22:52:59 +00:00
committed by GitHub
parent 3acd6ede7c
commit a5dbd1cb2f
30 changed files with 443 additions and 46 deletions
@@ -102,13 +102,14 @@ export const MOBILE_WEB_PAGE_ROUTES = [
'native.audio.read',
'native.audio.stop'
],
// Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act; dictation uses the computer without native.speech.act.
// Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act; dictation uses the computer without native.speech.act; without native.a11y.announce only the page's live region speaks.
optionalGrants: [
'externalNavigation',
'native.media.name',
'native.pdf.view',
'native.image.act',
'native.speech.act',
'native.a11y.announce',
'terminalKeyRail',
DEVICE_STORE_GRANT
]
+4 -6
View File
@@ -24,6 +24,7 @@ import {
} from './mobile-dictation-controls-copy'
import { dictationAwayCopy } from './mobile-dictation-away-copy'
import { composerInputStyles } from '../components/composer-input/composer-input-styles'
import { MobileDictationStatusAnnouncement } from './MobileDictationStatusAnnouncement'
import {
useMobileDictationHost,
useMobileDictationHostLive,
@@ -163,13 +164,9 @@ function MobileDictationToast({
const timer = setTimeout(onDone, toast.open ? TOAST_WITH_OPEN_MS : TOAST_MS)
return () => clearTimeout(timer)
}, [onDone, toast])
// The live region stays mounted so a screen reader hears the message appear.
// The host's announcer reads the message (shell or live region), so the toast itself stays quiet.
return (
<View
style={[styles.toastRow, { bottom }]}
pointerEvents="box-none"
accessibilityLiveRegion="polite"
>
<View style={[styles.toastRow, { bottom }]} pointerEvents="box-none">
{toast ? (
<View style={styles.toast}>
<Text style={styles.toastText} numberOfLines={2}>
@@ -212,6 +209,7 @@ export function MobileDictationPillLayer(): React.JSX.Element | null {
<MobileDictationPill title={pill.title} onOpen={pill.open} onStop={stop} />
</View>
) : null}
<MobileDictationStatusAnnouncement liveText={pill.liveText} />
<MobileDictationToast
toast={pill.toast}
onDone={pill.clearToast}
@@ -1,7 +1,7 @@
import { StyleSheet, Text, View } from 'react-native'
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
import { translate } from '../i18n/mobile-i18n'
import { useMobileTranslation } from '../i18n/use-mobile-translation'
import type { MobileDictationLiveText } from './use-mobile-dictation-announcements'
export type MobileDictationStatusFlags = Pick<
UseMobileDictationResult,
@@ -18,16 +18,16 @@ export function mobileDictationStatusTitle(dictation: MobileDictationStatusFlags
: null
}
// Mic labels change silently; one live region tells screen readers for every mic, like desktop's indicator.
// The live region for announcements the shell could not speak itself (TalkBack reads it; older shells fall back to it).
// A new Text per message, so the same words twice are read twice.
export function MobileDictationStatusAnnouncement({
dictation
liveText
}: {
dictation: MobileDictationStatusFlags
liveText: MobileDictationLiveText
}): React.JSX.Element {
useMobileTranslation()
return (
<View style={styles.hidden} pointerEvents="none" accessibilityLiveRegion="polite">
<Text>{mobileDictationStatusTitle(dictation) ?? ''}</Text>
<Text key={liveText.id}>{liveText.text}</Text>
</View>
)
}
@@ -0,0 +1,39 @@
import { translate } from '../i18n/mobile-i18n'
// What a screen reader hears as a dictation moves along; nothing while you talk, so it never talks over you.
export const dictationAnnounceCopy = {
listening: (engine: 'phone' | 'computer', computer: string | null) =>
engine === 'phone'
? translate('dictation.announce.listeningOnPhone', 'Listening. Transcribing on this phone.')
: computer
? translate('dictation.announce.listeningOn', 'Listening. Transcribing on {{computer}}.', {
computer
})
: translate(
'dictation.announce.listeningOnComputer',
'Listening. Transcribing on the computer.'
),
stillListening: () => translate('dictation.announce.stillListening', 'Still listening.'),
finishing: () => translate('dictation.announce.finishing', 'Finishing.'),
inserted: () => translate('dictation.announce.inserted', 'Inserted.'),
cancelled: () => translate('dictation.announce.cancelled', 'Dictation cancelled.'),
movedToComputer: (computer: string | null) =>
computer
? translate(
'dictation.announce.movedTo',
'This phone stopped transcribing. Continuing on {{computer}}.',
{ computer }
)
: translate(
'dictation.announce.movedToComputer',
'This phone stopped transcribing. Continuing on the computer.'
),
stoppedKept: (reason: string) =>
translate(
'dictation.announce.stoppedKept',
'Dictation stopped. {{reason}}. Your recording is kept.',
{
reason
}
)
}
@@ -2,6 +2,7 @@ import { createContext, useContext, useEffect, useState, useSyncExternalStore }
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
import type { MobileDictationAwayTarget } from './mobile-dictation-away-insert'
import type { MobileDictationLiveText } from './use-mobile-dictation-announcements'
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
export type MobileDictationHostToast = { message: string; open?: () => void }
@@ -25,6 +26,8 @@ export type MobileDictationOwner = {
readonly dictationTerms: readonly DictationTerm[] | undefined
// The computer spells its own words with this worktree's repo terms.
readonly worktreeId: string
// The computer that would transcribe it, for what a screen reader hears; null when the phone lacks its name.
readonly computerName: string | null
}
// A mounted session screen, as the dictation host sees it.
@@ -48,6 +51,8 @@ export type MobileDictationHost = {
readonly start: (owner: MobileDictationOwner) => Promise<void>
// Stop with the selection tick, as every stop button does.
readonly stop: () => void
// Say something to the screen reader; a screen's own toast is not read on the page.
readonly announce: (message: string) => void
readonly attach: (screen: MobileDictationScreen) => () => void
// A screen's shown() answer changed; the pill reads it on render.
readonly refresh: () => void
@@ -69,6 +74,8 @@ export type MobileDictationPill = {
readonly clearToast: () => void
// The tallest bottom bar a screen reported, so the pill floats above it.
readonly clearance: number
// Announcements the shell could not speak, for the page's live region (TalkBack, older shells).
readonly liveText: MobileDictationLiveText
}
export const MobileDictationHostContext = createContext<MobileDictationHost | null>(null)
+52 -9
View File
@@ -4,12 +4,16 @@ import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session
import { useRouteHandoff } from '../navigation/route-handoff'
import { triggerError } from '../platform/haptics'
import { useClipboardWriter } from '../platform/clipboard'
import { dictationFailureMessage } from './mobile-dictation-setup'
import { dictationFailureMessage, isDictationSetupRequiredError } from './mobile-dictation-setup'
import { useMobileDictationHaptics } from './use-mobile-dictation-haptics'
import { useMobileDictationSilenceStop } from './use-mobile-dictation-silence-stop'
import { insertMobileDictationAway } from './mobile-dictation-away-insert'
import { nativeChatDictationCopy } from './mobile-session-native-chat-dictation-copy'
import { dictationAwayCopy } from './mobile-dictation-away-copy'
import {
useMobileDictationAnnouncements,
useMobileDictationSpeech
} from './use-mobile-dictation-announcements'
import {
MobileDictationClearanceContext,
MobileDictationHostContext,
@@ -58,6 +62,8 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
const clearToast = useCallback(() => setToast(null), [])
const [clearances, setClearances] = useState<ReadonlyMap<symbol, number>>(new Map())
const clipboard = useClipboardWriter()
const speech = useMobileDictationSpeech()
const { say: speak } = speech
// Bring a dictation's target back: its screen if one is still mounted, otherwise open the session again.
const openOwner = useCallback(
@@ -78,10 +84,12 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
}, [openOwner])
// Say it on the screen that shows the dictation, else in the host's toast (with Open for the dictation it is about).
// A screen's toast is not read by VoiceOver on the page, so it is spoken too; the host's toast is spoken when it shows.
const say = (message: string, about?: MobileDictationOwner) => {
const screen = screenFor(screensRef.current, ownerRef.current, false)
if (screen?.shown()) {
screen.toast(message)
speak(message)
} else {
setToast(about ? { message, open: () => openOwner(about) } : { message })
}
@@ -90,6 +98,10 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
const screen = screenFor(screensRef.current, ownerRef.current, false)
if (screen?.shown()) {
screen.reportFailure(err)
// A setup refusal opens the setup sheet, which is read on its own.
if (!isDictationSetupRequiredError(err.message)) {
speak(dictationFailureMessage(err.message))
}
return
}
triggerError()
@@ -136,9 +148,30 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
useMobileDictationSilenceStop(dictation, owner?.dictationMode, stop, (message) => say(message))
// Screens subscribe to the live dictation; the host's context only changes when its owner or pill does.
const liveRef = useRef(dictation)
const liveRef = useRef<UseMobileDictationResult>(dictation)
const listenersRef = useRef(new Set<() => void>())
const { meter, start: startDictation, stop: stopDictation, cancel, retry } = dictation
const {
meter,
start: startDictation,
stop: stopDictation,
cancel: cancelDictation,
retry
} = dictation
const away =
owner !== null &&
isUnderway(dictation) &&
screenFor(screensRef.current, owner, false)?.shown() !== true
const { noteCancel } = useMobileDictationAnnouncements(
dictation,
{ computerName: owner?.computerName ?? null, away, toast: toast?.message ?? null },
speak
)
// Every ✕ (in place, the pill, the terminal's long press) ends here, so a cancel is never read as "Inserted."
const cancel = useCallback(() => {
noteCancel()
return cancelDictation()
}, [cancelDictation, noteCancel])
const published = useMemo(() => ({ ...dictation, cancel }), [cancel, dictation])
const idle = useMemo<UseMobileDictationResult>(
() => ({
status: 'idle',
@@ -159,7 +192,7 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
)
const idleRef = useRef(idle)
useLayoutEffect(() => {
liveRef.current = dictation
liveRef.current = published
idleRef.current = idle
for (const listener of listenersRef.current) {
listener()
@@ -215,19 +248,18 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
})
}, [])
const away =
owner !== null && isUnderway(dictation) && !screenFor(screensRef.current, owner, false)?.shown()
const title = owner?.title ?? ''
const clearance = Math.max(0, ...clearances.values())
const pill = useMemo(
() => ({ away, title, open, toast, clearToast, clearance }),
[away, clearToast, clearance, open, title, toast]
() => ({ away, title, open, toast, clearToast, clearance, liveText: speech.liveText }),
[speech.liveText, away, clearToast, clearance, open, title, toast]
)
const value = useMemo<MobileDictationHost>(
() => ({
owner,
start,
stop,
announce: speak,
attach,
refresh,
subscribe,
@@ -235,7 +267,18 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
getIdleDictation,
getUnderway
}),
[attach, getDictation, getIdleDictation, getUnderway, owner, refresh, start, stop, subscribe]
[
attach,
getDictation,
getIdleDictation,
getUnderway,
owner,
refresh,
speak,
start,
stop,
subscribe
]
)
return (
<MobileDictationClearanceContext.Provider value={setClearance}>
@@ -0,0 +1,113 @@
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
import { useAccessibilityAnnouncer } from '../platform/accessibility-announcer'
import { dictationAnnounceCopy } from './mobile-dictation-announce-copy'
import { dictationFailureReason } from './mobile-dictation-controls-copy'
// Finishing is said only when it is slow.
const FINISHING_ANNOUNCE_DELAY_MS = 1000
// What the shell could not speak, for the page's live region; the id changes on every message so a repeat is read again.
export type MobileDictationLiveText = { readonly text: string; readonly id: number }
// Speak to the screen reader through the shell where it can, else into the live region. say keeps its identity.
export function useMobileDictationSpeech(): {
say: (message: string) => void
liveText: MobileDictationLiveText
} {
const announce = useAccessibilityAnnouncer()
const announceRef = useRef(announce)
useLayoutEffect(() => {
announceRef.current = announce
})
const [liveText, setLiveText] = useState<MobileDictationLiveText>({ text: '', id: 0 })
const say = useCallback((message: string) => {
void announceRef.current(message).then((spoken) => {
if (!spoken) {
setLiveText((current) => ({ text: message, id: current.id + 1 }))
}
})
}, [])
return { say, liveText }
}
// The dictation's own spoken moments: start, leaving its screen, a move to the computer, a slow finish, a cancel, a kept failure, the host's toasts.
// Outcomes (inserted, failed) are said by whoever knows them: the screen that inserted, or the host's failure report.
export function useMobileDictationAnnouncements(
dictation: Pick<UseMobileDictationResult, 'status' | 'engine' | 'failure'>,
{
computerName,
away,
toast
}: { computerName: string | null; away: boolean; toast: string | null },
say: (message: string) => void
): { noteCancel: () => void } {
const { status, engine, failure } = dictation
const previousStatus = useRef(status)
const cancelled = useRef(false)
const listeningSaid = useRef(false)
const movedAtSaid = useRef<number | null>(null)
const statusRef = useRef(status)
useEffect(() => {
const previous = previousStatus.current
previousStatus.current = status
statusRef.current = status
if (status === 'starting') {
cancelled.current = false
listeningSaid.current = false
}
// Only a ✕ (including Discard on a kept failure); a refused start's reason is said where it is reported.
if (status === 'idle' && previous !== 'idle' && cancelled.current) {
say(dictationAnnounceCopy.cancelled())
}
if (status === 'idle') {
cancelled.current = false
}
}, [say, status])
// Leaving the screen keeps it going; say so once, then the next screen reads as usual.
useEffect(() => {
if (away && statusRef.current === 'recording') {
say(dictationAnnounceCopy.stillListening())
}
}, [away, say])
useEffect(() => {
if (toast) {
say(toast)
}
}, [say, toast])
useEffect(() => {
if (status === 'recording' && engine && !listeningSaid.current) {
listeningSaid.current = true
say(dictationAnnounceCopy.listening(engine.kind, computerName))
}
if (engine?.movedAt != null && engine.movedAt !== movedAtSaid.current) {
movedAtSaid.current = engine.movedAt
say(dictationAnnounceCopy.movedToComputer(computerName))
}
}, [computerName, engine, say, status])
useEffect(() => {
if (status !== 'processing') {
return
}
const timer = setTimeout(
() => say(dictationAnnounceCopy.finishing()),
FINISHING_ANNOUNCE_DELAY_MS
)
return () => clearTimeout(timer)
}, [say, status])
useEffect(() => {
if (failure) {
say(dictationAnnounceCopy.stoppedKept(dictationFailureReason(failure)))
}
}, [failure, say])
const noteCancel = useCallback(() => {
cancelled.current = true
}, [])
return { noteCancel }
}
@@ -17,10 +17,7 @@ export function useMobileDictationLive(
cancelLatest.current = cancel
}, [cancel])
const stableCancel = useCallback(() => cancelLatest.current(), [])
const hostProfile = useHostProfile(hostId)
const hostTitle = useHostDisplay(hostProfile).title
// A workspace routed to another server transcribes there, and the phone lacks that server's name.
const computerName = useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
const computerName = useDictationComputerName(hostId)
const { meter, engine, failure, retry } = dictation
const failureReason = failure === null ? null : dictationFailureReason(failure)
return useMemo(
@@ -35,3 +32,10 @@ export function useMobileDictationLive(
[meter, engine, computerName, stableCancel, failureReason, retry]
)
}
// The computer that transcribes a session's dictation; null when another server runs the workspace and the phone lacks its name.
export function useDictationComputerName(hostId: string): string | null {
const hostProfile = useHostProfile(hostId)
const hostTitle = useHostDisplay(hostProfile).title
return useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
}
+11 -1
View File
@@ -1339,7 +1339,17 @@
},
"announce": {
"typedInto": "Eingetippt in {{target}}",
"insertedIn": "Eingefügt in {{target}}"
"insertedIn": "Eingefügt in {{target}}",
"listeningOnPhone": "Hört zu. Transkription auf diesem Telefon.",
"listeningOn": "Hört zu. Transkription auf {{computer}}.",
"listeningOnComputer": "Hört zu. Transkription auf dem Computer.",
"finishing": "Wird abgeschlossen.",
"inserted": "Eingefügt.",
"cancelled": "Diktat abgebrochen.",
"movedTo": "Dieses Telefon hat die Transkription beendet. Weiter auf {{computer}}.",
"movedToComputer": "Dieses Telefon hat die Transkription beendet. Weiter auf dem Computer.",
"stoppedKept": "Diktat gestoppt. {{reason}}. Deine Aufnahme bleibt erhalten.",
"stillListening": "Hört weiter zu."
},
"pill": {
"open": "Öffnen",
+11 -1
View File
@@ -554,7 +554,17 @@
},
"announce": {
"typedInto": "Typed into {{target}}",
"insertedIn": "Inserted in {{target}}"
"insertedIn": "Inserted in {{target}}",
"listeningOnPhone": "Listening. Transcribing on this phone.",
"listeningOn": "Listening. Transcribing on {{computer}}.",
"listeningOnComputer": "Listening. Transcribing on the computer.",
"finishing": "Finishing.",
"inserted": "Inserted.",
"cancelled": "Dictation cancelled.",
"movedTo": "This phone stopped transcribing. Continuing on {{computer}}.",
"movedToComputer": "This phone stopped transcribing. Continuing on the computer.",
"stoppedKept": "Dictation stopped. {{reason}}. Your recording is kept.",
"stillListening": "Still listening."
},
"pill": {
"open": "Open",
+11 -1
View File
@@ -1380,7 +1380,17 @@
},
"announce": {
"typedInto": "Escrito en {{target}}",
"insertedIn": "Insertado en {{target}}"
"insertedIn": "Insertado en {{target}}",
"listeningOnPhone": "Escuchando. Transcribiendo en este teléfono.",
"listeningOn": "Escuchando. Transcribiendo en {{computer}}.",
"listeningOnComputer": "Escuchando. Transcribiendo en el ordenador.",
"finishing": "Terminando.",
"inserted": "Insertado.",
"cancelled": "Dictado cancelado.",
"movedTo": "Este teléfono dejó de transcribir. Continúa en {{computer}}.",
"movedToComputer": "Este teléfono dejó de transcribir. Continúa en el ordenador.",
"stoppedKept": "Dictado detenido. {{reason}}. Tu grabación se conserva.",
"stillListening": "Sigue escuchando."
},
"pill": {
"open": "Abrir",
+11 -1
View File
@@ -1380,7 +1380,17 @@
},
"announce": {
"typedInto": "Saisi dans {{target}}",
"insertedIn": "Inséré dans {{target}}"
"insertedIn": "Inséré dans {{target}}",
"listeningOnPhone": "Écoute. Transcription sur ce téléphone.",
"listeningOn": "Écoute. Transcription sur {{computer}}.",
"listeningOnComputer": "Écoute. Transcription sur l’ordinateur.",
"finishing": "Finalisation.",
"inserted": "Inséré.",
"cancelled": "Dictée annulée.",
"movedTo": "Ce téléphone a arrêté de transcrire. Suite sur {{computer}}.",
"movedToComputer": "Ce téléphone a arrêté de transcrire. Suite sur l’ordinateur.",
"stoppedKept": "Dictée arrêtée. {{reason}}. Votre enregistrement est conservé.",
"stillListening": "Toujours à l’écoute."
},
"pill": {
"open": "Ouvrir",
+11 -1
View File
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "{{target}} に入力しました",
"insertedIn": "{{target}} に挿入しました"
"insertedIn": "{{target}} に挿入しました",
"listeningOnPhone": "聞き取り中。このスマートフォンで文字起こししています。",
"listeningOn": "聞き取り中。{{computer}} で文字起こししています。",
"listeningOnComputer": "聞き取り中。コンピュータで文字起こししています。",
"finishing": "仕上げ中。",
"inserted": "挿入しました。",
"cancelled": "音声入力をキャンセルしました。",
"movedTo": "このスマートフォンでの文字起こしが止まりました。{{computer}} で続けます。",
"movedToComputer": "このスマートフォンでの文字起こしが止まりました。コンピュータで続けます。",
"stoppedKept": "音声入力が停止しました。{{reason}}。録音は保持されています。",
"stillListening": "聞き取りを続けています。"
},
"pill": {
"open": "開く",
+11 -1
View File
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "{{target}}에 입력됨",
"insertedIn": "{{target}}에 삽입됨"
"insertedIn": "{{target}}에 삽입됨",
"listeningOnPhone": "듣는 중. 이 휴대폰에서 받아쓰는 중입니다.",
"listeningOn": "듣는 중. {{computer}}에서 받아쓰는 중입니다.",
"listeningOnComputer": "듣는 중. 컴퓨터에서 받아쓰는 중입니다.",
"finishing": "마무리 중.",
"inserted": "삽입됨.",
"cancelled": "받아쓰기가 취소되었습니다.",
"movedTo": "이 휴대폰의 받아쓰기가 멈췄습니다. {{computer}}에서 계속합니다.",
"movedToComputer": "이 휴대폰의 받아쓰기가 멈췄습니다. 컴퓨터에서 계속합니다.",
"stoppedKept": "받아쓰기가 중지되었습니다. {{reason}}. 녹음은 보관됩니다.",
"stillListening": "계속 듣는 중입니다."
},
"pill": {
"open": "열기",
+11 -1
View File
@@ -1380,7 +1380,17 @@
},
"announce": {
"typedInto": "Digitado em {{target}}",
"insertedIn": "Inserido em {{target}}"
"insertedIn": "Inserido em {{target}}",
"listeningOnPhone": "Ouvindo. Transcrevendo neste telefone.",
"listeningOn": "Ouvindo. Transcrevendo em {{computer}}.",
"listeningOnComputer": "Ouvindo. Transcrevendo no computador.",
"finishing": "Finalizando.",
"inserted": "Inserido.",
"cancelled": "Ditado cancelado.",
"movedTo": "Este telefone parou de transcrever. Continuando em {{computer}}.",
"movedToComputer": "Este telefone parou de transcrever. Continuando no computador.",
"stoppedKept": "Ditado interrompido. {{reason}}. Sua gravação foi mantida.",
"stillListening": "Ainda ouvindo."
},
"pill": {
"open": "Abrir",
+11 -1
View File
@@ -1421,7 +1421,17 @@
},
"announce": {
"typedInto": "Введено в {{target}}",
"insertedIn": "Вставлено в {{target}}"
"insertedIn": "Вставлено в {{target}}",
"listeningOnPhone": "Слушаю. Распознавание на этом телефоне.",
"listeningOn": "Слушаю. Распознавание на {{computer}}.",
"listeningOnComputer": "Слушаю. Распознавание на компьютере.",
"finishing": "Завершение.",
"inserted": "Вставлено.",
"cancelled": "Диктовка отменена.",
"movedTo": "Этот телефон перестал распознавать. Продолжение на {{computer}}.",
"movedToComputer": "Этот телефон перестал распознавать. Продолжение на компьютере.",
"stoppedKept": "Диктовка остановлена. {{reason}}. Запись сохранена.",
"stillListening": "Продолжаю слушать."
},
"pill": {
"open": "Открыть",
+11 -1
View File
@@ -1339,7 +1339,17 @@
},
"announce": {
"typedInto": "{{target}} içine yazıldı",
"insertedIn": "{{target}} içine eklendi"
"insertedIn": "{{target}} içine eklendi",
"listeningOnPhone": "Dinleniyor. Bu telefonda yazıya dökülüyor.",
"listeningOn": "Dinleniyor. {{computer}} üzerinde yazıya dökülüyor.",
"listeningOnComputer": "Dinleniyor. Bilgisayarda yazıya dökülüyor.",
"finishing": "Tamamlanıyor.",
"inserted": "Eklendi.",
"cancelled": "Dikte iptal edildi.",
"movedTo": "Bu telefon yazıya dökmeyi bıraktı. {{computer}} üzerinde devam ediliyor.",
"movedToComputer": "Bu telefon yazıya dökmeyi bıraktı. Bilgisayarda devam ediliyor.",
"stoppedKept": "Dikte durdu. {{reason}}. Kaydınız saklandı.",
"stillListening": "Dinlemeye devam ediliyor."
},
"pill": {
"open": "Aç",
+11 -1
View File
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "已輸入到 {{target}}",
"insertedIn": "已插入到 {{target}}"
"insertedIn": "已插入到 {{target}}",
"listeningOnPhone": "正在聽。正在此手機上轉寫。",
"listeningOn": "正在聽。正在 {{computer}} 上轉寫。",
"listeningOnComputer": "正在聽。正在電腦上轉寫。",
"finishing": "正在完成。",
"inserted": "已插入。",
"cancelled": "已取消聽寫。",
"movedTo": "此手機已停止轉寫。在 {{computer}} 上繼續。",
"movedToComputer": "此手機已停止轉寫。在電腦上繼續。",
"stoppedKept": "聽寫已停止。{{reason}}。錄音已保留。",
"stillListening": "仍在聽。"
},
"pill": {
"open": "開啟",
+11 -1
View File
@@ -1335,7 +1335,17 @@
},
"announce": {
"typedInto": "已输入到 {{target}}",
"insertedIn": "已插入到 {{target}}"
"insertedIn": "已插入到 {{target}}",
"listeningOnPhone": "正在听。正在此手机上转写。",
"listeningOn": "正在听。正在 {{computer}} 上转写。",
"listeningOnComputer": "正在听。正在电脑上转写。",
"finishing": "正在完成。",
"inserted": "已插入。",
"cancelled": "已取消听写。",
"movedTo": "此手机已停止转写。在 {{computer}} 上继续。",
"movedToComputer": "此手机已停止转写。在电脑上继续。",
"stoppedKept": "听写已停止。{{reason}}。录音已保留。",
"stillListening": "仍在听。"
},
"pill": {
"open": "打开",
@@ -41,7 +41,8 @@ export const BRIDGE_NATIVE_VERB_NAMES = [
'native.pdf.view',
'native.image.act',
BRIDGE_SPEECH_VERB,
BRIDGE_STORE_VERB
BRIDGE_STORE_VERB,
'native.a11y.announce'
] as const
export type BridgeNativeVerb = (typeof BRIDGE_NATIVE_VERB_NAMES)[number]
@@ -93,6 +94,11 @@ export type BridgePdfOpenParams = Omit<
'action'
>
// The page has no announce call of its own (RNW's is a no-op); the shell speaks for it. Long text is a page bug, so it is refused.
export const a11yAnnounceParamsSchema = z.strictObject({ message: z.string().min(1).max(500) })
// announced:false when the shell leaves it to the page's live region (Android, where TalkBack reads it).
export const a11yAnnounceResultSchema = z.strictObject({ announced: z.boolean() })
export const BRIDGE_NATIVE_VERBS: Readonly<Record<BridgeNativeVerb, BridgeNativeVerbSpec>> = {
'native.clipboard.write': {
params: clipboardWriteParamsSchema,
@@ -150,6 +156,10 @@ export const BRIDGE_NATIVE_VERBS: Readonly<Record<BridgeNativeVerb, BridgeNative
[BRIDGE_STORE_VERB]: {
params: storeParamsSchema,
result: storeResultSchema
},
'native.a11y.announce': {
params: a11yAnnounceParamsSchema,
result: a11yAnnounceResultSchema
}
}
@@ -18,6 +18,7 @@ import {
} from './bridge-audio-verbs'
import {
clipboardReadResultSchema,
a11yAnnounceResultSchema,
clipboardWriteResultSchema,
pdfViewResultSchema,
type BridgeClipboardMime,
@@ -70,6 +71,10 @@ export type NativeVerbs = {
speechRead: () => Promise<BridgeSpeechRead>
speechFinish: () => Promise<BridgeSpeechFinish>
speechCancel: () => Promise<void>
// Older shells lack it; the page then keeps its live region, which VoiceOver may not read.
canAnnounce: boolean
// True when the shell spoke it.
announce: (message: string) => Promise<boolean>
}
// Expose stable reason codes, including local ungranted and frame refusals. Unknown newer-shell codes floor to unreported so caller switches remain exhaustive.
@@ -204,7 +209,10 @@ export function useNativeVerbs(): NativeVerbs {
speechFinish: () => call(BRIDGE_SPEECH_VERB, { action: 'finish' }, speechFinishResultSchema),
speechCancel: async () => {
await call(BRIDGE_SPEECH_VERB, { action: 'cancel' }, speechCancelResultSchema)
}
},
canAnnounce: has('native.a11y.announce'),
announce: async (message) =>
(await call('native.a11y.announce', { message }, a11yAnnounceResultSchema)).announced
}
}, [client])
}
@@ -0,0 +1,10 @@
import { useCallback } from 'react'
import { AccessibilityInfo } from 'react-native'
// Speak a message to the screen reader; false when the caller should put it in a live region instead.
export function useAccessibilityAnnouncer(): (message: string) => Promise<boolean> {
return useCallback(async (message: string) => {
AccessibilityInfo.announceForAccessibility(message)
return true
}, [])
}
@@ -0,0 +1,16 @@
import { useCallback } from 'react'
import { useNativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
// RNW's announceForAccessibility does nothing, so the shell announces; older shells and Android leave it to the page's live region.
export function useAccessibilityAnnouncer(): (message: string) => Promise<boolean> {
const verbs = useNativeVerbs()
return useCallback(
async (message: string) => {
if (!verbs.canAnnounce) {
return false
}
return verbs.announce(message).catch(() => false)
},
[verbs]
)
}
@@ -0,0 +1,12 @@
import { AccessibilityInfo, Platform } from 'react-native'
import { a11yAnnounceParamsSchema } from '../mobile-web-shell/bridge/bridge-native-verbs'
// VoiceOver ignores the page's live regions inside the shell's WebView, so the shell announces for it; TalkBack reads them, so Android leaves it to the page.
export function serveNativeAnnounceVerb(params: unknown): { announced: boolean } {
const { message } = a11yAnnounceParamsSchema.parse(params)
if (Platform.OS !== 'ios') {
return { announced: false }
}
AccessibilityInfo.announceForAccessibilityWithOptions(message, { queue: true })
return { announced: true }
}
@@ -17,6 +17,11 @@ const device = vi.hoisted(() => ({
wakeTags: new Array<string>()
}))
// The shell's announce verb speaks through React Native's AccessibilityInfo.
vi.mock('react-native', () => ({
AccessibilityInfo: { announceForAccessibilityWithOptions: () => {} },
Platform: { OS: 'ios' }
}))
vi.mock('expo-clipboard', () => ({
setStringAsync: () => Promise.resolve(true),
getStringAsync: device.clipboard.getStringAsync,
@@ -7,6 +7,7 @@ import { NATIVE_AUDIO_DEVICE_SAMPLE_RATE, nativeAudioDeviceEngine } from './nati
import { createNativeSpeechServer } from './native-speech'
import { nativeSpeechDeviceEngine } from './native-speech-device'
import { serveNativeClipboardVerb } from './native-clipboard'
import { serveNativeAnnounceVerb } from './native-accessibility-announce'
import { createNativeMediaVerbServer } from './native-media'
import { discardStagedMedia, nativeMediaDeviceDeps } from './native-media-device'
@@ -37,6 +38,9 @@ export function useNativeDeviceVerbs(
if (verb === 'native.speech.act') {
return speech.serve(params)
}
if (verb === 'native.a11y.announce') {
return Promise.resolve(serveNativeAnnounceVerb(params))
}
if (verb === 'native.audio.start') {
return speech.mayCapture().then((may) =>
may
@@ -9,7 +9,6 @@ import { styles } from './mobile-session-styles'
import type { MobileSessionController } from './use-mobile-session-controller'
import { MobileSessionActiveContent } from './MobileSessionActiveContent'
import { MobileSessionCommandDock } from './MobileSessionCommandDock'
import { MobileDictationStatusAnnouncement } from '../dictation/MobileDictationStatusAnnouncement'
import { MobileDictationLiveContext } from '../dictation/mobile-dictation-live-context'
import { useMobileDictationLive } from '../dictation/use-mobile-dictation-live'
@@ -56,7 +55,6 @@ export function MobileSessionContentRow({ controller }: { controller: MobileSess
{/* Why: translate instead of resize so keyboard toggles don't trigger a server-side PTY viewport change. */}
<MobileSessionCommandDock controller={controller} />
</MobileDictationLiveContext.Provider>
<MobileDictationStatusAnnouncement dictation={dictation} />
</View>
{canDockPanel && activePanel !== null && (
<SessionDockColumn
@@ -26,8 +26,11 @@ const seen = vi.hoisted(() => {
vi.mock('react-native', () => ({
AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) },
AccessibilityInfo: { announceForAccessibility: () => {} },
Platform: { OS: 'ios' }
}))
// The computer's name only words announcements; the stored host profile pulls in secure storage.
vi.mock('../transport/hosts/use-host-profile', () => ({ useHostProfile: () => null }))
vi.mock('@orca/expo-two-way-audio', () => ({
addExpoTwoWayAudioEventListener: (name: string, listener: (event: { data: string }) => void) => {
seen.audioListeners.set(name, listener)
@@ -22,7 +22,9 @@ import {
import { useMobileDictationLeaveHold } from '../dictation/mobile-dictation-leave-hold'
import { useMobileDictationHost } from '../dictation/mobile-dictation-host'
import { mobileDictationShownOnScreen } from '../dictation/mobile-dictation-shown'
import { useDictationComputerName } from '../dictation/use-mobile-dictation-live'
import { nativeChatDictationCopy } from '../dictation/mobile-session-native-chat-dictation-copy'
import { dictationAnnounceCopy } from '../dictation/mobile-dictation-announce-copy'
import {
useWorkspaceExecutionHost,
workspaceRouteHref
@@ -54,6 +56,7 @@ export function useMobileSessionDictation(
} = scope
const { showNativeChat, showNativeChatRef } = nativeChatController
const host = useMobileDictationHost()
const computerName = useDictationComputerName(hostId)
const { start: startHost } = host
const executionHost = useWorkspaceExecutionHost()
const sessionKey = JSON.stringify([hostId, executionHost ?? null, worktreeId])
@@ -93,12 +96,17 @@ export function useMobileSessionDictation(
)
const clipboard = useClipboardWriter()
// The screen's toast is not read on the page, so where the words went is also spoken.
const tell = (message: string) => {
showToast(message)
host.announce(message)
}
// A transcript its target refused must not vanish: keep it on the clipboard and say so.
const keepUndeliveredDictation = (text: string) => {
triggerError()
void clipboard.writeText(text).then(
() => showToast(nativeChatDictationCopy.insertFailedCopied()),
() => showToast(nativeChatDictationCopy.insertFailed())
() => tell(nativeChatDictationCopy.insertFailedCopied()),
() => tell(nativeChatDictationCopy.insertFailed())
)
}
@@ -106,7 +114,9 @@ export function useMobileSessionDictation(
// Chat dictation is inserted locally; a terminal's words are typed into it live, while it is the active one.
const insertHere = (text: string): boolean => {
if (destination.landsInChat(showNativeChatRef.current)) {
if (!destination.insertInChat(text)) {
if (destination.insertInChat(text)) {
host.announce(dictationAnnounceCopy.inserted())
} else {
keepUndeliveredDictation(text)
}
return true
@@ -124,7 +134,7 @@ export function useMobileSessionDictation(
(await sendLiveTerminalInput(insertHandle, text))
// Live input lands in a terminal the keyboard may cover, so say where the words went.
if (sent) {
showToast(nativeChatDictationCopy.inserted())
tell(nativeChatDictationCopy.inserted())
} else {
keepUndeliveredDictation(text)
}
@@ -237,9 +247,11 @@ export function useMobileSessionDictation(
dictationMode: scope.dictationMode,
removeFillerWords,
dictationTerms,
worktreeId
worktreeId,
computerName
})
}, [
computerName,
activeHandle,
activeSessionTab,
busyElsewhere,
+4
View File
@@ -244,6 +244,10 @@
{
"file": "src/platform/page-box.web.ts",
"reason": "RNW's measure() is asynchronous and drops its callback for a detached node, so a held mic released over the dictation cancel button would never stop. The page reads the DOM box synchronously; native keeps measure()."
},
{
"file": "src/platform/accessibility-announcer.web.ts",
"reason": "React Native Web's announceForAccessibility is a no-op and VoiceOver in the shell's WebView doesn't read the page's live regions, so the page asks the shell to announce through native.a11y.announce; without it, the caller keeps its live region."
}
]
}