mirror of
https://github.com/stablyai/orca.git
synced 2026-11-13 09:27:29 +00:00
feat(mobile): read dictation progress to VoiceOver and TalkBack (#40504)
* feat(mobile): announce dictation to VoiceOver through the shell * feat(mobile): speak dictation from the host, so leaving the session and the pill's toasts are announced too * fix(mobile): say dictation outcomes where they are known, and re-read a repeated live message * fix(mobile): leave a setup refusal to its setup sheet instead of reading the raw code * fix(mobile): say cancelled when a kept recording is discarded
This commit is contained in:
@@ -102,13 +102,14 @@ export const MOBILE_WEB_PAGE_ROUTES = [
|
||||
'native.audio.read',
|
||||
'native.audio.stop'
|
||||
],
|
||||
// Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act; dictation uses the computer without native.speech.act.
|
||||
// Optional media.name falls back to the file type; terminalKeyRail falls back to the page's row; the image viewer drops its actions without native.image.act; dictation uses the computer without native.speech.act; without native.a11y.announce only the page's live region speaks.
|
||||
optionalGrants: [
|
||||
'externalNavigation',
|
||||
'native.media.name',
|
||||
'native.pdf.view',
|
||||
'native.image.act',
|
||||
'native.speech.act',
|
||||
'native.a11y.announce',
|
||||
'terminalKeyRail',
|
||||
DEVICE_STORE_GRANT
|
||||
]
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
} from './mobile-dictation-controls-copy'
|
||||
import { dictationAwayCopy } from './mobile-dictation-away-copy'
|
||||
import { composerInputStyles } from '../components/composer-input/composer-input-styles'
|
||||
import { MobileDictationStatusAnnouncement } from './MobileDictationStatusAnnouncement'
|
||||
import {
|
||||
useMobileDictationHost,
|
||||
useMobileDictationHostLive,
|
||||
@@ -163,13 +164,9 @@ function MobileDictationToast({
|
||||
const timer = setTimeout(onDone, toast.open ? TOAST_WITH_OPEN_MS : TOAST_MS)
|
||||
return () => clearTimeout(timer)
|
||||
}, [onDone, toast])
|
||||
// The live region stays mounted so a screen reader hears the message appear.
|
||||
// The host's announcer reads the message (shell or live region), so the toast itself stays quiet.
|
||||
return (
|
||||
<View
|
||||
style={[styles.toastRow, { bottom }]}
|
||||
pointerEvents="box-none"
|
||||
accessibilityLiveRegion="polite"
|
||||
>
|
||||
<View style={[styles.toastRow, { bottom }]} pointerEvents="box-none">
|
||||
{toast ? (
|
||||
<View style={styles.toast}>
|
||||
<Text style={styles.toastText} numberOfLines={2}>
|
||||
@@ -212,6 +209,7 @@ export function MobileDictationPillLayer(): React.JSX.Element | null {
|
||||
<MobileDictationPill title={pill.title} onOpen={pill.open} onStop={stop} />
|
||||
</View>
|
||||
) : null}
|
||||
<MobileDictationStatusAnnouncement liveText={pill.liveText} />
|
||||
<MobileDictationToast
|
||||
toast={pill.toast}
|
||||
onDone={pill.clearToast}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { StyleSheet, Text, View } from 'react-native'
|
||||
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
|
||||
import { translate } from '../i18n/mobile-i18n'
|
||||
import { useMobileTranslation } from '../i18n/use-mobile-translation'
|
||||
import type { MobileDictationLiveText } from './use-mobile-dictation-announcements'
|
||||
|
||||
export type MobileDictationStatusFlags = Pick<
|
||||
UseMobileDictationResult,
|
||||
@@ -18,16 +18,16 @@ export function mobileDictationStatusTitle(dictation: MobileDictationStatusFlags
|
||||
: null
|
||||
}
|
||||
|
||||
// Mic labels change silently; one live region tells screen readers for every mic, like desktop's indicator.
|
||||
// The live region for announcements the shell could not speak itself (TalkBack reads it; older shells fall back to it).
|
||||
// A new Text per message, so the same words twice are read twice.
|
||||
export function MobileDictationStatusAnnouncement({
|
||||
dictation
|
||||
liveText
|
||||
}: {
|
||||
dictation: MobileDictationStatusFlags
|
||||
liveText: MobileDictationLiveText
|
||||
}): React.JSX.Element {
|
||||
useMobileTranslation()
|
||||
return (
|
||||
<View style={styles.hidden} pointerEvents="none" accessibilityLiveRegion="polite">
|
||||
<Text>{mobileDictationStatusTitle(dictation) ?? ''}</Text>
|
||||
<Text key={liveText.id}>{liveText.text}</Text>
|
||||
</View>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { translate } from '../i18n/mobile-i18n'
|
||||
|
||||
// What a screen reader hears as a dictation moves along; nothing while you talk, so it never talks over you.
|
||||
export const dictationAnnounceCopy = {
|
||||
listening: (engine: 'phone' | 'computer', computer: string | null) =>
|
||||
engine === 'phone'
|
||||
? translate('dictation.announce.listeningOnPhone', 'Listening. Transcribing on this phone.')
|
||||
: computer
|
||||
? translate('dictation.announce.listeningOn', 'Listening. Transcribing on {{computer}}.', {
|
||||
computer
|
||||
})
|
||||
: translate(
|
||||
'dictation.announce.listeningOnComputer',
|
||||
'Listening. Transcribing on the computer.'
|
||||
),
|
||||
stillListening: () => translate('dictation.announce.stillListening', 'Still listening.'),
|
||||
finishing: () => translate('dictation.announce.finishing', 'Finishing.'),
|
||||
inserted: () => translate('dictation.announce.inserted', 'Inserted.'),
|
||||
cancelled: () => translate('dictation.announce.cancelled', 'Dictation cancelled.'),
|
||||
movedToComputer: (computer: string | null) =>
|
||||
computer
|
||||
? translate(
|
||||
'dictation.announce.movedTo',
|
||||
'This phone stopped transcribing. Continuing on {{computer}}.',
|
||||
{ computer }
|
||||
)
|
||||
: translate(
|
||||
'dictation.announce.movedToComputer',
|
||||
'This phone stopped transcribing. Continuing on the computer.'
|
||||
),
|
||||
stoppedKept: (reason: string) =>
|
||||
translate(
|
||||
'dictation.announce.stoppedKept',
|
||||
'Dictation stopped. {{reason}}. Your recording is kept.',
|
||||
{
|
||||
reason
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { createContext, useContext, useEffect, useState, useSyncExternalStore }
|
||||
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
|
||||
import type { RpcClient } from '../transport/rpc/rpc-client-contract'
|
||||
import type { MobileDictationAwayTarget } from './mobile-dictation-away-insert'
|
||||
import type { MobileDictationLiveText } from './use-mobile-dictation-announcements'
|
||||
import type { DictationTerm } from '../../../src/shared/dictation-dictionary'
|
||||
|
||||
export type MobileDictationHostToast = { message: string; open?: () => void }
|
||||
@@ -25,6 +26,8 @@ export type MobileDictationOwner = {
|
||||
readonly dictationTerms: readonly DictationTerm[] | undefined
|
||||
// The computer spells its own words with this worktree's repo terms.
|
||||
readonly worktreeId: string
|
||||
// The computer that would transcribe it, for what a screen reader hears; null when the phone lacks its name.
|
||||
readonly computerName: string | null
|
||||
}
|
||||
|
||||
// A mounted session screen, as the dictation host sees it.
|
||||
@@ -48,6 +51,8 @@ export type MobileDictationHost = {
|
||||
readonly start: (owner: MobileDictationOwner) => Promise<void>
|
||||
// Stop with the selection tick, as every stop button does.
|
||||
readonly stop: () => void
|
||||
// Say something to the screen reader; a screen's own toast is not read on the page.
|
||||
readonly announce: (message: string) => void
|
||||
readonly attach: (screen: MobileDictationScreen) => () => void
|
||||
// A screen's shown() answer changed; the pill reads it on render.
|
||||
readonly refresh: () => void
|
||||
@@ -69,6 +74,8 @@ export type MobileDictationPill = {
|
||||
readonly clearToast: () => void
|
||||
// The tallest bottom bar a screen reported, so the pill floats above it.
|
||||
readonly clearance: number
|
||||
// Announcements the shell could not speak, for the page's live region (TalkBack, older shells).
|
||||
readonly liveText: MobileDictationLiveText
|
||||
}
|
||||
|
||||
export const MobileDictationHostContext = createContext<MobileDictationHost | null>(null)
|
||||
|
||||
@@ -4,12 +4,16 @@ import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session
|
||||
import { useRouteHandoff } from '../navigation/route-handoff'
|
||||
import { triggerError } from '../platform/haptics'
|
||||
import { useClipboardWriter } from '../platform/clipboard'
|
||||
import { dictationFailureMessage } from './mobile-dictation-setup'
|
||||
import { dictationFailureMessage, isDictationSetupRequiredError } from './mobile-dictation-setup'
|
||||
import { useMobileDictationHaptics } from './use-mobile-dictation-haptics'
|
||||
import { useMobileDictationSilenceStop } from './use-mobile-dictation-silence-stop'
|
||||
import { insertMobileDictationAway } from './mobile-dictation-away-insert'
|
||||
import { nativeChatDictationCopy } from './mobile-session-native-chat-dictation-copy'
|
||||
import { dictationAwayCopy } from './mobile-dictation-away-copy'
|
||||
import {
|
||||
useMobileDictationAnnouncements,
|
||||
useMobileDictationSpeech
|
||||
} from './use-mobile-dictation-announcements'
|
||||
import {
|
||||
MobileDictationClearanceContext,
|
||||
MobileDictationHostContext,
|
||||
@@ -58,6 +62,8 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
const clearToast = useCallback(() => setToast(null), [])
|
||||
const [clearances, setClearances] = useState<ReadonlyMap<symbol, number>>(new Map())
|
||||
const clipboard = useClipboardWriter()
|
||||
const speech = useMobileDictationSpeech()
|
||||
const { say: speak } = speech
|
||||
|
||||
// Bring a dictation's target back: its screen if one is still mounted, otherwise open the session again.
|
||||
const openOwner = useCallback(
|
||||
@@ -78,10 +84,12 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
}, [openOwner])
|
||||
|
||||
// Say it on the screen that shows the dictation, else in the host's toast (with Open for the dictation it is about).
|
||||
// A screen's toast is not read by VoiceOver on the page, so it is spoken too; the host's toast is spoken when it shows.
|
||||
const say = (message: string, about?: MobileDictationOwner) => {
|
||||
const screen = screenFor(screensRef.current, ownerRef.current, false)
|
||||
if (screen?.shown()) {
|
||||
screen.toast(message)
|
||||
speak(message)
|
||||
} else {
|
||||
setToast(about ? { message, open: () => openOwner(about) } : { message })
|
||||
}
|
||||
@@ -90,6 +98,10 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
const screen = screenFor(screensRef.current, ownerRef.current, false)
|
||||
if (screen?.shown()) {
|
||||
screen.reportFailure(err)
|
||||
// A setup refusal opens the setup sheet, which is read on its own.
|
||||
if (!isDictationSetupRequiredError(err.message)) {
|
||||
speak(dictationFailureMessage(err.message))
|
||||
}
|
||||
return
|
||||
}
|
||||
triggerError()
|
||||
@@ -136,9 +148,30 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
useMobileDictationSilenceStop(dictation, owner?.dictationMode, stop, (message) => say(message))
|
||||
|
||||
// Screens subscribe to the live dictation; the host's context only changes when its owner or pill does.
|
||||
const liveRef = useRef(dictation)
|
||||
const liveRef = useRef<UseMobileDictationResult>(dictation)
|
||||
const listenersRef = useRef(new Set<() => void>())
|
||||
const { meter, start: startDictation, stop: stopDictation, cancel, retry } = dictation
|
||||
const {
|
||||
meter,
|
||||
start: startDictation,
|
||||
stop: stopDictation,
|
||||
cancel: cancelDictation,
|
||||
retry
|
||||
} = dictation
|
||||
const away =
|
||||
owner !== null &&
|
||||
isUnderway(dictation) &&
|
||||
screenFor(screensRef.current, owner, false)?.shown() !== true
|
||||
const { noteCancel } = useMobileDictationAnnouncements(
|
||||
dictation,
|
||||
{ computerName: owner?.computerName ?? null, away, toast: toast?.message ?? null },
|
||||
speak
|
||||
)
|
||||
// Every ✕ (in place, the pill, the terminal's long press) ends here, so a cancel is never read as "Inserted."
|
||||
const cancel = useCallback(() => {
|
||||
noteCancel()
|
||||
return cancelDictation()
|
||||
}, [cancelDictation, noteCancel])
|
||||
const published = useMemo(() => ({ ...dictation, cancel }), [cancel, dictation])
|
||||
const idle = useMemo<UseMobileDictationResult>(
|
||||
() => ({
|
||||
status: 'idle',
|
||||
@@ -159,7 +192,7 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
)
|
||||
const idleRef = useRef(idle)
|
||||
useLayoutEffect(() => {
|
||||
liveRef.current = dictation
|
||||
liveRef.current = published
|
||||
idleRef.current = idle
|
||||
for (const listener of listenersRef.current) {
|
||||
listener()
|
||||
@@ -215,19 +248,18 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
})
|
||||
}, [])
|
||||
|
||||
const away =
|
||||
owner !== null && isUnderway(dictation) && !screenFor(screensRef.current, owner, false)?.shown()
|
||||
const title = owner?.title ?? ''
|
||||
const clearance = Math.max(0, ...clearances.values())
|
||||
const pill = useMemo(
|
||||
() => ({ away, title, open, toast, clearToast, clearance }),
|
||||
[away, clearToast, clearance, open, title, toast]
|
||||
() => ({ away, title, open, toast, clearToast, clearance, liveText: speech.liveText }),
|
||||
[speech.liveText, away, clearToast, clearance, open, title, toast]
|
||||
)
|
||||
const value = useMemo<MobileDictationHost>(
|
||||
() => ({
|
||||
owner,
|
||||
start,
|
||||
stop,
|
||||
announce: speak,
|
||||
attach,
|
||||
refresh,
|
||||
subscribe,
|
||||
@@ -235,7 +267,18 @@ export function MobileDictationHostProvider({ children }: { children: ReactNode
|
||||
getIdleDictation,
|
||||
getUnderway
|
||||
}),
|
||||
[attach, getDictation, getIdleDictation, getUnderway, owner, refresh, start, stop, subscribe]
|
||||
[
|
||||
attach,
|
||||
getDictation,
|
||||
getIdleDictation,
|
||||
getUnderway,
|
||||
owner,
|
||||
refresh,
|
||||
speak,
|
||||
start,
|
||||
stop,
|
||||
subscribe
|
||||
]
|
||||
)
|
||||
return (
|
||||
<MobileDictationClearanceContext.Provider value={setClearance}>
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
|
||||
import type { UseMobileDictationResult } from '../hooks/mobile-dictation-session-state'
|
||||
import { useAccessibilityAnnouncer } from '../platform/accessibility-announcer'
|
||||
import { dictationAnnounceCopy } from './mobile-dictation-announce-copy'
|
||||
import { dictationFailureReason } from './mobile-dictation-controls-copy'
|
||||
|
||||
// Finishing is said only when it is slow.
|
||||
const FINISHING_ANNOUNCE_DELAY_MS = 1000
|
||||
|
||||
// What the shell could not speak, for the page's live region; the id changes on every message so a repeat is read again.
|
||||
export type MobileDictationLiveText = { readonly text: string; readonly id: number }
|
||||
|
||||
// Speak to the screen reader through the shell where it can, else into the live region. say keeps its identity.
|
||||
export function useMobileDictationSpeech(): {
|
||||
say: (message: string) => void
|
||||
liveText: MobileDictationLiveText
|
||||
} {
|
||||
const announce = useAccessibilityAnnouncer()
|
||||
const announceRef = useRef(announce)
|
||||
useLayoutEffect(() => {
|
||||
announceRef.current = announce
|
||||
})
|
||||
const [liveText, setLiveText] = useState<MobileDictationLiveText>({ text: '', id: 0 })
|
||||
const say = useCallback((message: string) => {
|
||||
void announceRef.current(message).then((spoken) => {
|
||||
if (!spoken) {
|
||||
setLiveText((current) => ({ text: message, id: current.id + 1 }))
|
||||
}
|
||||
})
|
||||
}, [])
|
||||
return { say, liveText }
|
||||
}
|
||||
|
||||
// The dictation's own spoken moments: start, leaving its screen, a move to the computer, a slow finish, a cancel, a kept failure, the host's toasts.
|
||||
// Outcomes (inserted, failed) are said by whoever knows them: the screen that inserted, or the host's failure report.
|
||||
export function useMobileDictationAnnouncements(
|
||||
dictation: Pick<UseMobileDictationResult, 'status' | 'engine' | 'failure'>,
|
||||
{
|
||||
computerName,
|
||||
away,
|
||||
toast
|
||||
}: { computerName: string | null; away: boolean; toast: string | null },
|
||||
say: (message: string) => void
|
||||
): { noteCancel: () => void } {
|
||||
const { status, engine, failure } = dictation
|
||||
const previousStatus = useRef(status)
|
||||
const cancelled = useRef(false)
|
||||
const listeningSaid = useRef(false)
|
||||
const movedAtSaid = useRef<number | null>(null)
|
||||
const statusRef = useRef(status)
|
||||
|
||||
useEffect(() => {
|
||||
const previous = previousStatus.current
|
||||
previousStatus.current = status
|
||||
statusRef.current = status
|
||||
if (status === 'starting') {
|
||||
cancelled.current = false
|
||||
listeningSaid.current = false
|
||||
}
|
||||
// Only a ✕ (including Discard on a kept failure); a refused start's reason is said where it is reported.
|
||||
if (status === 'idle' && previous !== 'idle' && cancelled.current) {
|
||||
say(dictationAnnounceCopy.cancelled())
|
||||
}
|
||||
if (status === 'idle') {
|
||||
cancelled.current = false
|
||||
}
|
||||
}, [say, status])
|
||||
|
||||
// Leaving the screen keeps it going; say so once, then the next screen reads as usual.
|
||||
useEffect(() => {
|
||||
if (away && statusRef.current === 'recording') {
|
||||
say(dictationAnnounceCopy.stillListening())
|
||||
}
|
||||
}, [away, say])
|
||||
useEffect(() => {
|
||||
if (toast) {
|
||||
say(toast)
|
||||
}
|
||||
}, [say, toast])
|
||||
|
||||
useEffect(() => {
|
||||
if (status === 'recording' && engine && !listeningSaid.current) {
|
||||
listeningSaid.current = true
|
||||
say(dictationAnnounceCopy.listening(engine.kind, computerName))
|
||||
}
|
||||
if (engine?.movedAt != null && engine.movedAt !== movedAtSaid.current) {
|
||||
movedAtSaid.current = engine.movedAt
|
||||
say(dictationAnnounceCopy.movedToComputer(computerName))
|
||||
}
|
||||
}, [computerName, engine, say, status])
|
||||
|
||||
useEffect(() => {
|
||||
if (status !== 'processing') {
|
||||
return
|
||||
}
|
||||
const timer = setTimeout(
|
||||
() => say(dictationAnnounceCopy.finishing()),
|
||||
FINISHING_ANNOUNCE_DELAY_MS
|
||||
)
|
||||
return () => clearTimeout(timer)
|
||||
}, [say, status])
|
||||
|
||||
useEffect(() => {
|
||||
if (failure) {
|
||||
say(dictationAnnounceCopy.stoppedKept(dictationFailureReason(failure)))
|
||||
}
|
||||
}, [failure, say])
|
||||
|
||||
const noteCancel = useCallback(() => {
|
||||
cancelled.current = true
|
||||
}, [])
|
||||
return { noteCancel }
|
||||
}
|
||||
@@ -17,10 +17,7 @@ export function useMobileDictationLive(
|
||||
cancelLatest.current = cancel
|
||||
}, [cancel])
|
||||
const stableCancel = useCallback(() => cancelLatest.current(), [])
|
||||
const hostProfile = useHostProfile(hostId)
|
||||
const hostTitle = useHostDisplay(hostProfile).title
|
||||
// A workspace routed to another server transcribes there, and the phone lacks that server's name.
|
||||
const computerName = useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
|
||||
const computerName = useDictationComputerName(hostId)
|
||||
const { meter, engine, failure, retry } = dictation
|
||||
const failureReason = failure === null ? null : dictationFailureReason(failure)
|
||||
return useMemo(
|
||||
@@ -35,3 +32,10 @@ export function useMobileDictationLive(
|
||||
[meter, engine, computerName, stableCancel, failureReason, retry]
|
||||
)
|
||||
}
|
||||
|
||||
// The computer that transcribes a session's dictation; null when another server runs the workspace and the phone lacks its name.
|
||||
export function useDictationComputerName(hostId: string): string | null {
|
||||
const hostProfile = useHostProfile(hostId)
|
||||
const hostTitle = useHostDisplay(hostProfile).title
|
||||
return useWorkspaceExecutionHost() === undefined && hostProfile ? hostTitle : null
|
||||
}
|
||||
|
||||
@@ -1339,7 +1339,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "Eingetippt in {{target}}",
|
||||
"insertedIn": "Eingefügt in {{target}}"
|
||||
"insertedIn": "Eingefügt in {{target}}",
|
||||
"listeningOnPhone": "Hört zu. Transkription auf diesem Telefon.",
|
||||
"listeningOn": "Hört zu. Transkription auf {{computer}}.",
|
||||
"listeningOnComputer": "Hört zu. Transkription auf dem Computer.",
|
||||
"finishing": "Wird abgeschlossen.",
|
||||
"inserted": "Eingefügt.",
|
||||
"cancelled": "Diktat abgebrochen.",
|
||||
"movedTo": "Dieses Telefon hat die Transkription beendet. Weiter auf {{computer}}.",
|
||||
"movedToComputer": "Dieses Telefon hat die Transkription beendet. Weiter auf dem Computer.",
|
||||
"stoppedKept": "Diktat gestoppt. {{reason}}. Deine Aufnahme bleibt erhalten.",
|
||||
"stillListening": "Hört weiter zu."
|
||||
},
|
||||
"pill": {
|
||||
"open": "Öffnen",
|
||||
|
||||
@@ -554,7 +554,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "Typed into {{target}}",
|
||||
"insertedIn": "Inserted in {{target}}"
|
||||
"insertedIn": "Inserted in {{target}}",
|
||||
"listeningOnPhone": "Listening. Transcribing on this phone.",
|
||||
"listeningOn": "Listening. Transcribing on {{computer}}.",
|
||||
"listeningOnComputer": "Listening. Transcribing on the computer.",
|
||||
"finishing": "Finishing.",
|
||||
"inserted": "Inserted.",
|
||||
"cancelled": "Dictation cancelled.",
|
||||
"movedTo": "This phone stopped transcribing. Continuing on {{computer}}.",
|
||||
"movedToComputer": "This phone stopped transcribing. Continuing on the computer.",
|
||||
"stoppedKept": "Dictation stopped. {{reason}}. Your recording is kept.",
|
||||
"stillListening": "Still listening."
|
||||
},
|
||||
"pill": {
|
||||
"open": "Open",
|
||||
|
||||
@@ -1380,7 +1380,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "Escrito en {{target}}",
|
||||
"insertedIn": "Insertado en {{target}}"
|
||||
"insertedIn": "Insertado en {{target}}",
|
||||
"listeningOnPhone": "Escuchando. Transcribiendo en este teléfono.",
|
||||
"listeningOn": "Escuchando. Transcribiendo en {{computer}}.",
|
||||
"listeningOnComputer": "Escuchando. Transcribiendo en el ordenador.",
|
||||
"finishing": "Terminando.",
|
||||
"inserted": "Insertado.",
|
||||
"cancelled": "Dictado cancelado.",
|
||||
"movedTo": "Este teléfono dejó de transcribir. Continúa en {{computer}}.",
|
||||
"movedToComputer": "Este teléfono dejó de transcribir. Continúa en el ordenador.",
|
||||
"stoppedKept": "Dictado detenido. {{reason}}. Tu grabación se conserva.",
|
||||
"stillListening": "Sigue escuchando."
|
||||
},
|
||||
"pill": {
|
||||
"open": "Abrir",
|
||||
|
||||
@@ -1380,7 +1380,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "Saisi dans {{target}}",
|
||||
"insertedIn": "Inséré dans {{target}}"
|
||||
"insertedIn": "Inséré dans {{target}}",
|
||||
"listeningOnPhone": "Écoute. Transcription sur ce téléphone.",
|
||||
"listeningOn": "Écoute. Transcription sur {{computer}}.",
|
||||
"listeningOnComputer": "Écoute. Transcription sur l’ordinateur.",
|
||||
"finishing": "Finalisation.",
|
||||
"inserted": "Inséré.",
|
||||
"cancelled": "Dictée annulée.",
|
||||
"movedTo": "Ce téléphone a arrêté de transcrire. Suite sur {{computer}}.",
|
||||
"movedToComputer": "Ce téléphone a arrêté de transcrire. Suite sur l’ordinateur.",
|
||||
"stoppedKept": "Dictée arrêtée. {{reason}}. Votre enregistrement est conservé.",
|
||||
"stillListening": "Toujours à l’écoute."
|
||||
},
|
||||
"pill": {
|
||||
"open": "Ouvrir",
|
||||
|
||||
@@ -1335,7 +1335,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "{{target}} に入力しました",
|
||||
"insertedIn": "{{target}} に挿入しました"
|
||||
"insertedIn": "{{target}} に挿入しました",
|
||||
"listeningOnPhone": "聞き取り中。このスマートフォンで文字起こししています。",
|
||||
"listeningOn": "聞き取り中。{{computer}} で文字起こししています。",
|
||||
"listeningOnComputer": "聞き取り中。コンピュータで文字起こししています。",
|
||||
"finishing": "仕上げ中。",
|
||||
"inserted": "挿入しました。",
|
||||
"cancelled": "音声入力をキャンセルしました。",
|
||||
"movedTo": "このスマートフォンでの文字起こしが止まりました。{{computer}} で続けます。",
|
||||
"movedToComputer": "このスマートフォンでの文字起こしが止まりました。コンピュータで続けます。",
|
||||
"stoppedKept": "音声入力が停止しました。{{reason}}。録音は保持されています。",
|
||||
"stillListening": "聞き取りを続けています。"
|
||||
},
|
||||
"pill": {
|
||||
"open": "開く",
|
||||
|
||||
@@ -1335,7 +1335,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "{{target}}에 입력됨",
|
||||
"insertedIn": "{{target}}에 삽입됨"
|
||||
"insertedIn": "{{target}}에 삽입됨",
|
||||
"listeningOnPhone": "듣는 중. 이 휴대폰에서 받아쓰는 중입니다.",
|
||||
"listeningOn": "듣는 중. {{computer}}에서 받아쓰는 중입니다.",
|
||||
"listeningOnComputer": "듣는 중. 컴퓨터에서 받아쓰는 중입니다.",
|
||||
"finishing": "마무리 중.",
|
||||
"inserted": "삽입됨.",
|
||||
"cancelled": "받아쓰기가 취소되었습니다.",
|
||||
"movedTo": "이 휴대폰의 받아쓰기가 멈췄습니다. {{computer}}에서 계속합니다.",
|
||||
"movedToComputer": "이 휴대폰의 받아쓰기가 멈췄습니다. 컴퓨터에서 계속합니다.",
|
||||
"stoppedKept": "받아쓰기가 중지되었습니다. {{reason}}. 녹음은 보관됩니다.",
|
||||
"stillListening": "계속 듣는 중입니다."
|
||||
},
|
||||
"pill": {
|
||||
"open": "열기",
|
||||
|
||||
@@ -1380,7 +1380,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "Digitado em {{target}}",
|
||||
"insertedIn": "Inserido em {{target}}"
|
||||
"insertedIn": "Inserido em {{target}}",
|
||||
"listeningOnPhone": "Ouvindo. Transcrevendo neste telefone.",
|
||||
"listeningOn": "Ouvindo. Transcrevendo em {{computer}}.",
|
||||
"listeningOnComputer": "Ouvindo. Transcrevendo no computador.",
|
||||
"finishing": "Finalizando.",
|
||||
"inserted": "Inserido.",
|
||||
"cancelled": "Ditado cancelado.",
|
||||
"movedTo": "Este telefone parou de transcrever. Continuando em {{computer}}.",
|
||||
"movedToComputer": "Este telefone parou de transcrever. Continuando no computador.",
|
||||
"stoppedKept": "Ditado interrompido. {{reason}}. Sua gravação foi mantida.",
|
||||
"stillListening": "Ainda ouvindo."
|
||||
},
|
||||
"pill": {
|
||||
"open": "Abrir",
|
||||
|
||||
@@ -1421,7 +1421,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "Введено в {{target}}",
|
||||
"insertedIn": "Вставлено в {{target}}"
|
||||
"insertedIn": "Вставлено в {{target}}",
|
||||
"listeningOnPhone": "Слушаю. Распознавание на этом телефоне.",
|
||||
"listeningOn": "Слушаю. Распознавание на {{computer}}.",
|
||||
"listeningOnComputer": "Слушаю. Распознавание на компьютере.",
|
||||
"finishing": "Завершение.",
|
||||
"inserted": "Вставлено.",
|
||||
"cancelled": "Диктовка отменена.",
|
||||
"movedTo": "Этот телефон перестал распознавать. Продолжение на {{computer}}.",
|
||||
"movedToComputer": "Этот телефон перестал распознавать. Продолжение на компьютере.",
|
||||
"stoppedKept": "Диктовка остановлена. {{reason}}. Запись сохранена.",
|
||||
"stillListening": "Продолжаю слушать."
|
||||
},
|
||||
"pill": {
|
||||
"open": "Открыть",
|
||||
|
||||
@@ -1339,7 +1339,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "{{target}} içine yazıldı",
|
||||
"insertedIn": "{{target}} içine eklendi"
|
||||
"insertedIn": "{{target}} içine eklendi",
|
||||
"listeningOnPhone": "Dinleniyor. Bu telefonda yazıya dökülüyor.",
|
||||
"listeningOn": "Dinleniyor. {{computer}} üzerinde yazıya dökülüyor.",
|
||||
"listeningOnComputer": "Dinleniyor. Bilgisayarda yazıya dökülüyor.",
|
||||
"finishing": "Tamamlanıyor.",
|
||||
"inserted": "Eklendi.",
|
||||
"cancelled": "Dikte iptal edildi.",
|
||||
"movedTo": "Bu telefon yazıya dökmeyi bıraktı. {{computer}} üzerinde devam ediliyor.",
|
||||
"movedToComputer": "Bu telefon yazıya dökmeyi bıraktı. Bilgisayarda devam ediliyor.",
|
||||
"stoppedKept": "Dikte durdu. {{reason}}. Kaydınız saklandı.",
|
||||
"stillListening": "Dinlemeye devam ediliyor."
|
||||
},
|
||||
"pill": {
|
||||
"open": "Aç",
|
||||
|
||||
@@ -1335,7 +1335,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "已輸入到 {{target}}",
|
||||
"insertedIn": "已插入到 {{target}}"
|
||||
"insertedIn": "已插入到 {{target}}",
|
||||
"listeningOnPhone": "正在聽。正在此手機上轉寫。",
|
||||
"listeningOn": "正在聽。正在 {{computer}} 上轉寫。",
|
||||
"listeningOnComputer": "正在聽。正在電腦上轉寫。",
|
||||
"finishing": "正在完成。",
|
||||
"inserted": "已插入。",
|
||||
"cancelled": "已取消聽寫。",
|
||||
"movedTo": "此手機已停止轉寫。在 {{computer}} 上繼續。",
|
||||
"movedToComputer": "此手機已停止轉寫。在電腦上繼續。",
|
||||
"stoppedKept": "聽寫已停止。{{reason}}。錄音已保留。",
|
||||
"stillListening": "仍在聽。"
|
||||
},
|
||||
"pill": {
|
||||
"open": "開啟",
|
||||
|
||||
@@ -1335,7 +1335,17 @@
|
||||
},
|
||||
"announce": {
|
||||
"typedInto": "已输入到 {{target}}",
|
||||
"insertedIn": "已插入到 {{target}}"
|
||||
"insertedIn": "已插入到 {{target}}",
|
||||
"listeningOnPhone": "正在听。正在此手机上转写。",
|
||||
"listeningOn": "正在听。正在 {{computer}} 上转写。",
|
||||
"listeningOnComputer": "正在听。正在电脑上转写。",
|
||||
"finishing": "正在完成。",
|
||||
"inserted": "已插入。",
|
||||
"cancelled": "已取消听写。",
|
||||
"movedTo": "此手机已停止转写。在 {{computer}} 上继续。",
|
||||
"movedToComputer": "此手机已停止转写。在电脑上继续。",
|
||||
"stoppedKept": "听写已停止。{{reason}}。录音已保留。",
|
||||
"stillListening": "仍在听。"
|
||||
},
|
||||
"pill": {
|
||||
"open": "打开",
|
||||
|
||||
@@ -41,7 +41,8 @@ export const BRIDGE_NATIVE_VERB_NAMES = [
|
||||
'native.pdf.view',
|
||||
'native.image.act',
|
||||
BRIDGE_SPEECH_VERB,
|
||||
BRIDGE_STORE_VERB
|
||||
BRIDGE_STORE_VERB,
|
||||
'native.a11y.announce'
|
||||
] as const
|
||||
|
||||
export type BridgeNativeVerb = (typeof BRIDGE_NATIVE_VERB_NAMES)[number]
|
||||
@@ -93,6 +94,11 @@ export type BridgePdfOpenParams = Omit<
|
||||
'action'
|
||||
>
|
||||
|
||||
// The page has no announce call of its own (RNW's is a no-op); the shell speaks for it. Long text is a page bug, so it is refused.
|
||||
export const a11yAnnounceParamsSchema = z.strictObject({ message: z.string().min(1).max(500) })
|
||||
// announced:false when the shell leaves it to the page's live region (Android, where TalkBack reads it).
|
||||
export const a11yAnnounceResultSchema = z.strictObject({ announced: z.boolean() })
|
||||
|
||||
export const BRIDGE_NATIVE_VERBS: Readonly<Record<BridgeNativeVerb, BridgeNativeVerbSpec>> = {
|
||||
'native.clipboard.write': {
|
||||
params: clipboardWriteParamsSchema,
|
||||
@@ -150,6 +156,10 @@ export const BRIDGE_NATIVE_VERBS: Readonly<Record<BridgeNativeVerb, BridgeNative
|
||||
[BRIDGE_STORE_VERB]: {
|
||||
params: storeParamsSchema,
|
||||
result: storeResultSchema
|
||||
},
|
||||
'native.a11y.announce': {
|
||||
params: a11yAnnounceParamsSchema,
|
||||
result: a11yAnnounceResultSchema
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
} from './bridge-audio-verbs'
|
||||
import {
|
||||
clipboardReadResultSchema,
|
||||
a11yAnnounceResultSchema,
|
||||
clipboardWriteResultSchema,
|
||||
pdfViewResultSchema,
|
||||
type BridgeClipboardMime,
|
||||
@@ -70,6 +71,10 @@ export type NativeVerbs = {
|
||||
speechRead: () => Promise<BridgeSpeechRead>
|
||||
speechFinish: () => Promise<BridgeSpeechFinish>
|
||||
speechCancel: () => Promise<void>
|
||||
// Older shells lack it; the page then keeps its live region, which VoiceOver may not read.
|
||||
canAnnounce: boolean
|
||||
// True when the shell spoke it.
|
||||
announce: (message: string) => Promise<boolean>
|
||||
}
|
||||
|
||||
// Expose stable reason codes, including local ungranted and frame refusals. Unknown newer-shell codes floor to unreported so caller switches remain exhaustive.
|
||||
@@ -204,7 +209,10 @@ export function useNativeVerbs(): NativeVerbs {
|
||||
speechFinish: () => call(BRIDGE_SPEECH_VERB, { action: 'finish' }, speechFinishResultSchema),
|
||||
speechCancel: async () => {
|
||||
await call(BRIDGE_SPEECH_VERB, { action: 'cancel' }, speechCancelResultSchema)
|
||||
}
|
||||
},
|
||||
canAnnounce: has('native.a11y.announce'),
|
||||
announce: async (message) =>
|
||||
(await call('native.a11y.announce', { message }, a11yAnnounceResultSchema)).announced
|
||||
}
|
||||
}, [client])
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { useCallback } from 'react'
|
||||
import { AccessibilityInfo } from 'react-native'
|
||||
|
||||
// Speak a message to the screen reader; false when the caller should put it in a live region instead.
|
||||
export function useAccessibilityAnnouncer(): (message: string) => Promise<boolean> {
|
||||
return useCallback(async (message: string) => {
|
||||
AccessibilityInfo.announceForAccessibility(message)
|
||||
return true
|
||||
}, [])
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { useCallback } from 'react'
|
||||
import { useNativeVerbs } from '../mobile-web-shell/bridge/use-native-verbs'
|
||||
|
||||
// RNW's announceForAccessibility does nothing, so the shell announces; older shells and Android leave it to the page's live region.
|
||||
export function useAccessibilityAnnouncer(): (message: string) => Promise<boolean> {
|
||||
const verbs = useNativeVerbs()
|
||||
return useCallback(
|
||||
async (message: string) => {
|
||||
if (!verbs.canAnnounce) {
|
||||
return false
|
||||
}
|
||||
return verbs.announce(message).catch(() => false)
|
||||
},
|
||||
[verbs]
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { AccessibilityInfo, Platform } from 'react-native'
|
||||
import { a11yAnnounceParamsSchema } from '../mobile-web-shell/bridge/bridge-native-verbs'
|
||||
|
||||
// VoiceOver ignores the page's live regions inside the shell's WebView, so the shell announces for it; TalkBack reads them, so Android leaves it to the page.
|
||||
export function serveNativeAnnounceVerb(params: unknown): { announced: boolean } {
|
||||
const { message } = a11yAnnounceParamsSchema.parse(params)
|
||||
if (Platform.OS !== 'ios') {
|
||||
return { announced: false }
|
||||
}
|
||||
AccessibilityInfo.announceForAccessibilityWithOptions(message, { queue: true })
|
||||
return { announced: true }
|
||||
}
|
||||
@@ -17,6 +17,11 @@ const device = vi.hoisted(() => ({
|
||||
wakeTags: new Array<string>()
|
||||
}))
|
||||
|
||||
// The shell's announce verb speaks through React Native's AccessibilityInfo.
|
||||
vi.mock('react-native', () => ({
|
||||
AccessibilityInfo: { announceForAccessibilityWithOptions: () => {} },
|
||||
Platform: { OS: 'ios' }
|
||||
}))
|
||||
vi.mock('expo-clipboard', () => ({
|
||||
setStringAsync: () => Promise.resolve(true),
|
||||
getStringAsync: device.clipboard.getStringAsync,
|
||||
|
||||
@@ -7,6 +7,7 @@ import { NATIVE_AUDIO_DEVICE_SAMPLE_RATE, nativeAudioDeviceEngine } from './nati
|
||||
import { createNativeSpeechServer } from './native-speech'
|
||||
import { nativeSpeechDeviceEngine } from './native-speech-device'
|
||||
import { serveNativeClipboardVerb } from './native-clipboard'
|
||||
import { serveNativeAnnounceVerb } from './native-accessibility-announce'
|
||||
import { createNativeMediaVerbServer } from './native-media'
|
||||
import { discardStagedMedia, nativeMediaDeviceDeps } from './native-media-device'
|
||||
|
||||
@@ -37,6 +38,9 @@ export function useNativeDeviceVerbs(
|
||||
if (verb === 'native.speech.act') {
|
||||
return speech.serve(params)
|
||||
}
|
||||
if (verb === 'native.a11y.announce') {
|
||||
return Promise.resolve(serveNativeAnnounceVerb(params))
|
||||
}
|
||||
if (verb === 'native.audio.start') {
|
||||
return speech.mayCapture().then((may) =>
|
||||
may
|
||||
|
||||
@@ -9,7 +9,6 @@ import { styles } from './mobile-session-styles'
|
||||
import type { MobileSessionController } from './use-mobile-session-controller'
|
||||
import { MobileSessionActiveContent } from './MobileSessionActiveContent'
|
||||
import { MobileSessionCommandDock } from './MobileSessionCommandDock'
|
||||
import { MobileDictationStatusAnnouncement } from '../dictation/MobileDictationStatusAnnouncement'
|
||||
import { MobileDictationLiveContext } from '../dictation/mobile-dictation-live-context'
|
||||
import { useMobileDictationLive } from '../dictation/use-mobile-dictation-live'
|
||||
|
||||
@@ -56,7 +55,6 @@ export function MobileSessionContentRow({ controller }: { controller: MobileSess
|
||||
{/* Why: translate instead of resize so keyboard toggles don't trigger a server-side PTY viewport change. */}
|
||||
<MobileSessionCommandDock controller={controller} />
|
||||
</MobileDictationLiveContext.Provider>
|
||||
<MobileDictationStatusAnnouncement dictation={dictation} />
|
||||
</View>
|
||||
{canDockPanel && activePanel !== null && (
|
||||
<SessionDockColumn
|
||||
|
||||
@@ -26,8 +26,11 @@ const seen = vi.hoisted(() => {
|
||||
|
||||
vi.mock('react-native', () => ({
|
||||
AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) },
|
||||
AccessibilityInfo: { announceForAccessibility: () => {} },
|
||||
Platform: { OS: 'ios' }
|
||||
}))
|
||||
// The computer's name only words announcements; the stored host profile pulls in secure storage.
|
||||
vi.mock('../transport/hosts/use-host-profile', () => ({ useHostProfile: () => null }))
|
||||
vi.mock('@orca/expo-two-way-audio', () => ({
|
||||
addExpoTwoWayAudioEventListener: (name: string, listener: (event: { data: string }) => void) => {
|
||||
seen.audioListeners.set(name, listener)
|
||||
|
||||
@@ -22,7 +22,9 @@ import {
|
||||
import { useMobileDictationLeaveHold } from '../dictation/mobile-dictation-leave-hold'
|
||||
import { useMobileDictationHost } from '../dictation/mobile-dictation-host'
|
||||
import { mobileDictationShownOnScreen } from '../dictation/mobile-dictation-shown'
|
||||
import { useDictationComputerName } from '../dictation/use-mobile-dictation-live'
|
||||
import { nativeChatDictationCopy } from '../dictation/mobile-session-native-chat-dictation-copy'
|
||||
import { dictationAnnounceCopy } from '../dictation/mobile-dictation-announce-copy'
|
||||
import {
|
||||
useWorkspaceExecutionHost,
|
||||
workspaceRouteHref
|
||||
@@ -54,6 +56,7 @@ export function useMobileSessionDictation(
|
||||
} = scope
|
||||
const { showNativeChat, showNativeChatRef } = nativeChatController
|
||||
const host = useMobileDictationHost()
|
||||
const computerName = useDictationComputerName(hostId)
|
||||
const { start: startHost } = host
|
||||
const executionHost = useWorkspaceExecutionHost()
|
||||
const sessionKey = JSON.stringify([hostId, executionHost ?? null, worktreeId])
|
||||
@@ -93,12 +96,17 @@ export function useMobileSessionDictation(
|
||||
)
|
||||
|
||||
const clipboard = useClipboardWriter()
|
||||
// The screen's toast is not read on the page, so where the words went is also spoken.
|
||||
const tell = (message: string) => {
|
||||
showToast(message)
|
||||
host.announce(message)
|
||||
}
|
||||
// A transcript its target refused must not vanish: keep it on the clipboard and say so.
|
||||
const keepUndeliveredDictation = (text: string) => {
|
||||
triggerError()
|
||||
void clipboard.writeText(text).then(
|
||||
() => showToast(nativeChatDictationCopy.insertFailedCopied()),
|
||||
() => showToast(nativeChatDictationCopy.insertFailed())
|
||||
() => tell(nativeChatDictationCopy.insertFailedCopied()),
|
||||
() => tell(nativeChatDictationCopy.insertFailed())
|
||||
)
|
||||
}
|
||||
|
||||
@@ -106,7 +114,9 @@ export function useMobileSessionDictation(
|
||||
// Chat dictation is inserted locally; a terminal's words are typed into it live, while it is the active one.
|
||||
const insertHere = (text: string): boolean => {
|
||||
if (destination.landsInChat(showNativeChatRef.current)) {
|
||||
if (!destination.insertInChat(text)) {
|
||||
if (destination.insertInChat(text)) {
|
||||
host.announce(dictationAnnounceCopy.inserted())
|
||||
} else {
|
||||
keepUndeliveredDictation(text)
|
||||
}
|
||||
return true
|
||||
@@ -124,7 +134,7 @@ export function useMobileSessionDictation(
|
||||
(await sendLiveTerminalInput(insertHandle, text))
|
||||
// Live input lands in a terminal the keyboard may cover, so say where the words went.
|
||||
if (sent) {
|
||||
showToast(nativeChatDictationCopy.inserted())
|
||||
tell(nativeChatDictationCopy.inserted())
|
||||
} else {
|
||||
keepUndeliveredDictation(text)
|
||||
}
|
||||
@@ -237,9 +247,11 @@ export function useMobileSessionDictation(
|
||||
dictationMode: scope.dictationMode,
|
||||
removeFillerWords,
|
||||
dictationTerms,
|
||||
worktreeId
|
||||
worktreeId,
|
||||
computerName
|
||||
})
|
||||
}, [
|
||||
computerName,
|
||||
activeHandle,
|
||||
activeSessionTab,
|
||||
busyElsewhere,
|
||||
|
||||
@@ -244,6 +244,10 @@
|
||||
{
|
||||
"file": "src/platform/page-box.web.ts",
|
||||
"reason": "RNW's measure() is asynchronous and drops its callback for a detached node, so a held mic released over the dictation cancel button would never stop. The page reads the DOM box synchronously; native keeps measure()."
|
||||
},
|
||||
{
|
||||
"file": "src/platform/accessibility-announcer.web.ts",
|
||||
"reason": "React Native Web's announceForAccessibility is a no-op and VoiceOver in the shell's WebView doesn't read the page's live regions, so the page asks the shell to announce through native.a11y.announce; without it, the caller keeps its live region."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user