mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
* feat(mobile): announce dictation to VoiceOver through the shell * feat(mobile): speak dictation from the host, so leaving the session and the pill's toasts are announced too * fix(mobile): say dictation outcomes where they are known, and re-read a repeated live message * fix(mobile): leave a setup refusal to its setup sheet instead of reading the raw code * fix(mobile): say cancelled when a kept recording is discarded
60 lines
2.7 KiB
TypeScript
60 lines
2.7 KiB
TypeScript
import { useEffect, useMemo } from 'react'
|
|
import type { BridgeNativeVerb } from '../mobile-web-shell/bridge/bridge-native-verbs'
|
|
import type { BRIDGE_STORE_VERB } from '../mobile-web-shell/bridge/bridge-store-verb'
|
|
import { useMediaHandleRegistry } from '../mobile-web-shell/use-media-handle-registry'
|
|
import { createNativeAudioCapture } from './native-audio'
|
|
import { NATIVE_AUDIO_DEVICE_SAMPLE_RATE, nativeAudioDeviceEngine } from './native-audio-device'
|
|
import { createNativeSpeechServer } from './native-speech'
|
|
import { nativeSpeechDeviceEngine } from './native-speech-device'
|
|
import { serveNativeClipboardVerb } from './native-clipboard'
|
|
import { serveNativeAnnounceVerb } from './native-accessibility-announce'
|
|
import { createNativeMediaVerbServer } from './native-media'
|
|
import { discardStagedMedia, nativeMediaDeviceDeps } from './native-media-device'
|
|
|
|
export type NativeDeviceVerb = Exclude<
|
|
BridgeNativeVerb,
|
|
'native.storage.read' | 'native.pdf.view' | 'native.image.act' | typeof BRIDGE_STORE_VERB
|
|
>
|
|
|
|
// Tie stateful audio/media handlers to the page session so replacement documents cannot inherit abandoned microphones or staged handles.
|
|
export function useNativeDeviceVerbs(
|
|
sessionId: string | null
|
|
): (verb: NativeDeviceVerb, params: unknown) => Promise<unknown> {
|
|
const registry = useMediaHandleRegistry({ sessionId, discard: discardStagedMedia })
|
|
const serveMedia = useMemo(
|
|
() => createNativeMediaVerbServer(nativeMediaDeviceDeps(registry)),
|
|
[registry]
|
|
)
|
|
// A new session is a new document; close any mic its predecessor left open.
|
|
const audio = useMemo(() => createNativeAudioCapture(nativeAudioDeviceEngine), [sessionId])
|
|
useEffect(() => () => audio.dispose(), [audio])
|
|
const speech = useMemo(() => createNativeSpeechServer(nativeSpeechDeviceEngine), [sessionId])
|
|
useEffect(() => () => speech.dispose(), [speech])
|
|
return useMemo(
|
|
() => (verb, params) => {
|
|
if (verb === 'native.clipboard.write' || verb === 'native.clipboard.read') {
|
|
return serveNativeClipboardVerb(verb, params)
|
|
}
|
|
if (verb === 'native.speech.act') {
|
|
return speech.serve(params)
|
|
}
|
|
if (verb === 'native.a11y.announce') {
|
|
return Promise.resolve(serveNativeAnnounceVerb(params))
|
|
}
|
|
if (verb === 'native.audio.start') {
|
|
return speech.mayCapture().then((may) =>
|
|
may
|
|
? audio.serve(verb, params)
|
|
: {
|
|
started: false,
|
|
sampleRate: NATIVE_AUDIO_DEVICE_SAMPLE_RATE,
|
|
permission: 'granted'
|
|
}
|
|
)
|
|
}
|
|
return verb.startsWith('native.audio.') ? audio.serve(verb, params) : serveMedia(verb, params)
|
|
},
|
|
[audio, serveMedia, speech]
|
|
)
|
|
}
|