mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
* feat(mobile): define the on-device speech contract for the native audio module and page bridge * feat(mobile): dictate on the phone first and replay to the computer when the phone fails * refactor(mobile): type the on-device speech module without a cast * fix(mobile): point dictation's finish ref at the committed callback * test(mobile): wait on the takeover request instead of a fixed sleep * fix(mobile): release each dictation request's own share of a replayed backlog * fix(mobile): let the computer's text replace the phone's after a fallback
62 lines
2.5 KiB
TypeScript
62 lines
2.5 KiB
TypeScript
import {
|
|
addExpoTwoWayAudioEventListener,
|
|
initialize,
|
|
requestMicrophonePermissionsAsync,
|
|
speechFeed,
|
|
tearDown,
|
|
toggleRecording
|
|
} from '@orca/expo-two-way-audio'
|
|
import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake'
|
|
import { BRIDGE_AUDIO_INTERRUPTIONS } from '../mobile-web-shell/bridge/bridge-audio-verbs'
|
|
import { createMicrophoneScreenLock } from './microphone-screen-lock'
|
|
import type { NativeAudioEngine } from './native-audio'
|
|
|
|
// Isolate device imports so tests can inject engines without loading a device-only JSI binding.
|
|
|
|
// initialize() accepts no rate: both engines produce fixed 16 kHz, so report the actual rate instead of silently accepting another.
|
|
export const NATIVE_AUDIO_DEVICE_SAMPLE_RATE = 16_000
|
|
|
|
const MICROPHONE_SCREEN_LOCK_TAG = 'orca-microphone'
|
|
|
|
// Share one wake lock for the one microphone; native and bridge capture must not use independently releasable tags.
|
|
export const nativeMicrophoneScreenLock = createMicrophoneScreenLock(
|
|
{ activate: activateKeepAwakeAsync, deactivate: deactivateKeepAwake },
|
|
MICROPHONE_SCREEN_LOCK_TAG
|
|
)
|
|
|
|
function readInterruption(data: string): (typeof BRIDGE_AUDIO_INTERRUPTIONS)[number] | null {
|
|
return BRIDGE_AUDIO_INTERRUPTIONS.find((kind) => kind === data) ?? null
|
|
}
|
|
|
|
export const nativeAudioDeviceEngine: NativeAudioEngine = {
|
|
requestPermission: async () => {
|
|
const permission = await requestMicrophonePermissionsAsync()
|
|
// Permission refusal has the same page outcome whether or not the OS allows another prompt.
|
|
return permission.granted ? 'granted' : permission.canAskAgain ? 'undetermined' : 'denied'
|
|
},
|
|
open: async (sampleRate) => ({
|
|
opened: sampleRate === NATIVE_AUDIO_DEVICE_SAMPLE_RATE && (await initialize()),
|
|
sampleRate: NATIVE_AUDIO_DEVICE_SAMPLE_RATE
|
|
}),
|
|
begin: () => toggleRecording(true),
|
|
end: () => {
|
|
toggleRecording(false)
|
|
tearDown()
|
|
},
|
|
screenLock: nativeMicrophoneScreenLock,
|
|
onMicrophoneData: (handler) =>
|
|
addExpoTwoWayAudioEventListener('onMicrophoneData', (event) => {
|
|
const raw = event.data
|
|
handler(raw instanceof Uint8Array ? raw : new Uint8Array(raw))
|
|
}),
|
|
feedSpeech: (bytes) => speechFeed(bytes),
|
|
onInterruption: (handler) =>
|
|
addExpoTwoWayAudioEventListener('onAudioInterruption', (event) => {
|
|
// Ignore unknown engine interruption kinds instead of forwarding a value the page cannot describe.
|
|
const kind = readInterruption(event.data)
|
|
if (kind !== null) {
|
|
handler(kind)
|
|
}
|
|
})
|
|
}
|