Files
archived-orca/mobile/src/platform/native-audio-device.ts
Kelvin Amoaba ffaf887e78 feat(mobile): transcribe dictation on the phone, with the computer as fallback (#39386)
* feat(mobile): define the on-device speech contract for the native audio module and page bridge

* feat(mobile): dictate on the phone first and replay to the computer when the phone fails

* refactor(mobile): type the on-device speech module without a cast

* fix(mobile): point dictation's finish ref at the committed callback

* test(mobile): wait on the takeover request instead of a fixed sleep

* fix(mobile): release each dictation request's own share of a replayed backlog

* fix(mobile): let the computer's text replace the phone's after a fallback
2026-11-07 09:37:45 +00:00

62 lines
2.5 KiB
TypeScript

import {
addExpoTwoWayAudioEventListener,
initialize,
requestMicrophonePermissionsAsync,
speechFeed,
tearDown,
toggleRecording
} from '@orca/expo-two-way-audio'
import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake'
import { BRIDGE_AUDIO_INTERRUPTIONS } from '../mobile-web-shell/bridge/bridge-audio-verbs'
import { createMicrophoneScreenLock } from './microphone-screen-lock'
import type { NativeAudioEngine } from './native-audio'
// Isolate device imports so tests can inject engines without loading a device-only JSI binding.
// initialize() accepts no rate: both engines produce fixed 16 kHz, so report the actual rate instead of silently accepting another.
export const NATIVE_AUDIO_DEVICE_SAMPLE_RATE = 16_000
const MICROPHONE_SCREEN_LOCK_TAG = 'orca-microphone'
// Share one wake lock for the one microphone; native and bridge capture must not use independently releasable tags.
export const nativeMicrophoneScreenLock = createMicrophoneScreenLock(
{ activate: activateKeepAwakeAsync, deactivate: deactivateKeepAwake },
MICROPHONE_SCREEN_LOCK_TAG
)
function readInterruption(data: string): (typeof BRIDGE_AUDIO_INTERRUPTIONS)[number] | null {
return BRIDGE_AUDIO_INTERRUPTIONS.find((kind) => kind === data) ?? null
}
export const nativeAudioDeviceEngine: NativeAudioEngine = {
requestPermission: async () => {
const permission = await requestMicrophonePermissionsAsync()
// Permission refusal has the same page outcome whether or not the OS allows another prompt.
return permission.granted ? 'granted' : permission.canAskAgain ? 'undetermined' : 'denied'
},
open: async (sampleRate) => ({
opened: sampleRate === NATIVE_AUDIO_DEVICE_SAMPLE_RATE && (await initialize()),
sampleRate: NATIVE_AUDIO_DEVICE_SAMPLE_RATE
}),
begin: () => toggleRecording(true),
end: () => {
toggleRecording(false)
tearDown()
},
screenLock: nativeMicrophoneScreenLock,
onMicrophoneData: (handler) =>
addExpoTwoWayAudioEventListener('onMicrophoneData', (event) => {
const raw = event.data
handler(raw instanceof Uint8Array ? raw : new Uint8Array(raw))
}),
feedSpeech: (bytes) => speechFeed(bytes),
onInterruption: (handler) =>
addExpoTwoWayAudioEventListener('onAudioInterruption', (event) => {
// Ignore unknown engine interruption kinds instead of forwarding a value the page cannot describe.
const kind = readInterruption(event.data)
if (kind !== null) {
handler(kind)
}
})
}