mirror of
https://github.com/stablyai/orca.git
synced 2026-10-17 08:03:14 +00:00
Prevent mobile screen locking during voice dictation
Integrate expo-keep-awake to prevent the mobile device from locking or sleeping while a voice dictation session is active. - Modularize useMobileDictation logic into separate helper files for keep-awake, audio chunking, session state, and desktop startup. - Acquire keep-awake lock only after successfully establishing a desktop session to avoid locking on stale start attempts. - Release the keep-awake lock on all completion, cancellation, error, and unmount paths. - Add source invariant unit tests to verify keep-awake ownership and strict cleanup ordering.
This commit is contained in:
@@ -35,6 +35,7 @@
|
||||
"expo-haptics": "^55.0.14",
|
||||
"expo-image-manipulator": "^55.0.17",
|
||||
"expo-image-picker": "^55.0.20",
|
||||
"expo-keep-awake": "55.0.8",
|
||||
"expo-linking": "^55.0.15",
|
||||
"expo-modules-core": "~55.0.25",
|
||||
"expo-network": "~55.0.14",
|
||||
|
||||
Generated
+3
@@ -65,6 +65,9 @@ importers:
|
||||
expo-image-picker:
|
||||
specifier: ^55.0.20
|
||||
version: 55.0.20(expo@55.0.27)
|
||||
expo-keep-awake:
|
||||
specifier: 55.0.8
|
||||
version: 55.0.8(expo@55.0.27)(react@19.2.6)
|
||||
expo-linking:
|
||||
specifier: ^55.0.15
|
||||
version: 55.0.15(expo@55.0.27)(react-native@0.83.9(@babel/core@7.29.7)(@react-native/metro-config@0.85.2(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6)
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import {
|
||||
MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE,
|
||||
MOBILE_DICTATION_PCM_SAMPLE_RATE
|
||||
} from './mobile-dictation-pending-audio-budget'
|
||||
import { bytesToBase64 } from './mobile-dictation-session-state'
|
||||
import type { MicrophoneDataEvent } from '@orca/expo-two-way-audio'
|
||||
import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
|
||||
type MobileDictationAudioChunkQueue = {
|
||||
pendingChunks: Set<Promise<void>>
|
||||
pendingAudioBudget: MobileDictationPendingAudioBudget
|
||||
shouldReleaseBudget: (dictationId: string) => boolean
|
||||
failActiveDictation: (dictationId: string, err: unknown) => void
|
||||
}
|
||||
|
||||
export function enqueueMobileDictationAudioChunk(
|
||||
client: RpcClient,
|
||||
dictationId: string,
|
||||
event: MicrophoneDataEvent,
|
||||
queue: MobileDictationAudioChunkQueue
|
||||
): void {
|
||||
const raw = event.data
|
||||
const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw)
|
||||
const byteLength = bytes.byteLength
|
||||
if (!queue.pendingAudioBudget.tryReserve(byteLength)) {
|
||||
queue.failActiveDictation(
|
||||
dictationId,
|
||||
new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE)
|
||||
)
|
||||
return
|
||||
}
|
||||
const sendChunk = client
|
||||
.sendRequest('speech.dictation.chunk', {
|
||||
dictationId,
|
||||
audioBase64: bytesToBase64(bytes),
|
||||
sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE
|
||||
})
|
||||
.then((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
})
|
||||
.catch((err) => queue.failActiveDictation(dictationId, err))
|
||||
.finally(() => {
|
||||
if (queue.shouldReleaseBudget(dictationId)) {
|
||||
queue.pendingAudioBudget.release(byteLength)
|
||||
}
|
||||
queue.pendingChunks.delete(sendChunk)
|
||||
})
|
||||
queue.pendingChunks.add(sendChunk)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { isCurrentMobileDictationStart } from './mobile-dictation-session-state'
|
||||
import type { MobileDictationKeepAwakeOwner } from './mobile-dictation-keep-awake'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
|
||||
type StartMobileDictationDesktopSessionOptions = {
|
||||
client: RpcClient
|
||||
dictationId: string
|
||||
generation: number
|
||||
getCurrentGeneration: () => number
|
||||
getEnabled: () => boolean
|
||||
getActiveId: () => string | null
|
||||
clearActiveId: (dictationId: string) => void
|
||||
setIdle: () => void
|
||||
keepAwakeOwner: MobileDictationKeepAwakeOwner
|
||||
}
|
||||
|
||||
function isCurrentStart(options: StartMobileDictationDesktopSessionOptions): boolean {
|
||||
return isCurrentMobileDictationStart(
|
||||
options.getCurrentGeneration(),
|
||||
options.generation,
|
||||
options.getEnabled(),
|
||||
options.getActiveId(),
|
||||
options.dictationId
|
||||
)
|
||||
}
|
||||
|
||||
export async function startMobileDictationDesktopSession(
|
||||
options: StartMobileDictationDesktopSessionOptions
|
||||
): Promise<boolean> {
|
||||
const { client, dictationId, keepAwakeOwner, setIdle } = options
|
||||
|
||||
try {
|
||||
const response = await client.sendRequest('speech.dictation.start', { dictationId })
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
} catch (err) {
|
||||
options.clearActiveId(dictationId)
|
||||
await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
setIdle()
|
||||
throw err
|
||||
}
|
||||
|
||||
if (!isCurrentStart(options)) {
|
||||
await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
options.clearActiveId(dictationId)
|
||||
setIdle()
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
// Keep-awake is acquired only after the desktop session exists, so stale
|
||||
// mobile starts can be canceled without holding a screen-lock tag.
|
||||
await keepAwakeOwner.acquire(dictationId)
|
||||
} catch (err) {
|
||||
if (!isCurrentStart(options)) {
|
||||
return false
|
||||
}
|
||||
options.clearActiveId(dictationId)
|
||||
await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
setIdle()
|
||||
throw err
|
||||
}
|
||||
|
||||
if (!isCurrentStart(options)) {
|
||||
await keepAwakeOwner.release(dictationId).catch(() => undefined)
|
||||
await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
options.clearActiveId(dictationId)
|
||||
setIdle()
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake'
|
||||
|
||||
const MOBILE_DICTATION_KEEP_AWAKE_TAG_PREFIX = 'orca-mobile-dictation'
|
||||
|
||||
let nextOwnerId = 0
|
||||
|
||||
function createOwnerId(): string {
|
||||
nextOwnerId += 1
|
||||
return `${Date.now()}-${nextOwnerId}-${Math.random().toString(36).slice(2)}`
|
||||
}
|
||||
|
||||
export class MobileDictationKeepAwakeOwner {
|
||||
private readonly ownerId = createOwnerId()
|
||||
private acquiredTag: string | null = null
|
||||
private operation: Promise<void> = Promise.resolve()
|
||||
|
||||
acquire(dictationId: string): Promise<void> {
|
||||
const tag = this.createTag(dictationId)
|
||||
const operation = this.operation.then(async () => {
|
||||
if (this.acquiredTag === tag) {
|
||||
return
|
||||
}
|
||||
if (this.acquiredTag) {
|
||||
const previousTag = this.acquiredTag
|
||||
this.acquiredTag = null
|
||||
await deactivateKeepAwake(previousTag).catch(() => undefined)
|
||||
}
|
||||
await activateKeepAwakeAsync(tag)
|
||||
this.acquiredTag = tag
|
||||
})
|
||||
this.operation = operation.catch(() => undefined)
|
||||
return operation
|
||||
}
|
||||
|
||||
release(dictationId?: string): Promise<void> {
|
||||
const targetTag = dictationId ? this.createTag(dictationId) : null
|
||||
const operation = this.operation.then(async () => {
|
||||
const tag = this.acquiredTag
|
||||
if (!tag || (targetTag && tag !== targetTag)) {
|
||||
return
|
||||
}
|
||||
this.acquiredTag = null
|
||||
await deactivateKeepAwake(tag)
|
||||
})
|
||||
this.operation = operation.catch(() => undefined)
|
||||
return operation
|
||||
}
|
||||
|
||||
private createTag(dictationId: string): string {
|
||||
return `${MOBILE_DICTATION_KEEP_AWAKE_TAG_PREFIX}:${this.ownerId}:${dictationId}`
|
||||
}
|
||||
}
|
||||
|
||||
export function createMobileDictationKeepAwakeOwner(): MobileDictationKeepAwakeOwner {
|
||||
return new MobileDictationKeepAwakeOwner()
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
// Why: import from 'buffer' (the npm polyfill), not 'node:buffer' because
|
||||
// Metro cannot resolve Node builtins in a React Native bundle.
|
||||
import { Buffer } from 'buffer'
|
||||
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
|
||||
export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error'
|
||||
|
||||
export type UseMobileDictationOptions = {
|
||||
client: RpcClient | null
|
||||
enabled: boolean
|
||||
onTranscript: (text: string) => void
|
||||
onError?: (error: Error) => void
|
||||
}
|
||||
|
||||
export type UseMobileDictationResult = {
|
||||
status: DictationStatus
|
||||
isStarting: boolean
|
||||
isRecording: boolean
|
||||
isProcessing: boolean
|
||||
error: string | null
|
||||
start: () => Promise<void>
|
||||
stop: () => Promise<void>
|
||||
cancel: () => Promise<void>
|
||||
}
|
||||
|
||||
export const DICTATION_FINISH_TIMEOUT_MS = 75_000
|
||||
|
||||
export function bytesToBase64(bytes: Uint8Array): string {
|
||||
return Buffer.from(bytes).toString('base64')
|
||||
}
|
||||
|
||||
export function createMobileDictationId(): string {
|
||||
return `mobile-dictation-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
||||
}
|
||||
|
||||
export function isCurrentMobileDictationStart(
|
||||
currentGeneration: number,
|
||||
generation: number,
|
||||
enabled: boolean,
|
||||
activeId: string | null,
|
||||
dictationId: string
|
||||
): boolean {
|
||||
return currentGeneration === generation && enabled && activeId === dictationId
|
||||
}
|
||||
|
||||
export function isCurrentMobileDictationFinish(
|
||||
currentGeneration: number,
|
||||
generation: number,
|
||||
enabled: boolean,
|
||||
activeId: string | null,
|
||||
finishingId: string | null,
|
||||
dictationId: string
|
||||
): boolean {
|
||||
return (
|
||||
currentGeneration === generation &&
|
||||
enabled &&
|
||||
activeId === dictationId &&
|
||||
finishingId === dictationId
|
||||
)
|
||||
}
|
||||
@@ -2,13 +2,37 @@ import { readFileSync } from 'node:fs'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
const source = readFileSync(new URL('./use-mobile-dictation.ts', import.meta.url), 'utf8')
|
||||
const audioChunkSource = readFileSync(
|
||||
new URL('./mobile-dictation-audio-chunk.ts', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const keepAwakeSource = readFileSync(
|
||||
new URL('./mobile-dictation-keep-awake.ts', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const desktopStartSource = readFileSync(
|
||||
new URL('./mobile-dictation-desktop-start.ts', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
const sessionStateSource = readFileSync(
|
||||
new URL('./mobile-dictation-session-state.ts', import.meta.url),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
function sliceSource(sourceText: string, startPattern: string, endPattern: string): string {
|
||||
const start = sourceText.indexOf(startPattern)
|
||||
expect(start).toBeGreaterThanOrEqual(0)
|
||||
const end = sourceText.indexOf(endPattern, start)
|
||||
expect(end).toBeGreaterThan(start)
|
||||
return sourceText.slice(start, end)
|
||||
}
|
||||
|
||||
function sliceBetween(startPattern: string, endPattern: string): string {
|
||||
const start = source.indexOf(startPattern)
|
||||
expect(start).toBeGreaterThanOrEqual(0)
|
||||
const end = source.indexOf(endPattern, start)
|
||||
expect(end).toBeGreaterThan(start)
|
||||
return source.slice(start, end)
|
||||
return sliceSource(source, startPattern, endPattern)
|
||||
}
|
||||
|
||||
function sliceDesktopStartBetween(startPattern: string, endPattern: string): string {
|
||||
return sliceSource(desktopStartSource, startPattern, endPattern)
|
||||
}
|
||||
|
||||
describe('useMobileDictation source invariants', () => {
|
||||
@@ -28,9 +52,10 @@ describe('useMobileDictation source invariants', () => {
|
||||
})
|
||||
|
||||
it('reserves pending audio bytes before encoding microphone chunks', () => {
|
||||
const microphoneEffect = sliceBetween(
|
||||
"addExpoTwoWayAudioEventListener('onMicrophoneData'",
|
||||
'return () => sub.remove()'
|
||||
const microphoneEffect = sliceSource(
|
||||
audioChunkSource,
|
||||
'export function enqueueMobileDictationAudioChunk',
|
||||
' queue.pendingChunks.add(sendChunk)'
|
||||
)
|
||||
|
||||
const reserveIndex = microphoneEffect.indexOf('tryReserve(byteLength)')
|
||||
@@ -39,7 +64,8 @@ describe('useMobileDictation source invariants', () => {
|
||||
expect(encodeIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(reserveIndex).toBeLessThan(encodeIndex)
|
||||
expect(microphoneEffect).toContain('MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE')
|
||||
expect(microphoneEffect).toContain('pendingAudioBudgetRef.current.release(byteLength)')
|
||||
expect(microphoneEffect).toContain('queue.pendingAudioBudget.release(byteLength)')
|
||||
expect(source).toContain('enqueueMobileDictationAudioChunk(client, dictationId, event')
|
||||
})
|
||||
|
||||
it('resets pending audio bytes whenever pending chunk tracking is cleared', () => {
|
||||
@@ -48,4 +74,147 @@ describe('useMobileDictation source invariants', () => {
|
||||
|
||||
expect(pendingAudioResets).toHaveLength(pendingChunkClears.length)
|
||||
})
|
||||
|
||||
it('keeps mobile dictation keep-awake ownership beside the hook', () => {
|
||||
expect(source).toContain(
|
||||
"import { createMobileDictationKeepAwakeOwner } from './mobile-dictation-keep-awake'"
|
||||
)
|
||||
expect(source).toContain(
|
||||
'const keepAwakeOwner = useMemo(createMobileDictationKeepAwakeOwner, [])'
|
||||
)
|
||||
expect(keepAwakeSource).toContain('activateKeepAwakeAsync')
|
||||
expect(keepAwakeSource).toContain('deactivateKeepAwake')
|
||||
expect(keepAwakeSource).not.toMatch(/\bactivateKeepAwake\s*\(/)
|
||||
})
|
||||
|
||||
it('acquires keep-awake only after desktop start and stale-start guards', () => {
|
||||
const hookStartBody = sliceBetween('const start = useCallback(async () => {', 'const stop =')
|
||||
const startBody = sliceDesktopStartBetween(
|
||||
'export async function startMobileDictationDesktopSession',
|
||||
' return true'
|
||||
)
|
||||
const desktopStartIndex = startBody.indexOf(
|
||||
"client.sendRequest('speech.dictation.start', { dictationId })"
|
||||
)
|
||||
const acquireIndex = startBody.indexOf('await keepAwakeOwner.acquire(dictationId)')
|
||||
const desktopSessionIndex = hookStartBody.indexOf('await startMobileDictationDesktopSession')
|
||||
const toggleRecordingIndex = hookStartBody.indexOf('toggleRecording(true)')
|
||||
|
||||
expect(desktopStartIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(acquireIndex).toBeGreaterThan(desktopStartIndex)
|
||||
expect(desktopSessionIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(toggleRecordingIndex).toBeGreaterThan(desktopSessionIndex)
|
||||
|
||||
const beforeAcquire = startBody.slice(desktopStartIndex, acquireIndex)
|
||||
expect(beforeAcquire).toContain('isCurrentStart(options)')
|
||||
expect(desktopStartSource).toContain('options.getCurrentGeneration()')
|
||||
expect(desktopStartSource).toContain('options.getEnabled()')
|
||||
expect(desktopStartSource).toContain('options.getActiveId()')
|
||||
expect(sessionStateSource).toContain(
|
||||
'currentGeneration === generation && enabled && activeId === dictationId'
|
||||
)
|
||||
})
|
||||
|
||||
it('re-checks stale-start guards after awaited keep-awake acquisition', () => {
|
||||
const startBody = sliceDesktopStartBetween(
|
||||
'export async function startMobileDictationDesktopSession',
|
||||
' return true'
|
||||
)
|
||||
const acquireIndex = startBody.indexOf('await keepAwakeOwner.acquire(dictationId)')
|
||||
const returnStartedIndex = startBody.indexOf('return true')
|
||||
const afterAcquire = startBody.slice(acquireIndex, returnStartedIndex)
|
||||
|
||||
expect(afterAcquire).toContain('isCurrentStart(options)')
|
||||
expect(desktopStartSource).toContain('options.getCurrentGeneration()')
|
||||
expect(desktopStartSource).toContain('options.getEnabled()')
|
||||
expect(desktopStartSource).toContain('options.getActiveId()')
|
||||
expect(afterAcquire).toContain(
|
||||
'await keepAwakeOwner.release(dictationId).catch(() => undefined)'
|
||||
)
|
||||
expect(afterAcquire).toContain("client.sendRequest('speech.dictation.cancel', { dictationId })")
|
||||
})
|
||||
|
||||
it('only reports keep-awake acquisition failures for the current start', () => {
|
||||
const startBody = sliceDesktopStartBetween(
|
||||
'export async function startMobileDictationDesktopSession',
|
||||
' return true'
|
||||
)
|
||||
const acquireIndex = startBody.indexOf('await keepAwakeOwner.acquire(dictationId)')
|
||||
const catchIndex = startBody.indexOf('} catch (err)', acquireIndex)
|
||||
const staleCleanupIndex = startBody.indexOf('await keepAwakeOwner.release', catchIndex)
|
||||
expect(catchIndex).toBeGreaterThan(acquireIndex)
|
||||
expect(staleCleanupIndex).toBeGreaterThan(catchIndex)
|
||||
const acquireFailurePath = startBody.slice(catchIndex, staleCleanupIndex)
|
||||
|
||||
expect(acquireFailurePath).toContain('catch (err)')
|
||||
expect(acquireFailurePath).toContain('isCurrentStart(options)')
|
||||
expect(acquireFailurePath).toContain('return')
|
||||
expect(acquireFailurePath).toContain('options.clearActiveId(dictationId)')
|
||||
expect(acquireFailurePath).toContain('setIdle()')
|
||||
expect(acquireFailurePath).toContain('throw err')
|
||||
expect(acquireFailurePath).not.toContain('keepAwakeOwner.release')
|
||||
})
|
||||
|
||||
it('releases keep-awake on all dictation cleanup paths without delaying recording shutdown', () => {
|
||||
const closeAudio = sliceBetween(
|
||||
'const closeDictationAudio = useCallback(',
|
||||
'const failActiveDictation ='
|
||||
)
|
||||
expect(closeAudio.indexOf('toggleRecording(false)')).toBeLessThan(
|
||||
closeAudio.indexOf('void keepAwakeOwner.release')
|
||||
)
|
||||
expect(closeAudio).toContain('.catch(() => undefined)')
|
||||
|
||||
const cleanupSlices = [
|
||||
sliceBetween('const failActiveDictation = useCallback(', 'useEffect(() => {'),
|
||||
sliceBetween('const cancel = useCallback(async () => {', 'useEffect(() => {\n const sub'),
|
||||
sliceBetween('return () => {\n const dictationId = activeIdRef.current', ' return {')
|
||||
]
|
||||
|
||||
for (const cleanupSlice of cleanupSlices) {
|
||||
expect(cleanupSlice).toContain('closeDictationAudio(dictationId)')
|
||||
}
|
||||
|
||||
const stopBody = sliceBetween('const stop = useCallback(async () => {', 'const cancel =')
|
||||
expect(stopBody.indexOf('toggleRecording(false)')).toBeLessThan(
|
||||
stopBody.indexOf('void keepAwakeOwner.release')
|
||||
)
|
||||
expect(stopBody.indexOf('void keepAwakeOwner.release')).toBeLessThan(
|
||||
stopBody.indexOf('await Promise.allSettled')
|
||||
)
|
||||
expect(stopBody.indexOf('void keepAwakeOwner.release')).toBeLessThan(
|
||||
stopBody.indexOf('speech.dictation.finish')
|
||||
)
|
||||
})
|
||||
|
||||
it('routes disabled state and audio interruptions through cancel cleanup', () => {
|
||||
const interruptionEffect = sliceBetween(
|
||||
"addExpoTwoWayAudioEventListener('onAudioInterruption'",
|
||||
'return () => sub.remove()'
|
||||
)
|
||||
const disabledEffect = sliceBetween(
|
||||
'useEffect(() => {\n if (!enabled) {',
|
||||
' }, [cancel, enabled])'
|
||||
)
|
||||
|
||||
expect(interruptionEffect).toContain("event.data === 'began' || event.data === 'blocked'")
|
||||
expect(interruptionEffect).toContain('void cancel()')
|
||||
expect(disabledEffect).toContain('void cancel()')
|
||||
})
|
||||
|
||||
it('uses per-owner dictation keep-awake tags and serializes async ownership changes', () => {
|
||||
expect(keepAwakeSource).toContain('private readonly ownerId = createOwnerId()')
|
||||
expect(keepAwakeSource).toContain(
|
||||
'`${MOBILE_DICTATION_KEEP_AWAKE_TAG_PREFIX}:${this.ownerId}:${dictationId}`'
|
||||
)
|
||||
expect(keepAwakeSource).toContain('private operation: Promise<void> = Promise.resolve()')
|
||||
expect(keepAwakeSource.match(/this\.operation\.then/g)).toHaveLength(2)
|
||||
expect(
|
||||
keepAwakeSource.match(/this\.operation = operation\.catch\(\(\) => undefined\)/g)
|
||||
).toHaveLength(2)
|
||||
expect(keepAwakeSource).toContain(
|
||||
'const targetTag = dictationId ? this.createTag(dictationId) : null'
|
||||
)
|
||||
expect(keepAwakeSource).toContain('if (!tag || (targetTag && tag !== targetTag))')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
|
||||
// Why: import from 'buffer' (the npm polyfill), not 'node:buffer' — Metro
|
||||
// can't resolve Node's builtin in a React Native bundle.
|
||||
import { Buffer } from 'buffer'
|
||||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
||||
import {
|
||||
addExpoTwoWayAudioEventListener,
|
||||
initialize,
|
||||
@@ -9,45 +6,26 @@ import {
|
||||
tearDown,
|
||||
toggleRecording
|
||||
} from '@orca/expo-two-way-audio'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget'
|
||||
import { enqueueMobileDictationAudioChunk } from './mobile-dictation-audio-chunk'
|
||||
import { createMobileDictationKeepAwakeOwner } from './mobile-dictation-keep-awake'
|
||||
import {
|
||||
MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE,
|
||||
MOBILE_DICTATION_PCM_SAMPLE_RATE,
|
||||
MobileDictationPendingAudioBudget
|
||||
} from './mobile-dictation-pending-audio-budget'
|
||||
DICTATION_FINISH_TIMEOUT_MS,
|
||||
createMobileDictationId,
|
||||
isCurrentMobileDictationFinish
|
||||
} from './mobile-dictation-session-state'
|
||||
import { startMobileDictationDesktopSession } from './mobile-dictation-desktop-start'
|
||||
import type {
|
||||
DictationStatus,
|
||||
UseMobileDictationOptions,
|
||||
UseMobileDictationResult
|
||||
} from './mobile-dictation-session-state'
|
||||
|
||||
type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error'
|
||||
|
||||
type UseMobileDictationOptions = {
|
||||
client: RpcClient | null
|
||||
enabled: boolean
|
||||
onTranscript: (text: string) => void
|
||||
onError?: (error: Error) => void
|
||||
}
|
||||
|
||||
export type UseMobileDictationResult = {
|
||||
status: DictationStatus
|
||||
isStarting: boolean
|
||||
isRecording: boolean
|
||||
isProcessing: boolean
|
||||
error: string | null
|
||||
start: () => Promise<void>
|
||||
stop: () => Promise<void>
|
||||
cancel: () => Promise<void>
|
||||
}
|
||||
|
||||
const DICTATION_FINISH_TIMEOUT_MS = 75_000
|
||||
|
||||
function bytesToBase64(bytes: Uint8Array): string {
|
||||
return Buffer.from(bytes).toString('base64')
|
||||
}
|
||||
|
||||
function createDictationId(): string {
|
||||
return `mobile-dictation-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
||||
}
|
||||
export type { UseMobileDictationResult } from './mobile-dictation-session-state'
|
||||
|
||||
export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult {
|
||||
const { client, enabled, onTranscript, onError } = options
|
||||
const keepAwakeOwner = useMemo(createMobileDictationKeepAwakeOwner, [])
|
||||
const [status, setStatus] = useState<DictationStatus>('idle')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const activeIdRef = useRef<string | null>(null)
|
||||
@@ -77,6 +55,17 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
onErrorRef.current?.(normalized)
|
||||
}, [])
|
||||
|
||||
const closeDictationAudio = useCallback(
|
||||
(dictationId?: string | null) => {
|
||||
acceptingChunksRef.current = false
|
||||
pendingChunksRef.current.clear()
|
||||
pendingAudioBudgetRef.current.reset()
|
||||
toggleRecording(false)
|
||||
void keepAwakeOwner.release(dictationId ?? undefined).catch(() => undefined)
|
||||
},
|
||||
[keepAwakeOwner]
|
||||
)
|
||||
|
||||
const failActiveDictation = useCallback(
|
||||
(dictationId: string, err: unknown) => {
|
||||
const client = clientRef.current
|
||||
@@ -84,16 +73,13 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
return
|
||||
}
|
||||
activeIdRef.current = null
|
||||
acceptingChunksRef.current = false
|
||||
pendingChunksRef.current.clear()
|
||||
pendingAudioBudgetRef.current.reset()
|
||||
toggleRecording(false)
|
||||
closeDictationAudio(dictationId)
|
||||
if (client && dictationId) {
|
||||
void client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
}
|
||||
reportError(err)
|
||||
},
|
||||
[reportError]
|
||||
[closeDictationAudio, reportError]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
@@ -103,32 +89,12 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
if (!client || !dictationId || !enabledRef.current || !acceptingChunksRef.current) {
|
||||
return
|
||||
}
|
||||
const raw = event.data
|
||||
const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw)
|
||||
const byteLength = bytes.byteLength
|
||||
if (!pendingAudioBudgetRef.current.tryReserve(byteLength)) {
|
||||
failActiveDictation(dictationId, new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE))
|
||||
return
|
||||
}
|
||||
const sendChunk = client
|
||||
.sendRequest('speech.dictation.chunk', {
|
||||
dictationId,
|
||||
audioBase64: bytesToBase64(bytes),
|
||||
sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE
|
||||
})
|
||||
.then((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
})
|
||||
.catch((err) => failActiveDictation(dictationId, err))
|
||||
.finally(() => {
|
||||
if (activeIdRef.current === dictationId || finishingIdRef.current === dictationId) {
|
||||
pendingAudioBudgetRef.current.release(byteLength)
|
||||
}
|
||||
pendingChunksRef.current.delete(sendChunk)
|
||||
})
|
||||
pendingChunksRef.current.add(sendChunk)
|
||||
enqueueMobileDictationAudioChunk(client, dictationId, event, {
|
||||
pendingChunks: pendingChunksRef.current,
|
||||
pendingAudioBudget: pendingAudioBudgetRef.current,
|
||||
shouldReleaseBudget: (id) => activeIdRef.current === id || finishingIdRef.current === id,
|
||||
failActiveDictation
|
||||
})
|
||||
})
|
||||
return () => sub.remove()
|
||||
}, [failActiveDictation, reportError])
|
||||
@@ -168,31 +134,25 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
throw new Error('Failed to initialize microphone')
|
||||
}
|
||||
|
||||
const dictationId = createDictationId()
|
||||
const dictationId = createMobileDictationId()
|
||||
activeIdRef.current = dictationId
|
||||
try {
|
||||
const response = await client.sendRequest('speech.dictation.start', { dictationId })
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
} catch (err) {
|
||||
if (activeIdRef.current === dictationId) {
|
||||
activeIdRef.current = null
|
||||
}
|
||||
await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
setStatus('idle')
|
||||
throw err
|
||||
}
|
||||
if (
|
||||
generationRef.current !== generation ||
|
||||
!enabledRef.current ||
|
||||
activeIdRef.current !== dictationId
|
||||
) {
|
||||
await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
if (activeIdRef.current === dictationId) {
|
||||
activeIdRef.current = null
|
||||
}
|
||||
setStatus('idle')
|
||||
|
||||
const started = await startMobileDictationDesktopSession({
|
||||
client,
|
||||
dictationId,
|
||||
generation,
|
||||
getCurrentGeneration: () => generationRef.current,
|
||||
getEnabled: () => enabledRef.current,
|
||||
getActiveId: () => activeIdRef.current,
|
||||
clearActiveId: (id) => {
|
||||
if (activeIdRef.current === id) {
|
||||
activeIdRef.current = null
|
||||
}
|
||||
},
|
||||
setIdle: () => setStatus('idle'),
|
||||
keepAwakeOwner
|
||||
})
|
||||
if (!started) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -201,7 +161,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
pendingAudioBudgetRef.current.reset()
|
||||
toggleRecording(true)
|
||||
setStatus('recording')
|
||||
}, [])
|
||||
}, [keepAwakeOwner])
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
const client = clientRef.current
|
||||
@@ -216,13 +176,18 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
setStatus('processing')
|
||||
acceptingChunksRef.current = false
|
||||
toggleRecording(false)
|
||||
void keepAwakeOwner.release(dictationId).catch(() => undefined)
|
||||
try {
|
||||
await Promise.allSettled(Array.from(pendingChunksRef.current))
|
||||
if (
|
||||
generationRef.current !== generation ||
|
||||
activeIdRef.current !== dictationId ||
|
||||
finishingIdRef.current !== dictationId ||
|
||||
!enabledRef.current
|
||||
!isCurrentMobileDictationFinish(
|
||||
generationRef.current,
|
||||
generation,
|
||||
enabledRef.current,
|
||||
activeIdRef.current,
|
||||
finishingIdRef.current,
|
||||
dictationId
|
||||
)
|
||||
) {
|
||||
return
|
||||
}
|
||||
@@ -235,10 +200,14 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
if (
|
||||
generationRef.current !== generation ||
|
||||
activeIdRef.current !== dictationId ||
|
||||
finishingIdRef.current !== dictationId ||
|
||||
!enabledRef.current
|
||||
!isCurrentMobileDictationFinish(
|
||||
generationRef.current,
|
||||
generation,
|
||||
enabledRef.current,
|
||||
activeIdRef.current,
|
||||
finishingIdRef.current,
|
||||
dictationId
|
||||
)
|
||||
) {
|
||||
return
|
||||
}
|
||||
@@ -261,7 +230,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
finishingIdRef.current = null
|
||||
}
|
||||
}
|
||||
}, [failActiveDictation])
|
||||
}, [failActiveDictation, keepAwakeOwner])
|
||||
|
||||
const cancel = useCallback(async () => {
|
||||
const client = clientRef.current
|
||||
@@ -269,16 +238,13 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
generationRef.current += 1
|
||||
activeIdRef.current = null
|
||||
finishingIdRef.current = null
|
||||
acceptingChunksRef.current = false
|
||||
pendingChunksRef.current.clear()
|
||||
pendingAudioBudgetRef.current.reset()
|
||||
toggleRecording(false)
|
||||
closeDictationAudio(dictationId)
|
||||
if (client && dictationId) {
|
||||
await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
|
||||
}
|
||||
setStatus('idle')
|
||||
setError(null)
|
||||
}, [])
|
||||
}, [closeDictationAudio])
|
||||
|
||||
useEffect(() => {
|
||||
const sub = addExpoTwoWayAudioEventListener('onAudioInterruption', (event) => {
|
||||
@@ -301,10 +267,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
generationRef.current += 1
|
||||
activeIdRef.current = null
|
||||
finishingIdRef.current = null
|
||||
acceptingChunksRef.current = false
|
||||
pendingChunksRef.current.clear()
|
||||
pendingAudioBudgetRef.current.reset()
|
||||
toggleRecording(false)
|
||||
closeDictationAudio(dictationId)
|
||||
void tearDown()
|
||||
if (clientRef.current && dictationId) {
|
||||
void clientRef.current
|
||||
@@ -312,7 +275,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
|
||||
.catch(() => undefined)
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
}, [closeDictationAudio])
|
||||
|
||||
return {
|
||||
status,
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# Mobile Dictation Keep-Awake Perf Audit
|
||||
|
||||
Task: `task_6cc0dd2ad31c`
|
||||
|
||||
## Scope Reviewed
|
||||
|
||||
- Required perf skill: `/Users/jinjingliang/Documents/projects/noqa/devex/.orca-internal/eng/skills/perf/SKILL.md`
|
||||
- Design doc: `docs/mobile-dictation-keep-awake.md`
|
||||
- Changed tracked files: `mobile/package.json`, `mobile/pnpm-lock.yaml`, `mobile/src/hooks/use-mobile-dictation.ts`, `mobile/src/hooks/use-mobile-dictation-source.test.ts`
|
||||
- New untracked implementation files: `mobile/src/hooks/mobile-dictation-keep-awake.ts`, `mobile/src/hooks/mobile-dictation-session-state.ts`, `mobile/src/hooks/mobile-dictation-desktop-start.ts`, `mobile/src/hooks/mobile-dictation-audio-chunk.ts`
|
||||
|
||||
## Performance Surfaces Checked
|
||||
|
||||
- Polling/timers/startup: no added `setInterval`, `setTimeout`, startup probe, watcher, or background loop in the touched runtime files.
|
||||
- RPC volume: steady-state audio chunking remains one `speech.dictation.chunk` request per accepted microphone event; keep-awake adds no per-chunk RPC. New/relocated `speech.dictation.cancel` calls are stale/error cleanup only.
|
||||
- Render/store work: `useMobileDictation` adds one `useMemo` owner per hook instance and no broad store subscription or render-time scan.
|
||||
- Mobile native handles: `expo-keep-awake` is acquired once per successful desktop dictation start and released on stop, cancel, failure, disabled-state cancel, audio interruption cancel, and unmount.
|
||||
- Async cleanup ordering: recording shutdown calls `toggleRecording(false)` before best-effort keep-awake release, so release latency should not extend microphone capture.
|
||||
- Memory/resource leaks: owner tags are per hook owner plus dictation id; acquire/release operations are serialized so stale release should not deactivate a newer session tag.
|
||||
- Cache/invalidation: no cache, persisted state, or cross-device sync was added; keep-awake remains local mobile state.
|
||||
- Subprocess/filesystem: dependency metadata changed only; no runtime subprocess or filesystem work was added.
|
||||
|
||||
## Findings
|
||||
|
||||
Severity: None for Critical, High, and Medium.
|
||||
|
||||
I did not find a concrete Critical/High/Medium performance issue in the implemented diff. The change follows the intended shape from the design doc: no polling, no recurring native work, no added startup path, no extra chunk-path RPCs, and release is not awaited before recording shutdown.
|
||||
|
||||
Low: None.
|
||||
|
||||
Nit: None.
|
||||
|
||||
## Measurement Or Test Hooks For Residual Risks
|
||||
|
||||
- Native keep-awake handle cleanup: deterministic native/mock test that runs start -> stop, start -> cancel, start -> disable, start -> interruption, and start -> unmount loops while counting `activateKeepAwakeAsync` and `deactivateKeepAwake` calls by tag; expected count is one release for each acquired tag and no release of a later tag by an earlier operation.
|
||||
- Stale acquire/release race: unit test with mocked keep-awake promises where acquire is delayed, cancel/unmount runs, then acquire resolves; expected result is the stale tag is released and `toggleRecording(true)` is never called.
|
||||
- RPC volume: mocked `client.sendRequest` method-name counter over one dictation session with N microphone events; expected count is one start, N chunks, one finish or cancel, and zero keep-awake-related RPCs.
|
||||
- Shutdown latency: deterministic ordering test or trace asserting `toggleRecording(false)` is invoked before `keepAwakeOwner.release(...)`, `Promise.allSettled(...)`, and `speech.dictation.finish`.
|
||||
- Memory/resource retention: repeated hook mount/start/cancel/unmount test with mocked event listeners and keep-awake owner; expected listener count returns to baseline and pending chunk/budget state is cleared.
|
||||
|
||||
## Checks Run Or Inspected
|
||||
|
||||
- Inspected `git status --short`.
|
||||
- Inspected `git diff` for tracked changed files.
|
||||
- Read all required new untracked files directly.
|
||||
- Searched touched runtime files for timers/listeners/native keep-awake calls/RPC calls with `rg`.
|
||||
- Ran `pnpm --dir mobile test -- src/hooks/use-mobile-dictation-source.test.ts`.
|
||||
|
||||
## Test Result
|
||||
|
||||
`pnpm --dir mobile test -- src/hooks/use-mobile-dictation-source.test.ts` passed: 173 test files, 1258 tests passed, 2 skipped.
|
||||
|
||||
No implementation files were changed by this audit.
|
||||
Reference in New Issue
Block a user