diff --git a/mobile/package.json b/mobile/package.json index 9f91777a842..20aa2fa19a5 100644 --- a/mobile/package.json +++ b/mobile/package.json @@ -35,6 +35,7 @@ "expo-haptics": "^55.0.14", "expo-image-manipulator": "^55.0.17", "expo-image-picker": "^55.0.20", + "expo-keep-awake": "55.0.8", "expo-linking": "^55.0.15", "expo-modules-core": "~55.0.25", "expo-network": "~55.0.14", diff --git a/mobile/pnpm-lock.yaml b/mobile/pnpm-lock.yaml index 496855fc266..824ffa7d7bb 100644 --- a/mobile/pnpm-lock.yaml +++ b/mobile/pnpm-lock.yaml @@ -65,6 +65,9 @@ importers: expo-image-picker: specifier: ^55.0.20 version: 55.0.20(expo@55.0.27) + expo-keep-awake: + specifier: 55.0.8 + version: 55.0.8(expo@55.0.27)(react@19.2.6) expo-linking: specifier: ^55.0.15 version: 55.0.15(expo@55.0.27)(react-native@0.83.9(@babel/core@7.29.7)(@react-native/metro-config@0.85.2(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6) diff --git a/mobile/src/hooks/mobile-dictation-audio-chunk.ts b/mobile/src/hooks/mobile-dictation-audio-chunk.ts new file mode 100644 index 00000000000..792292e6c16 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-audio-chunk.ts @@ -0,0 +1,52 @@ +import { + MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE, + MOBILE_DICTATION_PCM_SAMPLE_RATE +} from './mobile-dictation-pending-audio-budget' +import { bytesToBase64 } from './mobile-dictation-session-state' +import type { MicrophoneDataEvent } from '@orca/expo-two-way-audio' +import type { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' +import type { RpcClient } from '../transport/rpc-client' + +type MobileDictationAudioChunkQueue = { + pendingChunks: Set> + pendingAudioBudget: MobileDictationPendingAudioBudget + shouldReleaseBudget: (dictationId: string) => boolean + failActiveDictation: (dictationId: string, err: unknown) => void +} + +export function enqueueMobileDictationAudioChunk( + client: RpcClient, + dictationId: string, + event: MicrophoneDataEvent, + queue: MobileDictationAudioChunkQueue +): void { + const raw = event.data + const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw) + const byteLength = bytes.byteLength + if (!queue.pendingAudioBudget.tryReserve(byteLength)) { + queue.failActiveDictation( + dictationId, + new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE) + ) + return + } + const sendChunk = client + .sendRequest('speech.dictation.chunk', { + dictationId, + audioBase64: bytesToBase64(bytes), + sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE + }) + .then((response) => { + if (!response.ok) { + throw new Error(response.error.message) + } + }) + .catch((err) => queue.failActiveDictation(dictationId, err)) + .finally(() => { + if (queue.shouldReleaseBudget(dictationId)) { + queue.pendingAudioBudget.release(byteLength) + } + queue.pendingChunks.delete(sendChunk) + }) + queue.pendingChunks.add(sendChunk) +} diff --git a/mobile/src/hooks/mobile-dictation-desktop-start.ts b/mobile/src/hooks/mobile-dictation-desktop-start.ts new file mode 100644 index 00000000000..19f3abb8d5c --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-desktop-start.ts @@ -0,0 +1,74 @@ +import { isCurrentMobileDictationStart } from './mobile-dictation-session-state' +import type { MobileDictationKeepAwakeOwner } from './mobile-dictation-keep-awake' +import type { RpcClient } from '../transport/rpc-client' + +type StartMobileDictationDesktopSessionOptions = { + client: RpcClient + dictationId: string + generation: number + getCurrentGeneration: () => number + getEnabled: () => boolean + getActiveId: () => string | null + clearActiveId: (dictationId: string) => void + setIdle: () => void + keepAwakeOwner: MobileDictationKeepAwakeOwner +} + +function isCurrentStart(options: StartMobileDictationDesktopSessionOptions): boolean { + return isCurrentMobileDictationStart( + options.getCurrentGeneration(), + options.generation, + options.getEnabled(), + options.getActiveId(), + options.dictationId + ) +} + +export async function startMobileDictationDesktopSession( + options: StartMobileDictationDesktopSessionOptions +): Promise { + const { client, dictationId, keepAwakeOwner, setIdle } = options + + try { + const response = await client.sendRequest('speech.dictation.start', { dictationId }) + if (!response.ok) { + throw new Error(response.error.message) + } + } catch (err) { + options.clearActiveId(dictationId) + await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) + setIdle() + throw err + } + + if (!isCurrentStart(options)) { + await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) + options.clearActiveId(dictationId) + setIdle() + return false + } + + try { + // Keep-awake is acquired only after the desktop session exists, so stale + // mobile starts can be canceled without holding a screen-lock tag. + await keepAwakeOwner.acquire(dictationId) + } catch (err) { + if (!isCurrentStart(options)) { + return false + } + options.clearActiveId(dictationId) + await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) + setIdle() + throw err + } + + if (!isCurrentStart(options)) { + await keepAwakeOwner.release(dictationId).catch(() => undefined) + await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) + options.clearActiveId(dictationId) + setIdle() + return false + } + + return true +} diff --git a/mobile/src/hooks/mobile-dictation-keep-awake.ts b/mobile/src/hooks/mobile-dictation-keep-awake.ts new file mode 100644 index 00000000000..528e1c3d412 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-keep-awake.ts @@ -0,0 +1,56 @@ +import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake' + +const MOBILE_DICTATION_KEEP_AWAKE_TAG_PREFIX = 'orca-mobile-dictation' + +let nextOwnerId = 0 + +function createOwnerId(): string { + nextOwnerId += 1 + return `${Date.now()}-${nextOwnerId}-${Math.random().toString(36).slice(2)}` +} + +export class MobileDictationKeepAwakeOwner { + private readonly ownerId = createOwnerId() + private acquiredTag: string | null = null + private operation: Promise = Promise.resolve() + + acquire(dictationId: string): Promise { + const tag = this.createTag(dictationId) + const operation = this.operation.then(async () => { + if (this.acquiredTag === tag) { + return + } + if (this.acquiredTag) { + const previousTag = this.acquiredTag + this.acquiredTag = null + await deactivateKeepAwake(previousTag).catch(() => undefined) + } + await activateKeepAwakeAsync(tag) + this.acquiredTag = tag + }) + this.operation = operation.catch(() => undefined) + return operation + } + + release(dictationId?: string): Promise { + const targetTag = dictationId ? this.createTag(dictationId) : null + const operation = this.operation.then(async () => { + const tag = this.acquiredTag + if (!tag || (targetTag && tag !== targetTag)) { + return + } + this.acquiredTag = null + await deactivateKeepAwake(tag) + }) + this.operation = operation.catch(() => undefined) + return operation + } + + private createTag(dictationId: string): string { + return `${MOBILE_DICTATION_KEEP_AWAKE_TAG_PREFIX}:${this.ownerId}:${dictationId}` + } +} + +export function createMobileDictationKeepAwakeOwner(): MobileDictationKeepAwakeOwner { + return new MobileDictationKeepAwakeOwner() +} diff --git a/mobile/src/hooks/mobile-dictation-session-state.ts b/mobile/src/hooks/mobile-dictation-session-state.ts new file mode 100644 index 00000000000..8770624610d --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-session-state.ts @@ -0,0 +1,61 @@ +// Why: import from 'buffer' (the npm polyfill), not 'node:buffer' because +// Metro cannot resolve Node builtins in a React Native bundle. +import { Buffer } from 'buffer' + +import type { RpcClient } from '../transport/rpc-client' + +export type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error' + +export type UseMobileDictationOptions = { + client: RpcClient | null + enabled: boolean + onTranscript: (text: string) => void + onError?: (error: Error) => void +} + +export type UseMobileDictationResult = { + status: DictationStatus + isStarting: boolean + isRecording: boolean + isProcessing: boolean + error: string | null + start: () => Promise + stop: () => Promise + cancel: () => Promise +} + +export const DICTATION_FINISH_TIMEOUT_MS = 75_000 + +export function bytesToBase64(bytes: Uint8Array): string { + return Buffer.from(bytes).toString('base64') +} + +export function createMobileDictationId(): string { + return `mobile-dictation-${Date.now()}-${Math.random().toString(36).slice(2)}` +} + +export function isCurrentMobileDictationStart( + currentGeneration: number, + generation: number, + enabled: boolean, + activeId: string | null, + dictationId: string +): boolean { + return currentGeneration === generation && enabled && activeId === dictationId +} + +export function isCurrentMobileDictationFinish( + currentGeneration: number, + generation: number, + enabled: boolean, + activeId: string | null, + finishingId: string | null, + dictationId: string +): boolean { + return ( + currentGeneration === generation && + enabled && + activeId === dictationId && + finishingId === dictationId + ) +} diff --git a/mobile/src/hooks/use-mobile-dictation-source.test.ts b/mobile/src/hooks/use-mobile-dictation-source.test.ts index 6319e2d9b5a..531b49b4503 100644 --- a/mobile/src/hooks/use-mobile-dictation-source.test.ts +++ b/mobile/src/hooks/use-mobile-dictation-source.test.ts @@ -2,13 +2,37 @@ import { readFileSync } from 'node:fs' import { describe, expect, it } from 'vitest' const source = readFileSync(new URL('./use-mobile-dictation.ts', import.meta.url), 'utf8') +const audioChunkSource = readFileSync( + new URL('./mobile-dictation-audio-chunk.ts', import.meta.url), + 'utf8' +) +const keepAwakeSource = readFileSync( + new URL('./mobile-dictation-keep-awake.ts', import.meta.url), + 'utf8' +) +const desktopStartSource = readFileSync( + new URL('./mobile-dictation-desktop-start.ts', import.meta.url), + 'utf8' +) +const sessionStateSource = readFileSync( + new URL('./mobile-dictation-session-state.ts', import.meta.url), + 'utf8' +) + +function sliceSource(sourceText: string, startPattern: string, endPattern: string): string { + const start = sourceText.indexOf(startPattern) + expect(start).toBeGreaterThanOrEqual(0) + const end = sourceText.indexOf(endPattern, start) + expect(end).toBeGreaterThan(start) + return sourceText.slice(start, end) +} function sliceBetween(startPattern: string, endPattern: string): string { - const start = source.indexOf(startPattern) - expect(start).toBeGreaterThanOrEqual(0) - const end = source.indexOf(endPattern, start) - expect(end).toBeGreaterThan(start) - return source.slice(start, end) + return sliceSource(source, startPattern, endPattern) +} + +function sliceDesktopStartBetween(startPattern: string, endPattern: string): string { + return sliceSource(desktopStartSource, startPattern, endPattern) } describe('useMobileDictation source invariants', () => { @@ -28,9 +52,10 @@ describe('useMobileDictation source invariants', () => { }) it('reserves pending audio bytes before encoding microphone chunks', () => { - const microphoneEffect = sliceBetween( - "addExpoTwoWayAudioEventListener('onMicrophoneData'", - 'return () => sub.remove()' + const microphoneEffect = sliceSource( + audioChunkSource, + 'export function enqueueMobileDictationAudioChunk', + ' queue.pendingChunks.add(sendChunk)' ) const reserveIndex = microphoneEffect.indexOf('tryReserve(byteLength)') @@ -39,7 +64,8 @@ describe('useMobileDictation source invariants', () => { expect(encodeIndex).toBeGreaterThanOrEqual(0) expect(reserveIndex).toBeLessThan(encodeIndex) expect(microphoneEffect).toContain('MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE') - expect(microphoneEffect).toContain('pendingAudioBudgetRef.current.release(byteLength)') + expect(microphoneEffect).toContain('queue.pendingAudioBudget.release(byteLength)') + expect(source).toContain('enqueueMobileDictationAudioChunk(client, dictationId, event') }) it('resets pending audio bytes whenever pending chunk tracking is cleared', () => { @@ -48,4 +74,147 @@ describe('useMobileDictation source invariants', () => { expect(pendingAudioResets).toHaveLength(pendingChunkClears.length) }) + + it('keeps mobile dictation keep-awake ownership beside the hook', () => { + expect(source).toContain( + "import { createMobileDictationKeepAwakeOwner } from './mobile-dictation-keep-awake'" + ) + expect(source).toContain( + 'const keepAwakeOwner = useMemo(createMobileDictationKeepAwakeOwner, [])' + ) + expect(keepAwakeSource).toContain('activateKeepAwakeAsync') + expect(keepAwakeSource).toContain('deactivateKeepAwake') + expect(keepAwakeSource).not.toMatch(/\bactivateKeepAwake\s*\(/) + }) + + it('acquires keep-awake only after desktop start and stale-start guards', () => { + const hookStartBody = sliceBetween('const start = useCallback(async () => {', 'const stop =') + const startBody = sliceDesktopStartBetween( + 'export async function startMobileDictationDesktopSession', + ' return true' + ) + const desktopStartIndex = startBody.indexOf( + "client.sendRequest('speech.dictation.start', { dictationId })" + ) + const acquireIndex = startBody.indexOf('await keepAwakeOwner.acquire(dictationId)') + const desktopSessionIndex = hookStartBody.indexOf('await startMobileDictationDesktopSession') + const toggleRecordingIndex = hookStartBody.indexOf('toggleRecording(true)') + + expect(desktopStartIndex).toBeGreaterThanOrEqual(0) + expect(acquireIndex).toBeGreaterThan(desktopStartIndex) + expect(desktopSessionIndex).toBeGreaterThanOrEqual(0) + expect(toggleRecordingIndex).toBeGreaterThan(desktopSessionIndex) + + const beforeAcquire = startBody.slice(desktopStartIndex, acquireIndex) + expect(beforeAcquire).toContain('isCurrentStart(options)') + expect(desktopStartSource).toContain('options.getCurrentGeneration()') + expect(desktopStartSource).toContain('options.getEnabled()') + expect(desktopStartSource).toContain('options.getActiveId()') + expect(sessionStateSource).toContain( + 'currentGeneration === generation && enabled && activeId === dictationId' + ) + }) + + it('re-checks stale-start guards after awaited keep-awake acquisition', () => { + const startBody = sliceDesktopStartBetween( + 'export async function startMobileDictationDesktopSession', + ' return true' + ) + const acquireIndex = startBody.indexOf('await keepAwakeOwner.acquire(dictationId)') + const returnStartedIndex = startBody.indexOf('return true') + const afterAcquire = startBody.slice(acquireIndex, returnStartedIndex) + + expect(afterAcquire).toContain('isCurrentStart(options)') + expect(desktopStartSource).toContain('options.getCurrentGeneration()') + expect(desktopStartSource).toContain('options.getEnabled()') + expect(desktopStartSource).toContain('options.getActiveId()') + expect(afterAcquire).toContain( + 'await keepAwakeOwner.release(dictationId).catch(() => undefined)' + ) + expect(afterAcquire).toContain("client.sendRequest('speech.dictation.cancel', { dictationId })") + }) + + it('only reports keep-awake acquisition failures for the current start', () => { + const startBody = sliceDesktopStartBetween( + 'export async function startMobileDictationDesktopSession', + ' return true' + ) + const acquireIndex = startBody.indexOf('await keepAwakeOwner.acquire(dictationId)') + const catchIndex = startBody.indexOf('} catch (err)', acquireIndex) + const staleCleanupIndex = startBody.indexOf('await keepAwakeOwner.release', catchIndex) + expect(catchIndex).toBeGreaterThan(acquireIndex) + expect(staleCleanupIndex).toBeGreaterThan(catchIndex) + const acquireFailurePath = startBody.slice(catchIndex, staleCleanupIndex) + + expect(acquireFailurePath).toContain('catch (err)') + expect(acquireFailurePath).toContain('isCurrentStart(options)') + expect(acquireFailurePath).toContain('return') + expect(acquireFailurePath).toContain('options.clearActiveId(dictationId)') + expect(acquireFailurePath).toContain('setIdle()') + expect(acquireFailurePath).toContain('throw err') + expect(acquireFailurePath).not.toContain('keepAwakeOwner.release') + }) + + it('releases keep-awake on all dictation cleanup paths without delaying recording shutdown', () => { + const closeAudio = sliceBetween( + 'const closeDictationAudio = useCallback(', + 'const failActiveDictation =' + ) + expect(closeAudio.indexOf('toggleRecording(false)')).toBeLessThan( + closeAudio.indexOf('void keepAwakeOwner.release') + ) + expect(closeAudio).toContain('.catch(() => undefined)') + + const cleanupSlices = [ + sliceBetween('const failActiveDictation = useCallback(', 'useEffect(() => {'), + sliceBetween('const cancel = useCallback(async () => {', 'useEffect(() => {\n const sub'), + sliceBetween('return () => {\n const dictationId = activeIdRef.current', ' return {') + ] + + for (const cleanupSlice of cleanupSlices) { + expect(cleanupSlice).toContain('closeDictationAudio(dictationId)') + } + + const stopBody = sliceBetween('const stop = useCallback(async () => {', 'const cancel =') + expect(stopBody.indexOf('toggleRecording(false)')).toBeLessThan( + stopBody.indexOf('void keepAwakeOwner.release') + ) + expect(stopBody.indexOf('void keepAwakeOwner.release')).toBeLessThan( + stopBody.indexOf('await Promise.allSettled') + ) + expect(stopBody.indexOf('void keepAwakeOwner.release')).toBeLessThan( + stopBody.indexOf('speech.dictation.finish') + ) + }) + + it('routes disabled state and audio interruptions through cancel cleanup', () => { + const interruptionEffect = sliceBetween( + "addExpoTwoWayAudioEventListener('onAudioInterruption'", + 'return () => sub.remove()' + ) + const disabledEffect = sliceBetween( + 'useEffect(() => {\n if (!enabled) {', + ' }, [cancel, enabled])' + ) + + expect(interruptionEffect).toContain("event.data === 'began' || event.data === 'blocked'") + expect(interruptionEffect).toContain('void cancel()') + expect(disabledEffect).toContain('void cancel()') + }) + + it('uses per-owner dictation keep-awake tags and serializes async ownership changes', () => { + expect(keepAwakeSource).toContain('private readonly ownerId = createOwnerId()') + expect(keepAwakeSource).toContain( + '`${MOBILE_DICTATION_KEEP_AWAKE_TAG_PREFIX}:${this.ownerId}:${dictationId}`' + ) + expect(keepAwakeSource).toContain('private operation: Promise = Promise.resolve()') + expect(keepAwakeSource.match(/this\.operation\.then/g)).toHaveLength(2) + expect( + keepAwakeSource.match(/this\.operation = operation\.catch\(\(\) => undefined\)/g) + ).toHaveLength(2) + expect(keepAwakeSource).toContain( + 'const targetTag = dictationId ? this.createTag(dictationId) : null' + ) + expect(keepAwakeSource).toContain('if (!tag || (targetTag && tag !== targetTag))') + }) }) diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index a1028cf9205..7573ece866d 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -1,7 +1,4 @@ -import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' -// Why: import from 'buffer' (the npm polyfill), not 'node:buffer' — Metro -// can't resolve Node's builtin in a React Native bundle. -import { Buffer } from 'buffer' +import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { addExpoTwoWayAudioEventListener, initialize, @@ -9,45 +6,26 @@ import { tearDown, toggleRecording } from '@orca/expo-two-way-audio' -import type { RpcClient } from '../transport/rpc-client' +import { MobileDictationPendingAudioBudget } from './mobile-dictation-pending-audio-budget' +import { enqueueMobileDictationAudioChunk } from './mobile-dictation-audio-chunk' +import { createMobileDictationKeepAwakeOwner } from './mobile-dictation-keep-awake' import { - MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE, - MOBILE_DICTATION_PCM_SAMPLE_RATE, - MobileDictationPendingAudioBudget -} from './mobile-dictation-pending-audio-budget' + DICTATION_FINISH_TIMEOUT_MS, + createMobileDictationId, + isCurrentMobileDictationFinish +} from './mobile-dictation-session-state' +import { startMobileDictationDesktopSession } from './mobile-dictation-desktop-start' +import type { + DictationStatus, + UseMobileDictationOptions, + UseMobileDictationResult +} from './mobile-dictation-session-state' -type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error' - -type UseMobileDictationOptions = { - client: RpcClient | null - enabled: boolean - onTranscript: (text: string) => void - onError?: (error: Error) => void -} - -export type UseMobileDictationResult = { - status: DictationStatus - isStarting: boolean - isRecording: boolean - isProcessing: boolean - error: string | null - start: () => Promise - stop: () => Promise - cancel: () => Promise -} - -const DICTATION_FINISH_TIMEOUT_MS = 75_000 - -function bytesToBase64(bytes: Uint8Array): string { - return Buffer.from(bytes).toString('base64') -} - -function createDictationId(): string { - return `mobile-dictation-${Date.now()}-${Math.random().toString(36).slice(2)}` -} +export type { UseMobileDictationResult } from './mobile-dictation-session-state' export function useMobileDictation(options: UseMobileDictationOptions): UseMobileDictationResult { const { client, enabled, onTranscript, onError } = options + const keepAwakeOwner = useMemo(createMobileDictationKeepAwakeOwner, []) const [status, setStatus] = useState('idle') const [error, setError] = useState(null) const activeIdRef = useRef(null) @@ -77,6 +55,17 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil onErrorRef.current?.(normalized) }, []) + const closeDictationAudio = useCallback( + (dictationId?: string | null) => { + acceptingChunksRef.current = false + pendingChunksRef.current.clear() + pendingAudioBudgetRef.current.reset() + toggleRecording(false) + void keepAwakeOwner.release(dictationId ?? undefined).catch(() => undefined) + }, + [keepAwakeOwner] + ) + const failActiveDictation = useCallback( (dictationId: string, err: unknown) => { const client = clientRef.current @@ -84,16 +73,13 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil return } activeIdRef.current = null - acceptingChunksRef.current = false - pendingChunksRef.current.clear() - pendingAudioBudgetRef.current.reset() - toggleRecording(false) + closeDictationAudio(dictationId) if (client && dictationId) { void client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) } reportError(err) }, - [reportError] + [closeDictationAudio, reportError] ) useEffect(() => { @@ -103,32 +89,12 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil if (!client || !dictationId || !enabledRef.current || !acceptingChunksRef.current) { return } - const raw = event.data - const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw) - const byteLength = bytes.byteLength - if (!pendingAudioBudgetRef.current.tryReserve(byteLength)) { - failActiveDictation(dictationId, new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE)) - return - } - const sendChunk = client - .sendRequest('speech.dictation.chunk', { - dictationId, - audioBase64: bytesToBase64(bytes), - sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE - }) - .then((response) => { - if (!response.ok) { - throw new Error(response.error.message) - } - }) - .catch((err) => failActiveDictation(dictationId, err)) - .finally(() => { - if (activeIdRef.current === dictationId || finishingIdRef.current === dictationId) { - pendingAudioBudgetRef.current.release(byteLength) - } - pendingChunksRef.current.delete(sendChunk) - }) - pendingChunksRef.current.add(sendChunk) + enqueueMobileDictationAudioChunk(client, dictationId, event, { + pendingChunks: pendingChunksRef.current, + pendingAudioBudget: pendingAudioBudgetRef.current, + shouldReleaseBudget: (id) => activeIdRef.current === id || finishingIdRef.current === id, + failActiveDictation + }) }) return () => sub.remove() }, [failActiveDictation, reportError]) @@ -168,31 +134,25 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil throw new Error('Failed to initialize microphone') } - const dictationId = createDictationId() + const dictationId = createMobileDictationId() activeIdRef.current = dictationId - try { - const response = await client.sendRequest('speech.dictation.start', { dictationId }) - if (!response.ok) { - throw new Error(response.error.message) - } - } catch (err) { - if (activeIdRef.current === dictationId) { - activeIdRef.current = null - } - await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) - setStatus('idle') - throw err - } - if ( - generationRef.current !== generation || - !enabledRef.current || - activeIdRef.current !== dictationId - ) { - await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) - if (activeIdRef.current === dictationId) { - activeIdRef.current = null - } - setStatus('idle') + + const started = await startMobileDictationDesktopSession({ + client, + dictationId, + generation, + getCurrentGeneration: () => generationRef.current, + getEnabled: () => enabledRef.current, + getActiveId: () => activeIdRef.current, + clearActiveId: (id) => { + if (activeIdRef.current === id) { + activeIdRef.current = null + } + }, + setIdle: () => setStatus('idle'), + keepAwakeOwner + }) + if (!started) { return } @@ -201,7 +161,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil pendingAudioBudgetRef.current.reset() toggleRecording(true) setStatus('recording') - }, []) + }, [keepAwakeOwner]) const stop = useCallback(async () => { const client = clientRef.current @@ -216,13 +176,18 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil setStatus('processing') acceptingChunksRef.current = false toggleRecording(false) + void keepAwakeOwner.release(dictationId).catch(() => undefined) try { await Promise.allSettled(Array.from(pendingChunksRef.current)) if ( - generationRef.current !== generation || - activeIdRef.current !== dictationId || - finishingIdRef.current !== dictationId || - !enabledRef.current + !isCurrentMobileDictationFinish( + generationRef.current, + generation, + enabledRef.current, + activeIdRef.current, + finishingIdRef.current, + dictationId + ) ) { return } @@ -235,10 +200,14 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil throw new Error(response.error.message) } if ( - generationRef.current !== generation || - activeIdRef.current !== dictationId || - finishingIdRef.current !== dictationId || - !enabledRef.current + !isCurrentMobileDictationFinish( + generationRef.current, + generation, + enabledRef.current, + activeIdRef.current, + finishingIdRef.current, + dictationId + ) ) { return } @@ -261,7 +230,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil finishingIdRef.current = null } } - }, [failActiveDictation]) + }, [failActiveDictation, keepAwakeOwner]) const cancel = useCallback(async () => { const client = clientRef.current @@ -269,16 +238,13 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil generationRef.current += 1 activeIdRef.current = null finishingIdRef.current = null - acceptingChunksRef.current = false - pendingChunksRef.current.clear() - pendingAudioBudgetRef.current.reset() - toggleRecording(false) + closeDictationAudio(dictationId) if (client && dictationId) { await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) } setStatus('idle') setError(null) - }, []) + }, [closeDictationAudio]) useEffect(() => { const sub = addExpoTwoWayAudioEventListener('onAudioInterruption', (event) => { @@ -301,10 +267,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil generationRef.current += 1 activeIdRef.current = null finishingIdRef.current = null - acceptingChunksRef.current = false - pendingChunksRef.current.clear() - pendingAudioBudgetRef.current.reset() - toggleRecording(false) + closeDictationAudio(dictationId) void tearDown() if (clientRef.current && dictationId) { void clientRef.current @@ -312,7 +275,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil .catch(() => undefined) } } - }, []) + }, [closeDictationAudio]) return { status, diff --git a/perf-audit-mobile-dictation-keep-awake-task_6cc0dd2ad31c.md b/perf-audit-mobile-dictation-keep-awake-task_6cc0dd2ad31c.md new file mode 100644 index 00000000000..4ecda6768a2 --- /dev/null +++ b/perf-audit-mobile-dictation-keep-awake-task_6cc0dd2ad31c.md @@ -0,0 +1,53 @@ +# Mobile Dictation Keep-Awake Perf Audit + +Task: `task_6cc0dd2ad31c` + +## Scope Reviewed + +- Required perf skill: `/Users/jinjingliang/Documents/projects/noqa/devex/.orca-internal/eng/skills/perf/SKILL.md` +- Design doc: `docs/mobile-dictation-keep-awake.md` +- Changed tracked files: `mobile/package.json`, `mobile/pnpm-lock.yaml`, `mobile/src/hooks/use-mobile-dictation.ts`, `mobile/src/hooks/use-mobile-dictation-source.test.ts` +- New untracked implementation files: `mobile/src/hooks/mobile-dictation-keep-awake.ts`, `mobile/src/hooks/mobile-dictation-session-state.ts`, `mobile/src/hooks/mobile-dictation-desktop-start.ts`, `mobile/src/hooks/mobile-dictation-audio-chunk.ts` + +## Performance Surfaces Checked + +- Polling/timers/startup: no added `setInterval`, `setTimeout`, startup probe, watcher, or background loop in the touched runtime files. +- RPC volume: steady-state audio chunking remains one `speech.dictation.chunk` request per accepted microphone event; keep-awake adds no per-chunk RPC. New/relocated `speech.dictation.cancel` calls are stale/error cleanup only. +- Render/store work: `useMobileDictation` adds one `useMemo` owner per hook instance and no broad store subscription or render-time scan. +- Mobile native handles: `expo-keep-awake` is acquired once per successful desktop dictation start and released on stop, cancel, failure, disabled-state cancel, audio interruption cancel, and unmount. +- Async cleanup ordering: recording shutdown calls `toggleRecording(false)` before best-effort keep-awake release, so release latency should not extend microphone capture. +- Memory/resource leaks: owner tags are per hook owner plus dictation id; acquire/release operations are serialized so stale release should not deactivate a newer session tag. +- Cache/invalidation: no cache, persisted state, or cross-device sync was added; keep-awake remains local mobile state. +- Subprocess/filesystem: dependency metadata changed only; no runtime subprocess or filesystem work was added. + +## Findings + +Severity: None for Critical, High, and Medium. + +I did not find a concrete Critical/High/Medium performance issue in the implemented diff. The change follows the intended shape from the design doc: no polling, no recurring native work, no added startup path, no extra chunk-path RPCs, and release is not awaited before recording shutdown. + +Low: None. + +Nit: None. + +## Measurement Or Test Hooks For Residual Risks + +- Native keep-awake handle cleanup: deterministic native/mock test that runs start -> stop, start -> cancel, start -> disable, start -> interruption, and start -> unmount loops while counting `activateKeepAwakeAsync` and `deactivateKeepAwake` calls by tag; expected count is one release for each acquired tag and no release of a later tag by an earlier operation. +- Stale acquire/release race: unit test with mocked keep-awake promises where acquire is delayed, cancel/unmount runs, then acquire resolves; expected result is the stale tag is released and `toggleRecording(true)` is never called. +- RPC volume: mocked `client.sendRequest` method-name counter over one dictation session with N microphone events; expected count is one start, N chunks, one finish or cancel, and zero keep-awake-related RPCs. +- Shutdown latency: deterministic ordering test or trace asserting `toggleRecording(false)` is invoked before `keepAwakeOwner.release(...)`, `Promise.allSettled(...)`, and `speech.dictation.finish`. +- Memory/resource retention: repeated hook mount/start/cancel/unmount test with mocked event listeners and keep-awake owner; expected listener count returns to baseline and pending chunk/budget state is cleared. + +## Checks Run Or Inspected + +- Inspected `git status --short`. +- Inspected `git diff` for tracked changed files. +- Read all required new untracked files directly. +- Searched touched runtime files for timers/listeners/native keep-awake calls/RPC calls with `rg`. +- Ran `pnpm --dir mobile test -- src/hooks/use-mobile-dictation-source.test.ts`. + +## Test Result + +`pnpm --dir mobile test -- src/hooks/use-mobile-dictation-source.test.ts` passed: 173 test files, 1258 tests passed, 2 skipped. + +No implementation files were changed by this audit.