From 1283e070e3769dbe84ba460be4cb7c71be4e0e41 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:51:50 +0000 Subject: [PATCH] feat(mobile): transcribe dictation on iPhone with Apple's on-device speech model (#39410) * feat(mobile): transcribe dictation on device with SpeechTranscriber on iOS 26 * fix(mobile): fall back to the dictation model, reserve locales and stop a finishing session on cancel * fix(mobile): resolve on-device speech locales deterministically and report the simulator unsupported * refactor(mobile): simplify the on-device speech engine and skip conversion when formats match --- .../ios/ExpoTwoWayAudioModule.swift | 61 +++- .../ios/OnDeviceSpeechController.swift | 182 +++++++++++ .../ios/OnDeviceSpeechSession.swift | 305 ++++++++++++++++++ 3 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechController.swift create mode 100644 mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechSession.swift diff --git a/mobile/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift b/mobile/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift index 5255b81cd41..95fa716d752 100644 --- a/mobile/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift +++ b/mobile/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift @@ -8,10 +8,13 @@ let ON_AUDIO_INTERRUPTION_EVENT_NAME = "onAudioInterruption" public class ExpoTwoWayAudioModule: Module { private var audioEngine: AudioEngine? + // OnDeviceSpeechController on iOS 26 and later; nil before, where every speech function answers unsupported. + private var speech: AnyObject? public func definition() -> ModuleDefinition { Name("ExpoTwoWayAudio") - OnCreate { + OnCreate { [weak self] in + guard let self else { return } let permissionsManager = self.appContext?.permissions EXPermissionsMethodsDelegate.register( [ @@ -19,9 +22,19 @@ public class ExpoTwoWayAudioModule: Module { ], withPermissionsManager: permissionsManager ) + if #available(iOS 26.0, *) { + self.speech = OnDeviceSpeechController { [weak self] name, body in + self?.sendEvent(name, body) + } + } } + OnDestroy { + guard #available(iOS 26.0, *) else { return } + self.onDeviceSpeech()?.cancel() + } + AsyncFunction("initialize") { () -> Bool in do { if self.audioEngine != nil { @@ -127,14 +140,60 @@ public class ExpoTwoWayAudioModule: Module { ) } + AsyncFunction("speechAvailability") { (locale: String) async -> [String: Any] in + guard #available(iOS 26.0, *) else { + return ["status": "unsupported", "locale": NSNull()] + } + return await OnDeviceSpeechController.availability(locale: locale) + } + + AsyncFunction("speechDownload") { (locale: String) async -> Bool in + guard #available(iOS 26.0, *), let speech = self.onDeviceSpeech() else { + return false + } + return await speech.download(locale: locale) + } + + AsyncFunction("speechStart") { (locale: String, sampleRate: Double) async -> [String: Any] in + guard #available(iOS 26.0, *), let speech = self.onDeviceSpeech() else { + return ["started": false, "reason": "unsupported"] + } + return await speech.start(locale: locale, sampleRate: sampleRate) + } + + Function("speechFeed") { (pcm: Data) in + guard #available(iOS 26.0, *) else { return } + self.onDeviceSpeech()?.feed(pcm) + } + + AsyncFunction("speechFinish") { () async throws -> String in + guard #available(iOS 26.0, *), let speech = self.onDeviceSpeech() else { + throw Exception(name: "OnDeviceSpeechException", description: "On-device speech needs iOS 26") + } + return try await speech.finish() + } + + Function("speechCancel") { + guard #available(iOS 26.0, *) else { return } + self.onDeviceSpeech()?.cancel() + } + Events([ ON_MIC_DATA_EVENT_NAME, ON_INPUT_VOLUME_LEVEL_EVENT_NAME, ON_RECORDING_CHANGE_EVENT_NAME, ON_AUDIO_INTERRUPTION_EVENT_NAME, + ON_SPEECH_TEXT_EVENT_NAME, + ON_SPEECH_ERROR_EVENT_NAME, + ON_SPEECH_DOWNLOAD_PROGRESS_EVENT_NAME, ]) } + @available(iOS 26.0, *) + private func onDeviceSpeech() -> OnDeviceSpeechController? { + speech as? OnDeviceSpeechController + } + private func setupMicrophoneCallback() { audioEngine?.onMicDataCallback = { [weak self] data in self?.sendEvent( diff --git a/mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechController.swift b/mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechController.swift new file mode 100644 index 00000000000..b3fcca46d03 --- /dev/null +++ b/mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechController.swift @@ -0,0 +1,182 @@ +import AVFoundation +import ExpoModulesCore +import Foundation +import Speech + +let ON_SPEECH_TEXT_EVENT_NAME = "onSpeechText" +let ON_SPEECH_ERROR_EVENT_NAME = "onSpeechError" +let ON_SPEECH_DOWNLOAD_PROGRESS_EVENT_NAME = "onSpeechDownloadProgress" + +// Implements the on-device speech contract in src/speech.ts: one live session, fed PCM from JavaScript. +@available(iOS 26.0, *) +final class OnDeviceSpeechController: @unchecked Sendable { + private let lock = NSLock() + private var current: OnDeviceSpeechSession? + // The session speechFinish is draining; speechCancel and a newer speechStart still stop it. + private var finishing: OnDeviceSpeechSession? + // Bumped whenever the live session changes so a stale session's events and start result are dropped. + private var generation = 0 + private let emit: @Sendable (String, [String: Any]) -> Void + + init(emit: @escaping @Sendable (String, [String: Any]) -> Void) { + self.emit = emit + } + + static func availability(locale identifier: String) async -> [String: Any] { + guard let locale = await OnDeviceTranscriber.supportedLocale(for: identifier) else { + return ["status": "unsupported", "locale": NSNull()] + } + switch await OnDeviceSpeechAssets.readiness(for: locale) { + case .ready: + return ["status": "ready", "locale": locale.identifier(.bcp47)] + case .needsDownload: + return ["status": "needs-download", "locale": locale.identifier(.bcp47)] + case .unsupported: + return ["status": "unsupported", "locale": NSNull()] + } + } + + func download(locale identifier: String) async -> Bool { + guard let locale = await OnDeviceTranscriber.supportedLocale(for: identifier) else { + return false + } + let resolved = locale.identifier(.bcp47) + do { + try await OnDeviceSpeechAssets.install(locale) { [emit] fraction in + emit(ON_SPEECH_DOWNLOAD_PROGRESS_EVENT_NAME, ["locale": resolved, "fraction": fraction]) + } + return await OnDeviceSpeechAssets.readiness(for: locale) == .ready + } catch { + print("Could not install on-device speech assets for \(resolved): \(error)") + return false + } + } + + func start(locale identifier: String, sampleRate: Double) async -> [String: Any] { + guard let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: sampleRate, channels: 1, interleaved: true) else { + return ["started": false, "reason": "failed"] + } + let (session, mine, stale) = lock.withLock { () -> (OnDeviceSpeechSession, Int, [OnDeviceSpeechSession]) in + let stale = detachAll() + let mine = generation + let session = OnDeviceSpeechSession( + locale: identifier, + inputFormat: format, + onText: { [weak self] revision, committed, tentative in + self?.emitIfCurrent( + mine, ON_SPEECH_TEXT_EVENT_NAME, + ["revision": revision, "committed": committed, "tentative": tentative]) + }, + onFailure: { [weak self] error in + self?.fail(mine, error) + }) + current = session + return (session, mine, stale) + } + stale.forEach { $0.cancel() } + + do { + let locale = try await session.ready.value + guard isCurrent(mine) else { + return ["started": false, "reason": "busy"] + } + return ["started": true, "locale": locale.identifier(.bcp47)] + } catch { + if take(mine) != nil { + session.cancel() + } + return ["started": false, "reason": Self.reason(for: error)] + } + } + + func feed(_ pcm: Data) { + lock.withLock { current }?.append(pcm: pcm) + } + + func finish() async throws -> String { + let session = lock.withLock { () -> OnDeviceSpeechSession? in + let session = current + current = nil + finishing = session + generation += 1 + return session + } + guard let session else { + throw Self.speechException("No on-device speech session is running") + } + defer { + lock.withLock { + if finishing === session { + finishing = nil + } + } + } + do { + return try await session.finish() + } catch { + throw Self.speechException("On-device speech recognition failed: \(error)") + } + } + + func cancel() { + let stale = lock.withLock { detachAll() } + stale.forEach { $0.cancel() } + } + + private func fail(_ generation: Int, _ error: Error) { + guard let session = take(generation) else { return } + session.cancel() + print("On-device speech session failed: \(error)") + emit(ON_SPEECH_ERROR_EVENT_NAME, ["code": Self.reason(for: error)]) + } + + // Emits under the lock (sendEvent only schedules) so a cancel or start can't slip in after the check. + private func emitIfCurrent(_ generation: Int, _ name: String, _ body: [String: Any]) { + lock.withLock { + if self.generation == generation { + emit(name, body) + } + } + } + + private func isCurrent(_ generation: Int) -> Bool { + lock.withLock { self.generation == generation } + } + + // Detaches the live session if it is still the one from `generation`. + private func take(_ generation: Int) -> OnDeviceSpeechSession? { + lock.withLock { + guard let session = current, generation == self.generation else { + return nil + } + current = nil + self.generation += 1 + return session + } + } + + + // Call inside `lock.withLock`; the caller cancels the returned sessions after unlocking. + private func detachAll() -> [OnDeviceSpeechSession] { + let stale = [current, finishing].compactMap { $0 } + current = nil + finishing = nil + generation += 1 + return stale + } + + private static func speechException(_ description: String) -> Exception { + Exception(name: "OnDeviceSpeechException", description: description) + } + + private static func reason(for error: Error) -> String { + switch error as? OnDeviceSpeechError { + case .unsupported: + return "unsupported" + case .needsDownload: + return "needs-download" + case .audioFormat, .none: + return "failed" + } + } +} diff --git a/mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechSession.swift b/mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechSession.swift new file mode 100644 index 00000000000..ac668819e68 --- /dev/null +++ b/mobile/packages/expo-two-way-audio/ios/OnDeviceSpeechSession.swift @@ -0,0 +1,305 @@ +import AVFoundation +import Foundation +import Speech + +enum OnDeviceSpeechError: Error { + case unsupported + case needsDownload + case audioFormat +} + +// SpeechTranscriber where the hardware runs it; elsewhere (older iPhones, the simulator) the keyboard dictation model. +@available(iOS 26.0, macOS 26.0, *) +enum OnDeviceTranscriber { + case speech(SpeechTranscriber) + case dictation(DictationTranscriber) + + init(locale: Locale) { + if SpeechTranscriber.isAvailable { + self = .speech(SpeechTranscriber(locale: locale, preset: .progressiveTranscription)) + } else { + self = .dictation(DictationTranscriber(locale: locale, preset: .progressiveLongDictation)) + } + } + + // nil when this language can't be transcribed on device. Picks deterministically: supportedLocale(equivalentTo:) + // answers a different region on each call for "en" or an unsupported region such as "en-GH". + static func supportedLocale(for identifier: String) async -> Locale? { + let requested = Locale(identifier: identifier) + let language = maximal(requested) + let supported = SpeechTranscriber.isAvailable + ? await SpeechTranscriber.supportedLocales + : await DictationTranscriber.supportedLocales + let sameLanguage = supported.filter { sameLanguageAndScript(maximal($0), language) } + .sorted { $0.identifier(.bcp47) < $1.identifier(.bcp47) } + // The requested region, then the regions of the phone's own languages and its region, then the language's usual one. + let phoneRegions = Locale.preferredLanguages.map(Locale.init(identifier:)) + .filter { sameLanguageAndScript(maximal($0), language) } + .map(\.region) + let regions = [requested.region] + phoneRegions + [Locale.current.region, language.region] + for region in regions.compactMap({ $0 }) { + if let match = sameLanguage.first(where: { $0.region == region }) { + return match + } + } + return sameLanguage.first + } + + // Fills in the likely script and region, so "zh-TW" compares as Hant and "en" carries US. + private static func maximal(_ locale: Locale) -> Locale.Language { + Locale.Language(identifier: locale.language.maximalIdentifier) + } + + private static func sameLanguageAndScript(_ a: Locale.Language, _ b: Locale.Language) -> Bool { + a.languageCode == b.languageCode && a.script == b.script + } + + var module: any SpeechModule { + switch self { + case .speech(let transcriber): transcriber + case .dictation(let transcriber): transcriber + } + } + + // Collects results until the analyzer finishes; resolves with every finalized word. + func collect( + onText: @escaping @Sendable (_ revision: Int, _ committed: String, _ tentative: String) -> Void, + onFailure: @escaping @Sendable (Error) -> Void + ) -> Task { + switch self { + case .speech(let transcriber): + Self.collect(transcriber.results, text: { $0.text }, onText: onText, onFailure: onFailure) + case .dictation(let transcriber): + Self.collect(transcriber.results, text: { $0.text }, onText: onText, onFailure: onFailure) + } + } + + private static func collect( + _ results: Results, + text: @escaping @Sendable (Results.Element) -> AttributedString, + onText: @escaping @Sendable (Int, String, String) -> Void, + onFailure: @escaping @Sendable (Error) -> Void + ) -> Task where Results.Element: SpeechModuleResult { + Task { + var committed = "" + var revision = 0 + do { + for try await result in results { + let words = String(text(result).characters) + revision += 1 + if result.isFinal { + committed += words + onText(revision, committed, "") + } else { + onText(revision, committed, words) + } + } + } catch { + if !Task.isCancelled { + onFailure(error) + } + throw error + } + return committed + } + } +} + +// Language assets for the on-device transcriber. +@available(iOS 26.0, macOS 26.0, *) +enum OnDeviceSpeechAssets { + enum Readiness { + case ready + case needsDownload + case unsupported + } + + static func readiness(for locale: Locale) async -> Readiness { + let module = OnDeviceTranscriber(locale: locale).module + switch await AssetInventory.status(forModules: [module]) { + case .installed: + // The simulator installs the assets but has no audio format it can run them with ("Model not available"). + return await module.availableCompatibleAudioFormats.isEmpty ? .unsupported : .ready + case .supported, .downloading: + return .needsDownload + default: + return .unsupported + } + } + + static func install(_ locale: Locale, onProgress: @escaping @Sendable (Double) -> Void) async throws { + let transcriber = OnDeviceTranscriber(locale: locale).module + if await AssetInventory.status(forModules: [transcriber]) == .unsupported { + throw OnDeviceSpeechError.unsupported + } + try await reserve(locale) + // A nil request means the assets are already installed. + guard let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) else { + return + } + let observation = request.progress.observe(\.fractionCompleted, options: [.initial, .new]) { progress, _ in + onProgress(progress.fractionCompleted) + } + defer { observation.invalidate() } + try await request.downloadAndInstall() + } + + // The system only downloads, and keeps, assets for locales the app has reserved, up to a per-app limit. + private static func reserve(_ locale: Locale) async throws { + let reserved = await AssetInventory.reservedLocales + if reserved.contains(where: { $0.identifier(.bcp47) == locale.identifier(.bcp47) }) { + return + } + // At the limit, give up another reservation rather than fail this download, a variant of this language first. + if reserved.count >= AssetInventory.maximumReservedLocales, + let other = reserved.first(where: { $0.language.languageCode == locale.language.languageCode }) ?? reserved.first { + await AssetInventory.release(reservedLocale: other) + } + try await AssetInventory.reserve(locale: locale) + } +} + +// One dictation: 16-bit PCM in, committed and tentative text out. +@available(iOS 26.0, macOS 26.0, *) +final class OnDeviceSpeechSession: @unchecked Sendable { + // Resolves to the transcriber's locale once its assets are known to be installed. + let ready: Task + private let inputFormat: AVAudioFormat + private let input: AsyncStream.Continuation + private let run: Task + + init( + locale identifier: String, + inputFormat: AVAudioFormat, + onText: @escaping @Sendable (_ revision: Int, _ committed: String, _ tentative: String) -> Void, + onFailure: @escaping @Sendable (Error) -> Void + ) { + self.inputFormat = inputFormat + // Unbounded so audio fed while the model loads waits here instead of being dropped. + let (stream, input) = AsyncStream.makeStream(of: AVAudioPCMBuffer.self) + self.input = input + let ready = Task { () throws -> Locale in + guard let locale = await OnDeviceTranscriber.supportedLocale(for: identifier) else { + throw OnDeviceSpeechError.unsupported + } + switch await OnDeviceSpeechAssets.readiness(for: locale) { + case .ready: + return locale + case .needsDownload: + throw OnDeviceSpeechError.needsDownload + case .unsupported: + throw OnDeviceSpeechError.unsupported + } + } + self.ready = ready + run = Task { + try await Self.transcribe( + stream, locale: ready.value, inputFormat: inputFormat, onText: onText, onFailure: onFailure) + } + } + + // Copies little-endian 16-bit mono samples; called on the JavaScript thread. + func append(pcm: Data) { + let frames = AVAudioFrameCount(pcm.count / MemoryLayout.size) + guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: frames), + let samples = buffer.int16ChannelData?[0] + else { + return + } + buffer.frameLength = frames + _ = pcm.copyBytes(to: UnsafeMutableBufferPointer(start: samples, count: Int(frames))) + input.yield(buffer) + } + + func finish() async throws -> String { + input.finish() + return try await run.value + } + + func cancel() { + input.finish() + run.cancel() + } + + private static func transcribe( + _ stream: AsyncStream, + locale: Locale, + inputFormat: AVAudioFormat, + onText: @escaping @Sendable (Int, String, String) -> Void, + onFailure: @escaping @Sendable (Error) -> Void + ) async throws -> String { + try Task.checkCancellation() + let transcriber = OnDeviceTranscriber(locale: locale) + guard + let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat( + compatibleWith: [transcriber.module], considering: inputFormat) + else { + onFailure(OnDeviceSpeechError.audioFormat) + throw OnDeviceSpeechError.audioFormat + } + // Convert only when the analyzer wants something other than the fed format. + var converter: AVAudioConverter? + if analyzerFormat != inputFormat { + guard let resampler = AVAudioConverter(from: inputFormat, to: analyzerFormat) else { + onFailure(OnDeviceSpeechError.audioFormat) + throw OnDeviceSpeechError.audioFormat + } + // Keep every input frame; the default priming drops the start of each converted buffer. + resampler.primeMethod = .none + converter = resampler + } + + let collector = transcriber.collect(onText: onText, onFailure: onFailure) + let (analyzerStream, analyzerInput) = AsyncStream.makeStream(of: AnalyzerInput.self) + let analyzer = SpeechAnalyzer(modules: [transcriber.module]) + do { + try await analyzer.start(inputSequence: analyzerStream) + for await buffer in stream { + // A cancelled stream still yields its backlog; don't convert audio nobody will read. + if Task.isCancelled { break } + let ready = if let converter { convert(buffer, with: converter) } else { Optional(buffer) } + if let ready { + analyzerInput.yield(AnalyzerInput(buffer: ready)) + } + } + try Task.checkCancellation() + analyzerInput.finish() + try await analyzer.finalizeAndFinishThroughEndOfInput() + // Results arrive asynchronously; returning before the collector drains loses the last words. + return try await collector.value + } catch { + // The controller ignores this once finish or cancel has detached the session. + if !Task.isCancelled { + onFailure(error) + } + analyzerInput.finish() + await analyzer.cancelAndFinishNow() + collector.cancel() + _ = try? await collector.value + throw error + } + } + + private static func convert(_ buffer: AVAudioPCMBuffer, with converter: AVAudioConverter) -> AVAudioPCMBuffer? { + let ratio = converter.outputFormat.sampleRate / buffer.format.sampleRate + let capacity = AVAudioFrameCount((Double(buffer.frameLength) * ratio).rounded(.up)) + guard capacity > 0, let output = AVAudioPCMBuffer(pcmFormat: converter.outputFormat, frameCapacity: capacity) else { + return nil + } + var consumed = false + var error: NSError? + let status = converter.convert(to: output, error: &error) { _, inputStatus in + if consumed { + inputStatus.pointee = .noDataNow + return nil + } + consumed = true + inputStatus.pointee = .haveData + return buffer + } + guard status != .error, output.frameLength > 0 else { + return nil + } + return output + } +}