feat(mobile): transcribe dictation on iPhone with Apple's on-device speech model (#39410)

* feat(mobile): transcribe dictation on device with SpeechTranscriber on iOS 26

* fix(mobile): fall back to the dictation model, reserve locales and stop a finishing session on cancel

* fix(mobile): resolve on-device speech locales deterministically and report the simulator unsupported

* refactor(mobile): simplify the on-device speech engine and skip conversion when formats match
This commit is contained in:
Kelvin Amoaba
2026-11-07 09:51:50 +00:00
committed by GitHub
parent e1379b34f3
commit 1283e070e3
3 changed files with 547 additions and 1 deletions
@@ -8,10 +8,13 @@ let ON_AUDIO_INTERRUPTION_EVENT_NAME = "onAudioInterruption"
public class ExpoTwoWayAudioModule: Module {
private var audioEngine: AudioEngine?
// OnDeviceSpeechController on iOS 26 and later; nil before, where every speech function answers unsupported.
private var speech: AnyObject?
public func definition() -> ModuleDefinition {
Name("ExpoTwoWayAudio")
OnCreate {
OnCreate { [weak self] in
guard let self else { return }
let permissionsManager = self.appContext?.permissions
EXPermissionsMethodsDelegate.register(
[
@@ -19,9 +22,19 @@ public class ExpoTwoWayAudioModule: Module {
],
withPermissionsManager: permissionsManager
)
if #available(iOS 26.0, *) {
self.speech = OnDeviceSpeechController { [weak self] name, body in
self?.sendEvent(name, body)
}
}
}
OnDestroy {
guard #available(iOS 26.0, *) else { return }
self.onDeviceSpeech()?.cancel()
}
AsyncFunction("initialize") { () -> Bool in
do {
if self.audioEngine != nil {
@@ -127,14 +140,60 @@ public class ExpoTwoWayAudioModule: Module {
)
}
AsyncFunction("speechAvailability") { (locale: String) async -> [String: Any] in
guard #available(iOS 26.0, *) else {
return ["status": "unsupported", "locale": NSNull()]
}
return await OnDeviceSpeechController.availability(locale: locale)
}
AsyncFunction("speechDownload") { (locale: String) async -> Bool in
guard #available(iOS 26.0, *), let speech = self.onDeviceSpeech() else {
return false
}
return await speech.download(locale: locale)
}
AsyncFunction("speechStart") { (locale: String, sampleRate: Double) async -> [String: Any] in
guard #available(iOS 26.0, *), let speech = self.onDeviceSpeech() else {
return ["started": false, "reason": "unsupported"]
}
return await speech.start(locale: locale, sampleRate: sampleRate)
}
Function("speechFeed") { (pcm: Data) in
guard #available(iOS 26.0, *) else { return }
self.onDeviceSpeech()?.feed(pcm)
}
AsyncFunction("speechFinish") { () async throws -> String in
guard #available(iOS 26.0, *), let speech = self.onDeviceSpeech() else {
throw Exception(name: "OnDeviceSpeechException", description: "On-device speech needs iOS 26")
}
return try await speech.finish()
}
Function("speechCancel") {
guard #available(iOS 26.0, *) else { return }
self.onDeviceSpeech()?.cancel()
}
Events([
ON_MIC_DATA_EVENT_NAME,
ON_INPUT_VOLUME_LEVEL_EVENT_NAME,
ON_RECORDING_CHANGE_EVENT_NAME,
ON_AUDIO_INTERRUPTION_EVENT_NAME,
ON_SPEECH_TEXT_EVENT_NAME,
ON_SPEECH_ERROR_EVENT_NAME,
ON_SPEECH_DOWNLOAD_PROGRESS_EVENT_NAME,
])
}
@available(iOS 26.0, *)
private func onDeviceSpeech() -> OnDeviceSpeechController? {
speech as? OnDeviceSpeechController
}
private func setupMicrophoneCallback() {
audioEngine?.onMicDataCallback = { [weak self] data in
self?.sendEvent(
@@ -0,0 +1,182 @@
import AVFoundation
import ExpoModulesCore
import Foundation
import Speech
let ON_SPEECH_TEXT_EVENT_NAME = "onSpeechText"
let ON_SPEECH_ERROR_EVENT_NAME = "onSpeechError"
let ON_SPEECH_DOWNLOAD_PROGRESS_EVENT_NAME = "onSpeechDownloadProgress"
// Implements the on-device speech contract in src/speech.ts: one live session, fed PCM from JavaScript.
@available(iOS 26.0, *)
final class OnDeviceSpeechController: @unchecked Sendable {
private let lock = NSLock()
private var current: OnDeviceSpeechSession?
// The session speechFinish is draining; speechCancel and a newer speechStart still stop it.
private var finishing: OnDeviceSpeechSession?
// Bumped whenever the live session changes so a stale session's events and start result are dropped.
private var generation = 0
private let emit: @Sendable (String, [String: Any]) -> Void
init(emit: @escaping @Sendable (String, [String: Any]) -> Void) {
self.emit = emit
}
static func availability(locale identifier: String) async -> [String: Any] {
guard let locale = await OnDeviceTranscriber.supportedLocale(for: identifier) else {
return ["status": "unsupported", "locale": NSNull()]
}
switch await OnDeviceSpeechAssets.readiness(for: locale) {
case .ready:
return ["status": "ready", "locale": locale.identifier(.bcp47)]
case .needsDownload:
return ["status": "needs-download", "locale": locale.identifier(.bcp47)]
case .unsupported:
return ["status": "unsupported", "locale": NSNull()]
}
}
func download(locale identifier: String) async -> Bool {
guard let locale = await OnDeviceTranscriber.supportedLocale(for: identifier) else {
return false
}
let resolved = locale.identifier(.bcp47)
do {
try await OnDeviceSpeechAssets.install(locale) { [emit] fraction in
emit(ON_SPEECH_DOWNLOAD_PROGRESS_EVENT_NAME, ["locale": resolved, "fraction": fraction])
}
return await OnDeviceSpeechAssets.readiness(for: locale) == .ready
} catch {
print("Could not install on-device speech assets for \(resolved): \(error)")
return false
}
}
func start(locale identifier: String, sampleRate: Double) async -> [String: Any] {
guard let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: sampleRate, channels: 1, interleaved: true) else {
return ["started": false, "reason": "failed"]
}
let (session, mine, stale) = lock.withLock { () -> (OnDeviceSpeechSession, Int, [OnDeviceSpeechSession]) in
let stale = detachAll()
let mine = generation
let session = OnDeviceSpeechSession(
locale: identifier,
inputFormat: format,
onText: { [weak self] revision, committed, tentative in
self?.emitIfCurrent(
mine, ON_SPEECH_TEXT_EVENT_NAME,
["revision": revision, "committed": committed, "tentative": tentative])
},
onFailure: { [weak self] error in
self?.fail(mine, error)
})
current = session
return (session, mine, stale)
}
stale.forEach { $0.cancel() }
do {
let locale = try await session.ready.value
guard isCurrent(mine) else {
return ["started": false, "reason": "busy"]
}
return ["started": true, "locale": locale.identifier(.bcp47)]
} catch {
if take(mine) != nil {
session.cancel()
}
return ["started": false, "reason": Self.reason(for: error)]
}
}
func feed(_ pcm: Data) {
lock.withLock { current }?.append(pcm: pcm)
}
func finish() async throws -> String {
let session = lock.withLock { () -> OnDeviceSpeechSession? in
let session = current
current = nil
finishing = session
generation += 1
return session
}
guard let session else {
throw Self.speechException("No on-device speech session is running")
}
defer {
lock.withLock {
if finishing === session {
finishing = nil
}
}
}
do {
return try await session.finish()
} catch {
throw Self.speechException("On-device speech recognition failed: \(error)")
}
}
func cancel() {
let stale = lock.withLock { detachAll() }
stale.forEach { $0.cancel() }
}
private func fail(_ generation: Int, _ error: Error) {
guard let session = take(generation) else { return }
session.cancel()
print("On-device speech session failed: \(error)")
emit(ON_SPEECH_ERROR_EVENT_NAME, ["code": Self.reason(for: error)])
}
// Emits under the lock (sendEvent only schedules) so a cancel or start can't slip in after the check.
private func emitIfCurrent(_ generation: Int, _ name: String, _ body: [String: Any]) {
lock.withLock {
if self.generation == generation {
emit(name, body)
}
}
}
private func isCurrent(_ generation: Int) -> Bool {
lock.withLock { self.generation == generation }
}
// Detaches the live session if it is still the one from `generation`.
private func take(_ generation: Int) -> OnDeviceSpeechSession? {
lock.withLock {
guard let session = current, generation == self.generation else {
return nil
}
current = nil
self.generation += 1
return session
}
}
// Call inside `lock.withLock`; the caller cancels the returned sessions after unlocking.
private func detachAll() -> [OnDeviceSpeechSession] {
let stale = [current, finishing].compactMap { $0 }
current = nil
finishing = nil
generation += 1
return stale
}
private static func speechException(_ description: String) -> Exception {
Exception(name: "OnDeviceSpeechException", description: description)
}
private static func reason(for error: Error) -> String {
switch error as? OnDeviceSpeechError {
case .unsupported:
return "unsupported"
case .needsDownload:
return "needs-download"
case .audioFormat, .none:
return "failed"
}
}
}
@@ -0,0 +1,305 @@
import AVFoundation
import Foundation
import Speech
enum OnDeviceSpeechError: Error {
case unsupported
case needsDownload
case audioFormat
}
// SpeechTranscriber where the hardware runs it; elsewhere (older iPhones, the simulator) the keyboard dictation model.
@available(iOS 26.0, macOS 26.0, *)
enum OnDeviceTranscriber {
case speech(SpeechTranscriber)
case dictation(DictationTranscriber)
init(locale: Locale) {
if SpeechTranscriber.isAvailable {
self = .speech(SpeechTranscriber(locale: locale, preset: .progressiveTranscription))
} else {
self = .dictation(DictationTranscriber(locale: locale, preset: .progressiveLongDictation))
}
}
// nil when this language can't be transcribed on device. Picks deterministically: supportedLocale(equivalentTo:)
// answers a different region on each call for "en" or an unsupported region such as "en-GH".
static func supportedLocale(for identifier: String) async -> Locale? {
let requested = Locale(identifier: identifier)
let language = maximal(requested)
let supported = SpeechTranscriber.isAvailable
? await SpeechTranscriber.supportedLocales
: await DictationTranscriber.supportedLocales
let sameLanguage = supported.filter { sameLanguageAndScript(maximal($0), language) }
.sorted { $0.identifier(.bcp47) < $1.identifier(.bcp47) }
// The requested region, then the regions of the phone's own languages and its region, then the language's usual one.
let phoneRegions = Locale.preferredLanguages.map(Locale.init(identifier:))
.filter { sameLanguageAndScript(maximal($0), language) }
.map(\.region)
let regions = [requested.region] + phoneRegions + [Locale.current.region, language.region]
for region in regions.compactMap({ $0 }) {
if let match = sameLanguage.first(where: { $0.region == region }) {
return match
}
}
return sameLanguage.first
}
// Fills in the likely script and region, so "zh-TW" compares as Hant and "en" carries US.
private static func maximal(_ locale: Locale) -> Locale.Language {
Locale.Language(identifier: locale.language.maximalIdentifier)
}
private static func sameLanguageAndScript(_ a: Locale.Language, _ b: Locale.Language) -> Bool {
a.languageCode == b.languageCode && a.script == b.script
}
var module: any SpeechModule {
switch self {
case .speech(let transcriber): transcriber
case .dictation(let transcriber): transcriber
}
}
// Collects results until the analyzer finishes; resolves with every finalized word.
func collect(
onText: @escaping @Sendable (_ revision: Int, _ committed: String, _ tentative: String) -> Void,
onFailure: @escaping @Sendable (Error) -> Void
) -> Task<String, Error> {
switch self {
case .speech(let transcriber):
Self.collect(transcriber.results, text: { $0.text }, onText: onText, onFailure: onFailure)
case .dictation(let transcriber):
Self.collect(transcriber.results, text: { $0.text }, onText: onText, onFailure: onFailure)
}
}
private static func collect<Results: AsyncSequence & Sendable>(
_ results: Results,
text: @escaping @Sendable (Results.Element) -> AttributedString,
onText: @escaping @Sendable (Int, String, String) -> Void,
onFailure: @escaping @Sendable (Error) -> Void
) -> Task<String, Error> where Results.Element: SpeechModuleResult {
Task {
var committed = ""
var revision = 0
do {
for try await result in results {
let words = String(text(result).characters)
revision += 1
if result.isFinal {
committed += words
onText(revision, committed, "")
} else {
onText(revision, committed, words)
}
}
} catch {
if !Task.isCancelled {
onFailure(error)
}
throw error
}
return committed
}
}
}
// Language assets for the on-device transcriber.
@available(iOS 26.0, macOS 26.0, *)
enum OnDeviceSpeechAssets {
enum Readiness {
case ready
case needsDownload
case unsupported
}
static func readiness(for locale: Locale) async -> Readiness {
let module = OnDeviceTranscriber(locale: locale).module
switch await AssetInventory.status(forModules: [module]) {
case .installed:
// The simulator installs the assets but has no audio format it can run them with ("Model not available").
return await module.availableCompatibleAudioFormats.isEmpty ? .unsupported : .ready
case .supported, .downloading:
return .needsDownload
default:
return .unsupported
}
}
static func install(_ locale: Locale, onProgress: @escaping @Sendable (Double) -> Void) async throws {
let transcriber = OnDeviceTranscriber(locale: locale).module
if await AssetInventory.status(forModules: [transcriber]) == .unsupported {
throw OnDeviceSpeechError.unsupported
}
try await reserve(locale)
// A nil request means the assets are already installed.
guard let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) else {
return
}
let observation = request.progress.observe(\.fractionCompleted, options: [.initial, .new]) { progress, _ in
onProgress(progress.fractionCompleted)
}
defer { observation.invalidate() }
try await request.downloadAndInstall()
}
// The system only downloads, and keeps, assets for locales the app has reserved, up to a per-app limit.
private static func reserve(_ locale: Locale) async throws {
let reserved = await AssetInventory.reservedLocales
if reserved.contains(where: { $0.identifier(.bcp47) == locale.identifier(.bcp47) }) {
return
}
// At the limit, give up another reservation rather than fail this download, a variant of this language first.
if reserved.count >= AssetInventory.maximumReservedLocales,
let other = reserved.first(where: { $0.language.languageCode == locale.language.languageCode }) ?? reserved.first {
await AssetInventory.release(reservedLocale: other)
}
try await AssetInventory.reserve(locale: locale)
}
}
// One dictation: 16-bit PCM in, committed and tentative text out.
@available(iOS 26.0, macOS 26.0, *)
final class OnDeviceSpeechSession: @unchecked Sendable {
// Resolves to the transcriber's locale once its assets are known to be installed.
let ready: Task<Locale, Error>
private let inputFormat: AVAudioFormat
private let input: AsyncStream<AVAudioPCMBuffer>.Continuation
private let run: Task<String, Error>
init(
locale identifier: String,
inputFormat: AVAudioFormat,
onText: @escaping @Sendable (_ revision: Int, _ committed: String, _ tentative: String) -> Void,
onFailure: @escaping @Sendable (Error) -> Void
) {
self.inputFormat = inputFormat
// Unbounded so audio fed while the model loads waits here instead of being dropped.
let (stream, input) = AsyncStream.makeStream(of: AVAudioPCMBuffer.self)
self.input = input
let ready = Task { () throws -> Locale in
guard let locale = await OnDeviceTranscriber.supportedLocale(for: identifier) else {
throw OnDeviceSpeechError.unsupported
}
switch await OnDeviceSpeechAssets.readiness(for: locale) {
case .ready:
return locale
case .needsDownload:
throw OnDeviceSpeechError.needsDownload
case .unsupported:
throw OnDeviceSpeechError.unsupported
}
}
self.ready = ready
run = Task {
try await Self.transcribe(
stream, locale: ready.value, inputFormat: inputFormat, onText: onText, onFailure: onFailure)
}
}
// Copies little-endian 16-bit mono samples; called on the JavaScript thread.
func append(pcm: Data) {
let frames = AVAudioFrameCount(pcm.count / MemoryLayout<Int16>.size)
guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: frames),
let samples = buffer.int16ChannelData?[0]
else {
return
}
buffer.frameLength = frames
_ = pcm.copyBytes(to: UnsafeMutableBufferPointer(start: samples, count: Int(frames)))
input.yield(buffer)
}
func finish() async throws -> String {
input.finish()
return try await run.value
}
func cancel() {
input.finish()
run.cancel()
}
private static func transcribe(
_ stream: AsyncStream<AVAudioPCMBuffer>,
locale: Locale,
inputFormat: AVAudioFormat,
onText: @escaping @Sendable (Int, String, String) -> Void,
onFailure: @escaping @Sendable (Error) -> Void
) async throws -> String {
try Task.checkCancellation()
let transcriber = OnDeviceTranscriber(locale: locale)
guard
let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(
compatibleWith: [transcriber.module], considering: inputFormat)
else {
onFailure(OnDeviceSpeechError.audioFormat)
throw OnDeviceSpeechError.audioFormat
}
// Convert only when the analyzer wants something other than the fed format.
var converter: AVAudioConverter?
if analyzerFormat != inputFormat {
guard let resampler = AVAudioConverter(from: inputFormat, to: analyzerFormat) else {
onFailure(OnDeviceSpeechError.audioFormat)
throw OnDeviceSpeechError.audioFormat
}
// Keep every input frame; the default priming drops the start of each converted buffer.
resampler.primeMethod = .none
converter = resampler
}
let collector = transcriber.collect(onText: onText, onFailure: onFailure)
let (analyzerStream, analyzerInput) = AsyncStream.makeStream(of: AnalyzerInput.self)
let analyzer = SpeechAnalyzer(modules: [transcriber.module])
do {
try await analyzer.start(inputSequence: analyzerStream)
for await buffer in stream {
// A cancelled stream still yields its backlog; don't convert audio nobody will read.
if Task.isCancelled { break }
let ready = if let converter { convert(buffer, with: converter) } else { Optional(buffer) }
if let ready {
analyzerInput.yield(AnalyzerInput(buffer: ready))
}
}
try Task.checkCancellation()
analyzerInput.finish()
try await analyzer.finalizeAndFinishThroughEndOfInput()
// Results arrive asynchronously; returning before the collector drains loses the last words.
return try await collector.value
} catch {
// The controller ignores this once finish or cancel has detached the session.
if !Task.isCancelled {
onFailure(error)
}
analyzerInput.finish()
await analyzer.cancelAndFinishNow()
collector.cancel()
_ = try? await collector.value
throw error
}
}
private static func convert(_ buffer: AVAudioPCMBuffer, with converter: AVAudioConverter) -> AVAudioPCMBuffer? {
let ratio = converter.outputFormat.sampleRate / buffer.format.sampleRate
let capacity = AVAudioFrameCount((Double(buffer.frameLength) * ratio).rounded(.up))
guard capacity > 0, let output = AVAudioPCMBuffer(pcmFormat: converter.outputFormat, frameCapacity: capacity) else {
return nil
}
var consumed = false
var error: NSError?
let status = converter.convert(to: output, error: &error) { _, inputStatus in
if consumed {
inputStatus.pointee = .noDataNow
return nil
}
consumed = true
inputStatus.pointee = .haveData
return buffer
}
guard status != .error, output.frameLength > 0 else {
return nil
}
return output
}
}