mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
refactor(speech): remove sherpa-onnx
transcribe-cpp is now the only on-device engine on every platform, so the sherpa-onnx packages, their packaging (extraResources, the darwin libonnxruntime prune, the native-variant assert and the Linux libstdc++ exemption), the sherpa recognizer in the speech process and the per-model engine choice all go. Hotwords are written as plain terms.
This commit is contained in:
@@ -174,19 +174,6 @@ const commonExtraResources = [
|
||||
nativeChatVisualsResource,
|
||||
emojiShortcodeDatasetResource
|
||||
]
|
||||
// Native speech addons need real disk paths; copy only the artifact target’s variant.
|
||||
const macSpeechNativeResource = {
|
||||
from: 'node_modules/sherpa-onnx-darwin-${arch}',
|
||||
to: 'node_modules/sherpa-onnx-darwin-${arch}'
|
||||
}
|
||||
const linuxSpeechNativeResource = {
|
||||
from: 'node_modules/sherpa-onnx-linux-${arch}',
|
||||
to: 'node_modules/sherpa-onnx-linux-${arch}'
|
||||
}
|
||||
const winSpeechNativeResource = {
|
||||
from: 'node_modules/sherpa-onnx-win-x64',
|
||||
to: 'node_modules/sherpa-onnx-win-x64'
|
||||
}
|
||||
// electron-builder replaces these defaults when `depends` is configured; retain
|
||||
// Electron's loader requirements alongside Orca's headless-host dependencies.
|
||||
const debElectronRuntimeDependencies = [
|
||||
@@ -332,8 +319,8 @@ module.exports = {
|
||||
'!resources/native-chat-visuals/**',
|
||||
// extraResources already ships plugin bytes; exclude source duplicates.
|
||||
'!resources/plugins/launch/**',
|
||||
// Platform extraResources selects speech natives; asar would duplicate it and ship other variants.
|
||||
'!node_modules/{sherpa-onnx*,transcribe-cpp,@transcribe-cpp,koffi,@koromix}{,/**/*}',
|
||||
// afterPack copies the target's speech engine to resources/node_modules; asar would duplicate it and ship other variants.
|
||||
'!node_modules/{transcribe-cpp,@transcribe-cpp,koffi,@koromix}{,/**/*}',
|
||||
// CLI shim belongs beside resources/bin/orca.exe; unpacking source creates a second .cmd without its executable (#7351).
|
||||
'!resources/win32{,/**/*}'
|
||||
],
|
||||
@@ -563,7 +550,6 @@ module.exports = {
|
||||
extraResources: [
|
||||
...commonExtraResources,
|
||||
...windowsRuntimeResources,
|
||||
winSpeechNativeResource,
|
||||
{
|
||||
from: 'resources/win32/bin/orca.cmd',
|
||||
to: 'bin/orca.cmd'
|
||||
@@ -620,7 +606,8 @@ module.exports = {
|
||||
rank: 'Alternate'
|
||||
}))
|
||||
],
|
||||
// Bundled ONNX Runtime (speech) needs macOS 13.4 (#22690); the update-manifest gate reads this from Info.plist.
|
||||
// Set for ONNX Runtime (#22690, #39135), now gone; transcribe-cpp needs 11.0, Electron 12.0. Kept until
|
||||
// Kelvin decides. The update-manifest gate reads this from Info.plist.
|
||||
minimumSystemVersion: '13.4',
|
||||
icon: 'resources/build/icon.icns',
|
||||
entitlements: 'resources/build/entitlements.mac.plist',
|
||||
@@ -643,7 +630,6 @@ module.exports = {
|
||||
extraResources: [
|
||||
...commonExtraResources,
|
||||
...createPackagedRuntimeNodeModuleResources('darwin'),
|
||||
macSpeechNativeResource,
|
||||
{
|
||||
from: 'resources/darwin/bin/orca',
|
||||
to: 'bin/orca'
|
||||
@@ -708,7 +694,6 @@ module.exports = {
|
||||
extraResources: [
|
||||
...commonExtraResources,
|
||||
...createPackagedRuntimeNodeModuleResources('linux'),
|
||||
linuxSpeechNativeResource,
|
||||
{
|
||||
from: 'native/wayland-clipboard/.build/${arch}/orca-wayland-clipboard',
|
||||
to: 'bin/orca-wayland-clipboard'
|
||||
|
||||
@@ -499,7 +499,6 @@
|
||||
1 src/main/speech/speech-model-files.ts
|
||||
2 src/main/speech/speech-model-http-download.ts
|
||||
2 src/main/speech/stt-gpu-failure-marker.ts
|
||||
1 src/main/speech/stt-worker-model-config.ts
|
||||
2 src/main/sqlite/harden-database-files.ts
|
||||
1 src/main/sqlite/row-cache/keyed-json-row-cache.ts
|
||||
2 src/main/sqlite/row-cache/legacy-json-source.ts
|
||||
@@ -610,7 +609,7 @@
|
||||
7 src/shared/serve-bundle-park.ts
|
||||
1 src/shared/serve-user-data-path.ts
|
||||
3 src/shared/source-scan/source-tree-scan.ts
|
||||
3 src/shared/speech-recognizer-package.ts
|
||||
2 src/shared/speech-recognizer-package.ts
|
||||
5 src/shared/startup-command-staging.ts
|
||||
1 src/shared/terminal/pty/native-process/pty-process-ceiling-state.ts
|
||||
8 src/shared/windows-breakaway-launcher.ts
|
||||
|
||||
@@ -82,7 +82,6 @@ const PACKAGED_MAIN_REQUIRED_FILES = [
|
||||
const PACKAGED_MAIN_SOURCE_RE = /^out\/main\/.+\.js$/
|
||||
const TYPE_DECLARATION_ARTIFACT_RE = /\.d\.(?:c|m)?ts(?:\.map)?$/
|
||||
const JS_SOURCE_MAP_ARTIFACT_RE = /\.(?:c|m)?js\.map$/
|
||||
const VERSIONED_ONNXRUNTIME_DYLIB_RE = /^libonnxruntime\.\d[\d.]*\.dylib$/
|
||||
|
||||
const NODE_BUILTINS = new Set([
|
||||
...builtinModules,
|
||||
@@ -519,31 +518,6 @@ function prunePackagedRuntimeTypeAndSourceMapArtifacts(resourcesDir) {
|
||||
pruneMatchingFiles(nodeModulesDir, isPrunableTypeOrSourceMapArtifact)
|
||||
}
|
||||
|
||||
function prunePackagedSherpaOnnx(resourcesDir, electronPlatformName) {
|
||||
if (electronPlatformName !== 'darwin') {
|
||||
return
|
||||
}
|
||||
const nodeModulesDir = join(resourcesDir, 'node_modules')
|
||||
if (!existsSync(nodeModulesDir)) {
|
||||
return
|
||||
}
|
||||
for (const entry of readdirSync(nodeModulesDir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || !entry.name.startsWith('sherpa-onnx-darwin-')) {
|
||||
continue
|
||||
}
|
||||
const packageDir = join(nodeModulesDir, entry.name)
|
||||
const packageEntries = readdirSync(packageDir)
|
||||
const hasVersionedOnnxRuntime = packageEntries.some((filename) =>
|
||||
VERSIONED_ONNXRUNTIME_DYLIB_RE.test(filename)
|
||||
)
|
||||
if (hasVersionedOnnxRuntime) {
|
||||
// Why: darwin sherpa-onnx binaries link to the versioned ONNX Runtime
|
||||
// install name; the unversioned dylib is a duplicate fallback copy.
|
||||
rmSync(join(packageDir, 'libonnxruntime.dylib'), { force: true })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function prunePackagedZodSources(resourcesDir) {
|
||||
// Why: Zod's src tree is TypeScript source only selected by the @zod/source
|
||||
// condition; packaged runtime import/require paths resolve to built JS.
|
||||
@@ -557,17 +531,6 @@ function assertPackagedNativeVariantsInstalled(electronPlatformName, electronArc
|
||||
const isInstalled = (name) => existsSync(join(nodeModulesDir, name, 'package.json'))
|
||||
const missing = []
|
||||
|
||||
const rootOptionalDependencies =
|
||||
JSON.parse(readFileSync(join(projectDir, 'package.json'), 'utf8')).optionalDependencies ?? {}
|
||||
// Why win32 is always x64: winSpeechNativeResource packages sherpa-onnx-win-x64 for every
|
||||
// Windows target (there is no sherpa-onnx-win-arm64; it runs under emulation).
|
||||
const sherpaName =
|
||||
electronPlatformName === 'win32'
|
||||
? 'sherpa-onnx-win-x64'
|
||||
: `sherpa-onnx-${electronPlatformName}-${architecture}`
|
||||
if (sherpaName in rootOptionalDependencies && !isInstalled(sherpaName)) {
|
||||
missing.push(sherpaName)
|
||||
}
|
||||
missing.push(...missingTranscribeCppPackages(readPackage, electronPlatformName, architecture))
|
||||
|
||||
// Why prefix, not equality: linux variants carry a libc suffix (watcher-linux-x64-glibc),
|
||||
@@ -636,7 +599,6 @@ function prunePackagedRuntimeNodeModules(resourcesDir, electronPlatformName, ele
|
||||
// Why before the filename walk: zod/src is deleted wholesale, so walking it first is wasted work.
|
||||
prunePackagedZodSources(resourcesDir)
|
||||
prunePackagedRuntimeTypeAndSourceMapArtifacts(resourcesDir)
|
||||
prunePackagedSherpaOnnx(resourcesDir, electronPlatformName)
|
||||
}
|
||||
|
||||
function pruneMatchingFiles(directory, shouldPrune) {
|
||||
@@ -665,7 +627,6 @@ module.exports = {
|
||||
prunePackagedParcelWatcher,
|
||||
prunePackagedRuntimeNodeModules,
|
||||
prunePackagedRuntimeTypeAndSourceMapArtifacts,
|
||||
prunePackagedSherpaOnnx,
|
||||
prunePackagedZodSources,
|
||||
readPackage,
|
||||
verifyPackagedMainRuntimeDeps
|
||||
|
||||
@@ -43,9 +43,6 @@ const COMPAT_SLOT_GLIBC_FLOOR = glibcFloorProfile(
|
||||
[1, 3, 7]
|
||||
)
|
||||
|
||||
// Lazy speech-worker sherpa prebuilt requires GCC11 GLIBCXX_3.4.29; exempt C++ floor only, retain glibc gate because it is not launch-critical.
|
||||
const LIBSTDCXX_FLOOR_EXEMPT = /(?:^|[/\\])sherpa-onnx/
|
||||
|
||||
// VER_FLG_WEAK needs may resolve null and use fallback; they must not raise required floor.
|
||||
const VER_FLG_WEAK = 0x2
|
||||
|
||||
@@ -118,15 +115,9 @@ function isLibstdcxxNode(name) {
|
||||
return name.startsWith('GLIBCXX_') || name.startsWith('CXXABI_')
|
||||
}
|
||||
|
||||
// Gate load-time requirements; exempt sherpa C++ needs only, never its glibc needs.
|
||||
function findFloorViolations(needs, filePath = '', glibcFloor = DESKTOP_GLIBC_FLOOR) {
|
||||
const exemptLibstdcxx = LIBSTDCXX_FLOOR_EXEMPT.test(filePath)
|
||||
return needs.filter(
|
||||
(need) =>
|
||||
!need.weak &&
|
||||
isVersionNodeAboveFloor(need.name, glibcFloor) &&
|
||||
!(exemptLibstdcxx && isLibstdcxxNode(need.name))
|
||||
)
|
||||
// Gate load-time requirements; weak needs can resolve to null and fall back.
|
||||
function findFloorViolations(needs, glibcFloor = DESKTOP_GLIBC_FLOOR) {
|
||||
return needs.filter((need) => !need.weak && isVersionNodeAboveFloor(need.name, glibcFloor))
|
||||
}
|
||||
|
||||
// Ubuntu20.04 keeps openpty/forkpty in libutil until glibc2.34 merge: require DT_NEEDED even when newer libc compat aliases satisfy link-time floor.
|
||||
@@ -402,7 +393,7 @@ function verifyLinuxGlibcFloor(rootDir, options = {}) {
|
||||
const { versionNeeds, neededLibraries, isStatic } = readDynamicInfo(filePath, objdumpPath)
|
||||
const isMuslTarget = isMuslTemplatePayload(filePath, neededLibraries, versionNeeds)
|
||||
// Remote musl payloads use their host's C++ runtime, not Ubuntu's libstdc++ or libutil.
|
||||
const floorViolations = findFloorViolations(versionNeeds, filePath, glibcFloor).filter(
|
||||
const floorViolations = findFloorViolations(versionNeeds, glibcFloor).filter(
|
||||
(need) => !isMuslTarget || !isLibstdcxxNode(need.name)
|
||||
)
|
||||
// Inspect dynamic symbols only if relocated provider is absent from DT_NEEDED; healthy case avoids extra objdump.
|
||||
|
||||
@@ -398,11 +398,6 @@
|
||||
"@transcribe-cpp/linux-x64-cpu-vulkan": "0.4.1",
|
||||
"@transcribe-cpp/win32-x64-cpu-vulkan": "0.4.1",
|
||||
"@vscode/windows-process-tree": "0.8.0",
|
||||
"sherpa-onnx-darwin-arm64": "1.12.37",
|
||||
"sherpa-onnx-darwin-x64": "1.12.37",
|
||||
"sherpa-onnx-linux-arm64": "1.12.37",
|
||||
"sherpa-onnx-linux-x64": "1.12.37",
|
||||
"sherpa-onnx-win-x64": "1.12.37",
|
||||
"transcribe-cpp": "0.4.1"
|
||||
},
|
||||
"lint-staged": {
|
||||
|
||||
Generated
-55
@@ -720,21 +720,6 @@ importers:
|
||||
'@vscode/windows-process-tree':
|
||||
specifier: 0.8.0
|
||||
version: 0.8.0(patch_hash=a09f2e8b2b4e73ef20c317af91f9e06c09abf62fa2c891d99b16f53047f596fd)
|
||||
sherpa-onnx-darwin-arm64:
|
||||
specifier: 1.12.37
|
||||
version: 1.12.37
|
||||
sherpa-onnx-darwin-x64:
|
||||
specifier: 1.12.37
|
||||
version: 1.12.37
|
||||
sherpa-onnx-linux-arm64:
|
||||
specifier: 1.12.37
|
||||
version: 1.12.37
|
||||
sherpa-onnx-linux-x64:
|
||||
specifier: 1.12.37
|
||||
version: 1.12.37
|
||||
sherpa-onnx-win-x64:
|
||||
specifier: 1.12.37
|
||||
version: 1.12.37
|
||||
transcribe-cpp:
|
||||
specifier: 0.4.1
|
||||
version: 0.4.1
|
||||
@@ -7660,31 +7645,6 @@ packages:
|
||||
resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
sherpa-onnx-darwin-arm64@1.12.37:
|
||||
resolution: {integrity: sha512-zpqbH+2TI6dvg7mxGm30Mnv17aJL3ZfRGshMiBK85dHBfhzqMbKUHNXCzsHhiKBQTzti6JqG1YoRbYrJwvdUjA==}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
|
||||
sherpa-onnx-darwin-x64@1.12.37:
|
||||
resolution: {integrity: sha512-2mB1BhEJJLG8Ow/YGAe7c3AL5fSkk9qDk3wmZRsfDW+Gno5eBaUusXseDI0ltjh2mZcW74l9F19ez4CyhwqNQQ==}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
sherpa-onnx-linux-arm64@1.12.37:
|
||||
resolution: {integrity: sha512-6yVnp7ucK4oJEjUfKbkyvJm+e6UdETTp7PnWFb1+Wr6nxSCN038BGSACCIKlVHEsQl+6huwrvymljkWrIro1mA==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
sherpa-onnx-linux-x64@1.12.37:
|
||||
resolution: {integrity: sha512-WF69clWjTlBu6NIBdL8OLEEN4NwOlYjjB/AjCMs29pB6gtSzKJEKIjmTATfaSSExTGPeXsOriCXcMdHaMH4WNg==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
sherpa-onnx-win-x64@1.12.37:
|
||||
resolution: {integrity: sha512-1h1UrJxzA1vkOO2G8JqRQkF9SfEhM/LYW+yFki5izCY5N+eaaxYKdZLmVZI3KAhxk28pkE6dnMETYi0nj2/Cuw==}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
shiki@4.5.0:
|
||||
resolution: {integrity: sha512-OcRW72ngQvc+MSVWBRIovC/BTW589iUE/Vxav7VeM4RcY9MUjqB+IR6ILG1k9nuW6LiG/nct9uma4/MBtnZbmQ==}
|
||||
engines: {node: '>=20'}
|
||||
@@ -15249,21 +15209,6 @@ snapshots:
|
||||
|
||||
shebang-regex@3.0.0: {}
|
||||
|
||||
sherpa-onnx-darwin-arm64@1.12.37:
|
||||
optional: true
|
||||
|
||||
sherpa-onnx-darwin-x64@1.12.37:
|
||||
optional: true
|
||||
|
||||
sherpa-onnx-linux-arm64@1.12.37:
|
||||
optional: true
|
||||
|
||||
sherpa-onnx-linux-x64@1.12.37:
|
||||
optional: true
|
||||
|
||||
sherpa-onnx-win-x64@1.12.37:
|
||||
optional: true
|
||||
|
||||
shiki@4.5.0:
|
||||
dependencies:
|
||||
'@shikijs/core': 4.5.0
|
||||
|
||||
@@ -22,21 +22,23 @@ function resourcesWith(files: string[]): string {
|
||||
return resources
|
||||
}
|
||||
|
||||
describe("finding the app's speech engines (#31174)", () => {
|
||||
it('answers nothing when the app ships no engine for this platform', () => {
|
||||
describe("finding the app's speech engine (#31174)", () => {
|
||||
it('answers nothing when the app ships no native package for this platform', () => {
|
||||
const resources = resourcesWith([
|
||||
'transcribe-cpp/package.json',
|
||||
'@transcribe-cpp/linux-arm64-cpu-vulkan/contract.json',
|
||||
'sherpa-onnx-linux-arm64/sherpa-onnx.node'
|
||||
'@transcribe-cpp/linux-arm64-cpu-vulkan/contract.json'
|
||||
])
|
||||
expect(
|
||||
findAppSpeechRecognizerDir(join(resources, 'app.asar.unpacked'), 'linux', 'x64')
|
||||
).toBeUndefined()
|
||||
})
|
||||
|
||||
// A loaded addon is locked on Windows; orcad holding the app's copy would block app updates.
|
||||
it('never hands orcad the engines on Windows', () => {
|
||||
const resources = resourcesWith(['sherpa-onnx-win-x64/sherpa-onnx.node'])
|
||||
// A loaded library is locked on Windows; orcad holding the app's copy would block app updates.
|
||||
it('never hands orcad the engine on Windows', () => {
|
||||
const resources = resourcesWith([
|
||||
'transcribe-cpp/package.json',
|
||||
'@transcribe-cpp/win32-x64-cpu-vulkan/contract.json'
|
||||
])
|
||||
expect(
|
||||
findAppSpeechRecognizerDir(join(resources, 'app.asar.unpacked'), 'win32', 'x64')
|
||||
).toBeUndefined()
|
||||
|
||||
@@ -2,9 +2,9 @@ import { dirname, join } from 'node:path'
|
||||
import { hasSpeechEnginePackage } from '../../shared/speech-recognizer-package'
|
||||
|
||||
/**
|
||||
* The installed app's node_modules holding its on-device speech engines, for orcad's dictation
|
||||
* (#31174). electron-builder puts them in `resources/node_modules`, beside the app root; a dev
|
||||
* checkout keeps them in node_modules.
|
||||
* The installed app's node_modules holding transcribe-cpp, for orcad's on-device dictation
|
||||
* (#31174). electron-builder puts it in `resources/node_modules`, beside the app root; a dev
|
||||
* checkout keeps it in node_modules.
|
||||
*
|
||||
* Not on Windows: a loaded addon locks its files for the server's life, which would make the next
|
||||
* app update fail to replace them. Windows orcad keeps cloud dictation only.
|
||||
@@ -17,9 +17,7 @@ export function findAppSpeechRecognizerDir(
|
||||
if (platform === 'win32') {
|
||||
return undefined
|
||||
}
|
||||
return [join(dirname(appRoot), 'node_modules'), join(appRoot, 'node_modules')].find(
|
||||
(dir) =>
|
||||
hasSpeechEnginePackage(dir, 'transcribe-cpp', platform, arch) ||
|
||||
hasSpeechEnginePackage(dir, 'sherpa', platform, arch)
|
||||
return [join(dirname(appRoot), 'node_modules'), join(appRoot, 'node_modules')].find((dir) =>
|
||||
hasSpeechEnginePackage(dir, platform, arch)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -114,7 +114,7 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
|
||||
const getHotwordsFilePath = (content: string): string => {
|
||||
const digest = createHash('sha256').update(content).digest('hex').slice(0, 12)
|
||||
// sherpa-onnx cannot read non-ASCII Windows paths; use the ASCII-safe model cache for hotwords.
|
||||
// Native speech libraries may open paths with ANSI Windows APIs; keep hotwords in the ASCII-safe model cache.
|
||||
return join(getSpeechModelManager(store).getModelsDir(), `speech-hotwords-${digest}.txt`)
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
}
|
||||
|
||||
if (hotwords && hotwords.length > 0) {
|
||||
const content = `${hotwords.map((w) => `${w} :2.0`).join('\n')}\n`
|
||||
const content = `${hotwords.join('\n')}\n`
|
||||
const hotwordsFilePath = getHotwordsFilePath(content)
|
||||
releaseHotwords = retainHotwordsFile(hotwordsFilePath)
|
||||
await writeFile(hotwordsFilePath, content, 'utf-8')
|
||||
|
||||
@@ -54,7 +54,7 @@ export class RuntimeMobileDictationController {
|
||||
throw new Error('voice_model_not_selected')
|
||||
}
|
||||
const manifest = getCatalogModel(modelId)
|
||||
if (manifest && isLocalSpeechModel(manifest) && !hasLocalSpeechRecognizer(manifest)) {
|
||||
if (manifest && isLocalSpeechModel(manifest) && !hasLocalSpeechRecognizer()) {
|
||||
throw new Error(LOCAL_SPEECH_UNAVAILABLE_ERROR)
|
||||
}
|
||||
const modelState = await getSpeechModelManager(store).getModelState(modelId)
|
||||
|
||||
@@ -43,7 +43,7 @@ export class RuntimeMobileSpeechCatalog {
|
||||
const runnable = SPEECH_MODEL_CATALOG.filter(
|
||||
(manifest) =>
|
||||
isOfferedSpeechModel(manifest) &&
|
||||
(!isLocalSpeechModel(manifest) || hasLocalSpeechRecognizer(manifest)) &&
|
||||
(!isLocalSpeechModel(manifest) || hasLocalSpeechRecognizer()) &&
|
||||
// Phones offer an unready cloud row as "add an OpenAI key", which can't set this one up.
|
||||
(manifest.provider !== 'openai-compatible' ||
|
||||
stateById.get(manifest.id)?.status === 'ready')
|
||||
@@ -74,7 +74,7 @@ export class RuntimeMobileSpeechCatalog {
|
||||
async download(modelId: string): Promise<{ started: true }> {
|
||||
const store = this.requireStore()
|
||||
const manifest = getCatalogModel(modelId)
|
||||
if (!manifest || !isLocalSpeechModel(manifest) || !hasLocalSpeechRecognizer(manifest)) {
|
||||
if (!manifest || !isLocalSpeechModel(manifest) || !hasLocalSpeechRecognizer()) {
|
||||
throw new Error('voice_model_not_downloadable')
|
||||
}
|
||||
void getSpeechModelManager(store)
|
||||
|
||||
@@ -59,7 +59,7 @@ export function getSpeechModelCacheDirCandidates(
|
||||
.filter((modelsDir) => !hasNonAsciiCharacters(modelsDir))
|
||||
.map((modelsDir) => ({ modelsDir, migrationSourceDir: requestedModelsDir }))
|
||||
|
||||
// sherpa-onnx 1.12.x needs ASCII Windows paths; retain the requested path if shared caches fail.
|
||||
// Native speech libraries may open paths with ANSI Windows APIs; retain the requested path if shared caches fail.
|
||||
return [...candidates, { modelsDir: requestedModelsDir, migrationSourceDir: null }]
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,6 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
||||
...getSpeechModelDownloadMetadata('zipformer-bilingual-zh-en'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'cjkchar+bpe',
|
||||
// transcribe-cpp has no zipformer; SenseVoice covers Chinese and English.
|
||||
replacedBy: 'sense-voice-zh-en-ja-ko-yue'
|
||||
},
|
||||
@@ -77,7 +76,6 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
||||
...getSpeechModelDownloadMetadata('zipformer-streaming-en-20m'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'bpe',
|
||||
// The model itself loses the first words of every utterance.
|
||||
replacedBy: 'parakeet-tdt-0.6b-v2-int8'
|
||||
},
|
||||
@@ -91,7 +89,6 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
||||
...getSpeechModelDownloadMetadata('zipformer-streaming-zh-14m'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'cjkchar',
|
||||
// transcribe-cpp has no zipformer; SenseVoice beat Nemotron 3.5 on Korean and Chinese.
|
||||
replacedBy: 'sense-voice-zh-en-ja-ko-yue'
|
||||
},
|
||||
@@ -105,7 +102,6 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
||||
...getSpeechModelDownloadMetadata('zipformer-streaming-korean'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'bpe',
|
||||
// transcribe-cpp has no zipformer; SenseVoice beat Nemotron 3.5 on Korean and Chinese.
|
||||
replacedBy: 'sense-voice-zh-en-ja-ko-yue'
|
||||
},
|
||||
|
||||
@@ -15,7 +15,7 @@ function gpuFailureKey(): string {
|
||||
return cachedGpuFailureKey
|
||||
}
|
||||
let version = 'unknown'
|
||||
const modulesDir = getSpeechEngineModulesDir('transcribe-cpp')
|
||||
const modulesDir = getSpeechEngineModulesDir()
|
||||
if (modulesDir) {
|
||||
try {
|
||||
version = JSON.parse(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Bound ONNX allocations: Chromium kills the app on a single allocation >=2GiB (#7925).
|
||||
// Parakeet has no long-form mode, and bounded chunks keep the decode left for stop short.
|
||||
export const OFFLINE_DECODE_CHUNK_SECONDS = 30
|
||||
|
||||
// Split near quiet tail pauses to avoid mid-word cuts; ~100ms windows exclude short intra-word closures.
|
||||
|
||||
@@ -1,23 +1,25 @@
|
||||
import type { RecognizerConfigInput } from './stt-worker-model-config'
|
||||
|
||||
export type SttRecognizerInit = RecognizerConfigInput & {
|
||||
/** The node_modules directory holding this model's engine package. */
|
||||
export type SttRecognizerInit = {
|
||||
/** The node_modules directory holding transcribe-cpp. */
|
||||
speechModulesDir: string
|
||||
modelDir: string
|
||||
files: string[]
|
||||
sampleRate: number
|
||||
hotwordsFilePath?: string
|
||||
/** Absent = auto-detect. */
|
||||
language?: string
|
||||
}
|
||||
|
||||
export type SttRecognizerEvent =
|
||||
| { type: 'partial'; streamId: string; text: string }
|
||||
| { type: 'final'; streamId: string; text: string }
|
||||
/** One decoded offline chunk; `samples` counts the stream's input-rate samples decoded so far. */
|
||||
| { type: 'chunk'; streamId: string; text: string; samples: number }
|
||||
{ type: 'chunk'; streamId: string; text: string; samples: number }
|
||||
|
||||
export type SttRecognizerPost = (event: SttRecognizerEvent) => void
|
||||
|
||||
/** One loaded model in the speech host child, shared by every dictation stream; calls never overlap. */
|
||||
export type SttRecognizer = {
|
||||
feed(streamId: string, samples: Float32Array, sampleRate: number): void | Promise<void>
|
||||
feed(streamId: string, samples: Float32Array, sampleRate: number): Promise<void>
|
||||
/** Decode the stream's buffered audio, post its text and drop the stream; may throw after posting. */
|
||||
finish(streamId: string): void | Promise<void>
|
||||
finish(streamId: string): Promise<void>
|
||||
/** Drop the stream's audio undecoded. */
|
||||
discard(streamId: string): void
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ export async function prewarmSttModel(
|
||||
const ready =
|
||||
manifest !== undefined &&
|
||||
config !== null &&
|
||||
hasLocalSpeechRecognizer(manifest) &&
|
||||
hasLocalSpeechRecognizer() &&
|
||||
(await state.modelManager.getModelState(modelId)).status === 'ready'
|
||||
for (;;) {
|
||||
// Never disturb a dictation; whatever it loaded is what the user is using.
|
||||
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
teardownIdleSttEngine,
|
||||
teardownSttEngine
|
||||
} from './stt-session-stop'
|
||||
import { speechEngineForModelFiles } from '../../shared/speech-model-engine'
|
||||
import { getSpeechEngineModulesDir, LOCAL_SPEECH_UNAVAILABLE_ERROR } from './stt-worker-paths'
|
||||
import {
|
||||
attachSttWorkerLifecycle,
|
||||
@@ -171,8 +170,7 @@ export function spawnSttEngine(
|
||||
config: SttRecognizerConfig,
|
||||
manifest: SpeechModelManifest
|
||||
): SttEngine {
|
||||
const files = manifest.files ?? []
|
||||
const speechModulesDir = getSpeechEngineModulesDir(speechEngineForModelFiles(files))
|
||||
const speechModulesDir = getSpeechEngineModulesDir()
|
||||
if (!speechModulesDir) {
|
||||
throw new Error(LOCAL_SPEECH_UNAVAILABLE_ERROR)
|
||||
}
|
||||
@@ -209,12 +207,9 @@ export function spawnSttEngine(
|
||||
initializeSttWorker(worker, {
|
||||
speechModulesDir,
|
||||
modelDir: state.modelManager.getModelDir(config.modelId),
|
||||
modelType: manifest.type,
|
||||
streaming: manifest.streaming,
|
||||
sampleRate: manifest.sampleRate,
|
||||
files,
|
||||
files: manifest.files ?? [],
|
||||
hotwordsFilePath: config.hotwordsFilePath,
|
||||
modelingUnit: manifest.modelingUnit,
|
||||
language: config.language
|
||||
})
|
||||
return engine
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
/* oxlint-disable typescript-eslint/no-explicit-any -- sherpa-onnx native addon has no type definitions */
|
||||
import { join } from 'node:path'
|
||||
import { speechRecognizerPackageName } from '../../shared/speech-recognizer-package'
|
||||
import { resampleToRate } from './stt-audio-resample'
|
||||
import { OfflineAudioChunker } from './stt-offline-audio-chunker'
|
||||
import { buildRecognizerConfig } from './stt-worker-model-config'
|
||||
import { createStopTailPadding } from './stt-stop-tail-padding'
|
||||
import type { SttRecognizer, SttRecognizerInit, SttRecognizerPost } from './stt-recognizer'
|
||||
|
||||
// One per dictation session; every session shares the one loaded recognizer.
|
||||
type SessionStream = {
|
||||
// Online streams live for the session; offline ones are made per decode.
|
||||
online: any
|
||||
offlineChunker: OfflineAudioChunker | null
|
||||
// Primed on the session's first feed so stop never waits on another session's stream.
|
||||
primed: boolean
|
||||
// Streaming models report the same hypothesis on most 100 ms feeds; only changes are posted.
|
||||
lastPartialText: string
|
||||
}
|
||||
|
||||
export function createSherpaRecognizer(
|
||||
init: SttRecognizerInit,
|
||||
post: SttRecognizerPost
|
||||
): SttRecognizer {
|
||||
// Use the platform native addon: the main sherpa-onnx WASM package cannot load host model files.
|
||||
const sherpa: any = require(
|
||||
join(init.speechModulesDir, speechRecognizerPackageName(process.platform, process.arch))
|
||||
)
|
||||
const { streaming: isStreaming, sampleRate: modelSampleRate } = init
|
||||
const { online, config } = buildRecognizerConfig(init)
|
||||
const recognizer = online
|
||||
? sherpa.createOnlineRecognizer(config)
|
||||
: sherpa.createOfflineRecognizer(config)
|
||||
const streams = new Map<string, SessionStream>()
|
||||
|
||||
const openStream = (streamId: string): SessionStream => {
|
||||
const existing = streams.get(streamId)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
const stream: SessionStream = {
|
||||
online: isStreaming ? sherpa.createOnlineStream(recognizer) : null,
|
||||
offlineChunker: isStreaming ? null : new OfflineAudioChunker(modelSampleRate),
|
||||
primed: false,
|
||||
lastPartialText: ''
|
||||
}
|
||||
streams.set(streamId, stream)
|
||||
return stream
|
||||
}
|
||||
|
||||
const decodeOfflineChunk = (samples: Float32Array): string => {
|
||||
const stream = sherpa.createOfflineStream(recognizer)
|
||||
sherpa.acceptWaveformOffline(stream, { sampleRate: modelSampleRate, samples })
|
||||
sherpa.decodeOfflineStream(recognizer, stream)
|
||||
const result = JSON.parse(sherpa.getOfflineStreamResultAsJson(stream))
|
||||
return result?.text?.trim() ?? ''
|
||||
}
|
||||
|
||||
const feedOnline = (streamId: string, stream: SessionStream, samples: Float32Array): void => {
|
||||
if (!stream.primed) {
|
||||
// New streams start with empty encoder caches; zipformer en-20M dropped most of the first second
|
||||
// (first word wrong on 71 of 73 LibriSpeech clips). A second of silence fills them first.
|
||||
sherpa.acceptWaveformOnline(stream.online, {
|
||||
sampleRate: modelSampleRate,
|
||||
samples: new Float32Array(modelSampleRate)
|
||||
})
|
||||
stream.primed = true
|
||||
}
|
||||
sherpa.acceptWaveformOnline(stream.online, { sampleRate: modelSampleRate, samples })
|
||||
while (sherpa.isOnlineStreamReady(recognizer, stream.online)) {
|
||||
sherpa.decodeOnlineStream(recognizer, stream.online)
|
||||
}
|
||||
const result = JSON.parse(sherpa.getOnlineStreamResultAsJson(recognizer, stream.online))
|
||||
const text = result?.text?.trim()
|
||||
if (text && text !== stream.lastPartialText) {
|
||||
stream.lastPartialText = text
|
||||
post({ type: 'partial', streamId, text })
|
||||
}
|
||||
if (sherpa.isEndpoint(recognizer, stream.online)) {
|
||||
if (text) {
|
||||
post({ type: 'final', streamId, text })
|
||||
}
|
||||
sherpa.reset(recognizer, stream.online)
|
||||
stream.lastPartialText = ''
|
||||
}
|
||||
}
|
||||
|
||||
const feedOffline = (streamId: string, stream: SessionStream, samples: Float32Array): void => {
|
||||
// Bound ONNX allocations below Chromium's >=2GiB crash threshold (#7925); consumers append final segments.
|
||||
const readyChunks = stream.offlineChunker?.push(new Float32Array(samples)) ?? []
|
||||
// push() already consumed these windows; keep decoding later ones after a failure.
|
||||
let firstError: unknown = null
|
||||
for (const chunk of readyChunks) {
|
||||
try {
|
||||
const text = decodeOfflineChunk(chunk)
|
||||
if (text) {
|
||||
post({ type: 'final', streamId, text })
|
||||
}
|
||||
} catch (err) {
|
||||
firstError ??= err
|
||||
}
|
||||
}
|
||||
if (firstError) {
|
||||
throw firstError
|
||||
}
|
||||
}
|
||||
|
||||
const finishStream = (stream: SessionStream): string => {
|
||||
if (stream.online) {
|
||||
sherpa.acceptWaveformOnline(stream.online, {
|
||||
sampleRate: modelSampleRate,
|
||||
samples: createStopTailPadding(modelSampleRate)
|
||||
})
|
||||
sherpa.inputFinished(stream.online)
|
||||
while (sherpa.isOnlineStreamReady(recognizer, stream.online)) {
|
||||
sherpa.decodeOnlineStream(recognizer, stream.online)
|
||||
}
|
||||
const result = JSON.parse(sherpa.getOnlineStreamResultAsJson(recognizer, stream.online))
|
||||
return result?.text?.trim() ?? ''
|
||||
}
|
||||
// flush() leaves less than one bounded chunk.
|
||||
const remaining = stream.offlineChunker?.flush()
|
||||
return remaining && remaining.length > 0 ? decodeOfflineChunk(remaining) : ''
|
||||
}
|
||||
|
||||
return {
|
||||
feed(streamId, samples, sampleRate) {
|
||||
const stream = openStream(streamId)
|
||||
// Normalize rates before native decode: mixed rates on one recognizer abort the process.
|
||||
const resampled = resampleToRate(samples, sampleRate || modelSampleRate, modelSampleRate)
|
||||
if (isStreaming) {
|
||||
feedOnline(streamId, stream, resampled)
|
||||
} else {
|
||||
feedOffline(streamId, stream, resampled)
|
||||
}
|
||||
},
|
||||
finish(streamId) {
|
||||
const stream = streams.get(streamId)
|
||||
streams.delete(streamId)
|
||||
const text = stream ? finishStream(stream) : ''
|
||||
if (text) {
|
||||
post({ type: 'final', streamId, text })
|
||||
}
|
||||
},
|
||||
discard(streamId) {
|
||||
streams.delete(streamId)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
// Streaming encoders emit a word only once lookahead frames follow it; sherpa-onnx's own decoders pad 0.66 s before inputFinished.
|
||||
const STOP_TAIL_PADDING_SECONDS = 0.66
|
||||
|
||||
export function createStopTailPadding(sampleRate: number): Float32Array {
|
||||
return new Float32Array(Math.round(STOP_TAIL_PADDING_SECONDS * sampleRate))
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import { pathToFileURL } from 'node:url'
|
||||
import type * as TranscribeCpp from 'transcribe-cpp'
|
||||
import { resampleToRate } from './stt-audio-resample'
|
||||
import { OfflineAudioChunker } from './stt-offline-audio-chunker'
|
||||
import { resolveFile } from './stt-worker-model-config'
|
||||
import type { SttRecognizer, SttRecognizerInit, SttRecognizerPost } from './stt-recognizer'
|
||||
|
||||
// Past ~8 threads decode slowed 2-4x under load in the 2026-11 benchmark; E-cores and SMT siblings
|
||||
@@ -15,13 +14,13 @@ function performanceCoreThreads(): number {
|
||||
return Math.min(logical, 8, Math.max(4, Math.floor(logical / 2)))
|
||||
}
|
||||
|
||||
// The hotwords file is written in sherpa's format, one `term :score` per line. The host deletes it
|
||||
// once dictation starts, so a CPU fallback's reload finds it gone and runs without.
|
||||
// One term per line. The host deletes the file once dictation starts, so a CPU fallback's reload
|
||||
// finds it gone and runs without.
|
||||
async function readVocabulary(path: string | undefined): Promise<string[]> {
|
||||
const text = path ? await readFile(path, 'utf8').catch(() => '') : ''
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/\s+:[\d.]+\s*$/, '').trim())
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
@@ -39,9 +38,11 @@ export async function createTranscribeCppRecognizer(
|
||||
): Promise<SttRecognizer> {
|
||||
const entry = join(init.speechModulesDir, 'transcribe-cpp', 'dist', 'index.js')
|
||||
const transcribe: typeof TranscribeCpp = await import(pathToFileURL(entry).href)
|
||||
const model = await transcribe.TranscribeModel.load(
|
||||
resolveFile(init.files, '.gguf', init.modelDir, '.gguf')
|
||||
)
|
||||
const gguf = init.files.find((file) => file.endsWith('.gguf'))
|
||||
if (!gguf) {
|
||||
throw new Error(`No .gguf model file in: ${init.files.join(', ')}`)
|
||||
}
|
||||
const model = await transcribe.TranscribeModel.load(join(init.modelDir, gguf))
|
||||
// The child runs one call at a time, so every stream can share one session.
|
||||
const session = model.createSession({ nThreads: performanceCoreThreads() })
|
||||
const vocabulary = await readVocabulary(init.hotwordsFilePath)
|
||||
|
||||
@@ -40,8 +40,6 @@ function setup(): {
|
||||
type: 'init',
|
||||
speechModulesDir: '/app/node_modules',
|
||||
modelDir: '/models/parakeet',
|
||||
modelType: 'transducer',
|
||||
streaming: false,
|
||||
sampleRate: 16000,
|
||||
files: ['model-Q8_0.gguf']
|
||||
})
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import type { MessagePortMain } from 'electron'
|
||||
import { speechEngineForModelFiles } from '../../shared/speech-model-engine'
|
||||
import { OFFLINE_DECODE_CHUNK_SECONDS } from './stt-offline-audio-chunker'
|
||||
import type { SttWorkerProcess, SttWorkerRequest } from './stt-worker-process'
|
||||
|
||||
@@ -44,7 +43,6 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
|
||||
private detachChild: () => void = () => undefined
|
||||
private cpuOnly: boolean
|
||||
private init: Extract<SttWorkerRequest, { type: 'init' }> | null = null
|
||||
private transcribeCpp = false
|
||||
private ready = false
|
||||
private exitRequested = false
|
||||
private markOnCpuReady = false
|
||||
@@ -63,7 +61,6 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
|
||||
postMessage(msg: SttWorkerRequest): void {
|
||||
if (msg.type === 'init') {
|
||||
this.init = msg
|
||||
this.transcribeCpp = speechEngineForModelFiles(msg.files) === 'transcribe-cpp'
|
||||
} else if (msg.type === 'feed' && this.canFallBack()) {
|
||||
const stream = this.backlog(msg.streamId)
|
||||
stream.feeds.push(msg)
|
||||
@@ -87,7 +84,7 @@ class SttCpuFallbackWorker extends EventEmitter implements SttWorkerProcess {
|
||||
}
|
||||
|
||||
private canFallBack(): boolean {
|
||||
return this.transcribeCpp && !this.cpuOnly && !this.exitRequested
|
||||
return this.init !== null && !this.cpuOnly && !this.exitRequested
|
||||
}
|
||||
|
||||
private backlog(streamId: string): StreamBacklog {
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
import { readdirSync } from 'node:fs'
|
||||
import { availableParallelism } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
// ONNX filenames vary by model; find each role in the manifest.
|
||||
export function resolveFile(
|
||||
files: string[],
|
||||
role: string,
|
||||
modelDir: string,
|
||||
ext = '.onnx'
|
||||
): string {
|
||||
const match = files.find((f) => f.includes(role) && f.endsWith(ext))
|
||||
if (!match) {
|
||||
throw new Error(`No *${role}*${ext} found in model files: ${files.join(', ')}`)
|
||||
}
|
||||
return join(modelDir, match)
|
||||
}
|
||||
|
||||
export function resolveTokens(files: string[], modelDir: string): string {
|
||||
const match = files.find((f) => f.endsWith('tokens.txt'))
|
||||
if (!match) {
|
||||
throw new Error(`No *tokens.txt found in model files: ${files.join(', ')}`)
|
||||
}
|
||||
return join(modelDir, match)
|
||||
}
|
||||
|
||||
// BPE hotword matching needs a vocab file that older caches may lack.
|
||||
function discoverBpeVocab(modelDir: string): string | undefined {
|
||||
try {
|
||||
const entries = readdirSync(modelDir)
|
||||
const vocabFile = entries.find((f) => f.endsWith('.vocab'))
|
||||
return vocabFile ? join(modelDir, vocabFile) : undefined
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export type HotwordsConfig = {
|
||||
decodingMethod: string
|
||||
hotwordsFile?: string
|
||||
hotwordsScore?: number
|
||||
modelingUnit?: string
|
||||
bpeVocab?: string
|
||||
}
|
||||
|
||||
export function buildHotwordsConfig(opts: {
|
||||
modelDir: string
|
||||
modelType: string
|
||||
hotwordsFilePath?: string
|
||||
modelingUnit?: string
|
||||
}): HotwordsConfig {
|
||||
if (opts.modelType !== 'transducer' || !opts.hotwordsFilePath) {
|
||||
return { decodingMethod: 'greedy_search' }
|
||||
}
|
||||
|
||||
const unit = opts.modelingUnit
|
||||
if (unit?.includes('bpe')) {
|
||||
const bpeVocab = discoverBpeVocab(opts.modelDir)
|
||||
if (!bpeVocab) {
|
||||
return { decodingMethod: 'greedy_search' }
|
||||
}
|
||||
return {
|
||||
decodingMethod: 'modified_beam_search',
|
||||
hotwordsFile: opts.hotwordsFilePath,
|
||||
hotwordsScore: 1.5,
|
||||
modelingUnit: unit,
|
||||
bpeVocab
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
decodingMethod: 'modified_beam_search',
|
||||
hotwordsFile: opts.hotwordsFilePath,
|
||||
hotwordsScore: 1.5,
|
||||
modelingUnit: unit
|
||||
}
|
||||
}
|
||||
|
||||
export type RecognizerConfigInput = {
|
||||
modelDir: string
|
||||
modelType: string
|
||||
streaming: boolean
|
||||
sampleRate: number
|
||||
files: string[]
|
||||
hotwordsFilePath?: string
|
||||
modelingUnit?: string
|
||||
// Absent = auto-detect; only Whisper and SenseVoice read it.
|
||||
language?: string
|
||||
}
|
||||
|
||||
// The user waits on offline decode at stop (and per 30 s chunk); 4 threads cut it ~35% at equal memory.
|
||||
const OFFLINE_NUM_THREADS = Math.min(4, availableParallelism())
|
||||
|
||||
// Online configs go to createOnlineRecognizer, the rest to createOfflineRecognizer.
|
||||
export function buildRecognizerConfig(msg: RecognizerConfigInput): {
|
||||
online: boolean
|
||||
config: Record<string, unknown>
|
||||
} {
|
||||
const { modelDir, modelType, streaming, sampleRate, files } = msg
|
||||
const tokens = resolveTokens(files, modelDir)
|
||||
const hotwords = buildHotwordsConfig(msg)
|
||||
|
||||
if (streaming && modelType === 'transducer') {
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
transducer: {
|
||||
encoder: resolveFile(files, 'encoder', modelDir),
|
||||
decoder: resolveFile(files, 'decoder', modelDir),
|
||||
joiner: resolveFile(files, 'joiner', modelDir)
|
||||
},
|
||||
tokens,
|
||||
numThreads: 1,
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
},
|
||||
...hotwords,
|
||||
enableEndpoint: 1,
|
||||
rule1MinTrailingSilence: 2.4,
|
||||
rule2MinTrailingSilence: 1.2,
|
||||
rule3MinUtteranceLength: 20
|
||||
}
|
||||
return { online: true, config }
|
||||
}
|
||||
if (streaming && modelType === 'paraformer') {
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
paraformer: {
|
||||
encoder: resolveFile(files, 'encoder', modelDir),
|
||||
decoder: resolveFile(files, 'decoder', modelDir)
|
||||
},
|
||||
tokens,
|
||||
numThreads: 1,
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
},
|
||||
decodingMethod: 'greedy_search',
|
||||
enableEndpoint: 1,
|
||||
rule1MinTrailingSilence: 2.4,
|
||||
rule2MinTrailingSilence: 1.2,
|
||||
rule3MinUtteranceLength: 20
|
||||
}
|
||||
return { online: true, config }
|
||||
}
|
||||
if (modelType === 'whisper') {
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
whisper: {
|
||||
encoder: resolveFile(files, 'encoder', modelDir),
|
||||
decoder: resolveFile(files, 'decoder', modelDir),
|
||||
language: msg.language ?? ''
|
||||
},
|
||||
tokens,
|
||||
numThreads: OFFLINE_NUM_THREADS,
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
},
|
||||
decodingMethod: 'greedy_search'
|
||||
}
|
||||
return { online: false, config }
|
||||
}
|
||||
if (modelType === 'nemo-ctc') {
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
nemoCtc: {
|
||||
model: resolveFile(files, 'model', modelDir)
|
||||
},
|
||||
tokens,
|
||||
numThreads: OFFLINE_NUM_THREADS,
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
},
|
||||
decodingMethod: 'greedy_search'
|
||||
}
|
||||
return { online: false, config }
|
||||
}
|
||||
if (modelType === 'senseVoice') {
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
senseVoice: {
|
||||
model: resolveFile(files, 'model', modelDir),
|
||||
language: msg.language ?? '',
|
||||
useInverseTextNormalization: 1
|
||||
},
|
||||
tokens,
|
||||
numThreads: OFFLINE_NUM_THREADS,
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
},
|
||||
decodingMethod: 'greedy_search'
|
||||
}
|
||||
return { online: false, config }
|
||||
}
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
transducer: {
|
||||
encoder: resolveFile(files, 'encoder', modelDir),
|
||||
decoder: resolveFile(files, 'decoder', modelDir),
|
||||
joiner: resolveFile(files, 'joiner', modelDir)
|
||||
},
|
||||
tokens,
|
||||
numThreads: OFFLINE_NUM_THREADS,
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
},
|
||||
...hotwords
|
||||
}
|
||||
return { online: false, config }
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { join } from 'node:path'
|
||||
import { getAppEnvironment } from '../../shared/app-environment'
|
||||
import { speechEngineForModelFiles, type SpeechEngine } from '../../shared/speech-model-engine'
|
||||
import {
|
||||
hasSpeechEnginePackage,
|
||||
SPEECH_RECOGNIZER_DIR_ENV
|
||||
@@ -23,10 +22,7 @@ function speechNodeModulesDirCandidates(): string[] {
|
||||
? process.resourcesPath
|
||||
: undefined
|
||||
if (getAppEnvironment().isPackaged() && resourcesPath) {
|
||||
return [
|
||||
join(resourcesPath, 'node_modules'),
|
||||
join(resourcesPath, 'app.asar.unpacked', 'node_modules')
|
||||
]
|
||||
return [join(resourcesPath, 'node_modules')]
|
||||
}
|
||||
if (getAppEnvironment().isPackaged()) {
|
||||
// orcad: only the installed app's copy, which `orca serve` names.
|
||||
@@ -36,23 +32,18 @@ function speechNodeModulesDirCandidates(): string[] {
|
||||
return require.resolve.paths('transcribe-cpp') ?? []
|
||||
}
|
||||
|
||||
// The app's engine packages don't change while it runs, so each engine is looked up once.
|
||||
const speechEngineModulesDirs = new Map<SpeechEngine, string | null>()
|
||||
// The app's engine packages don't change while it runs, so they are looked up once.
|
||||
let speechEngineModulesDir: string | null | undefined
|
||||
|
||||
/** The node_modules directory holding `engine`, or null on hosts without it. */
|
||||
export function getSpeechEngineModulesDir(engine: SpeechEngine): string | null {
|
||||
let dir = speechEngineModulesDirs.get(engine)
|
||||
if (dir === undefined) {
|
||||
dir =
|
||||
speechNodeModulesDirCandidates().find((candidate) =>
|
||||
hasSpeechEnginePackage(candidate, engine)
|
||||
) ?? null
|
||||
speechEngineModulesDirs.set(engine, dir)
|
||||
/** The node_modules directory holding transcribe-cpp, or null on hosts without it. */
|
||||
export function getSpeechEngineModulesDir(): string | null {
|
||||
if (speechEngineModulesDir === undefined) {
|
||||
speechEngineModulesDir =
|
||||
speechNodeModulesDirCandidates().find((dir) => hasSpeechEnginePackage(dir)) ?? null
|
||||
}
|
||||
return dir
|
||||
return speechEngineModulesDir
|
||||
}
|
||||
|
||||
/** Whether this host ships the engine `model` runs on. */
|
||||
export function hasLocalSpeechRecognizer(model: { files?: readonly string[] }): boolean {
|
||||
return getSpeechEngineModulesDir(speechEngineForModelFiles(model.files ?? [])) !== null
|
||||
export function hasLocalSpeechRecognizer(): boolean {
|
||||
return getSpeechEngineModulesDir() !== null
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import type { MessagePortMain } from 'electron'
|
||||
import { speechEngineForModelFiles } from '../../shared/speech-model-engine'
|
||||
import type { SpeechAudioPortMessage } from '../../shared/speech-types'
|
||||
import type { SttRecognizer, SttRecognizerInit, SttRecognizerPost } from './stt-recognizer'
|
||||
import { createSherpaRecognizer } from './stt-sherpa-recognizer'
|
||||
import { createTranscribeCppRecognizer } from './stt-transcribe-cpp-recognizer'
|
||||
import { connectSttWorkerParentPort } from './stt-worker-parent-port'
|
||||
import type { SttWorkerRequest } from './stt-worker-process'
|
||||
@@ -29,10 +27,7 @@ function enqueue(task: () => void | Promise<void>): void {
|
||||
async function handleInit(init: SttRecognizerInit): Promise<void> {
|
||||
try {
|
||||
const post: SttRecognizerPost = (event) => parentPort.post(event)
|
||||
recognizer =
|
||||
speechEngineForModelFiles(init.files) === 'transcribe-cpp'
|
||||
? await createTranscribeCppRecognizer(init, post)
|
||||
: createSherpaRecognizer(init, post)
|
||||
recognizer = await createTranscribeCppRecognizer(init, post)
|
||||
parentPort.post({ type: 'ready' })
|
||||
} catch (err) {
|
||||
parentPort.post({ type: 'error', error: String(err) })
|
||||
|
||||
@@ -17,7 +17,6 @@ import { VoiceModelUnloadSetting } from './VoiceModelUnloadSetting'
|
||||
import { VoiceSpeechModelSection } from './VoiceSpeechModelSection'
|
||||
import { SettingsSwitchRow } from '../form-controls/SettingsFormControls'
|
||||
import { VoiceSpeechGpuSetting } from './VoiceSpeechGpuSetting'
|
||||
import { speechEngineForModelFiles } from '../../../../../shared/speech-model-engine'
|
||||
import { matchesSettingsSearch } from '../search/settings-search'
|
||||
import { getOpenaiTranscriptionSearchEntry } from './voice-pane-search'
|
||||
import { useTranslate } from '@/i18n/use-translate'
|
||||
@@ -301,16 +300,15 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J
|
||||
onUpdateVoiceSettings={updateVoiceSettings}
|
||||
/>
|
||||
|
||||
{selectedModel &&
|
||||
speechEngineForModelFiles(selectedModel.files ?? []) === 'transcribe-cpp' && (
|
||||
<>
|
||||
<Separator />
|
||||
<VoiceSpeechGpuSetting
|
||||
voiceSettings={voiceSettings}
|
||||
onUpdateVoiceSettings={updateVoiceSettings}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
{selectedModel?.provider === 'local' && (
|
||||
<>
|
||||
<Separator />
|
||||
<VoiceSpeechGpuSetting
|
||||
voiceSettings={voiceSettings}
|
||||
onUpdateVoiceSettings={updateVoiceSettings}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{showOpenAiSettingsRow && (
|
||||
<>
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
export type SpeechEngine = 'sherpa' | 'transcribe-cpp'
|
||||
|
||||
/** GGUF models run on transcribe-cpp; everything else on sherpa-onnx. */
|
||||
export function speechEngineForModelFiles(files: readonly string[]): SpeechEngine {
|
||||
return files.some((file) => file.endsWith('.gguf')) ? 'transcribe-cpp' : 'sherpa'
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import type { SpeechEngine } from './speech-model-engine'
|
||||
import transcribeCppPackages from './transcribe-cpp-packages.json'
|
||||
|
||||
/** Desktop and orca serve share the app’s speech engine packages; orcad slots omit them. */
|
||||
@@ -8,26 +7,15 @@ import transcribeCppPackages from './transcribe-cpp-packages.json'
|
||||
/** Set by `orca serve` for orcad: the app's node_modules directory holding the speech engine packages. */
|
||||
export const SPEECH_RECOGNIZER_DIR_ENV = 'ORCA_SPEECH_RECOGNIZER_DIR'
|
||||
|
||||
export function speechRecognizerPackageName(platform: string, arch: string): string {
|
||||
return platform === 'win32' && arch === 'x64'
|
||||
? 'sherpa-onnx-win-x64'
|
||||
: `sherpa-onnx-${platform}-${arch}`
|
||||
}
|
||||
|
||||
// Packaging reads the same table. Intel Macs run transcribe-cpp on the CPU (its x64 build has no GPU backend).
|
||||
const TRANSCRIBE_CPP_PACKAGES: Record<string, string> = transcribeCppPackages
|
||||
|
||||
/** Whether `nodeModulesDir` holds a loadable copy of `engine` for this host. */
|
||||
/** Whether `nodeModulesDir` holds transcribe-cpp and this host's native package for it. */
|
||||
export function hasSpeechEnginePackage(
|
||||
nodeModulesDir: string,
|
||||
engine: SpeechEngine,
|
||||
platform: string = process.platform,
|
||||
arch: string = process.arch
|
||||
): boolean {
|
||||
if (engine === 'sherpa') {
|
||||
const sherpa = speechRecognizerPackageName(platform, arch)
|
||||
return existsSync(join(nodeModulesDir, sherpa, 'sherpa-onnx.node'))
|
||||
}
|
||||
const native = TRANSCRIBE_CPP_PACKAGES[`${platform}-${arch}`]
|
||||
return (
|
||||
native !== undefined &&
|
||||
|
||||
@@ -7,8 +7,6 @@ export type SpeechModelType =
|
||||
| 'openai'
|
||||
export type SpeechModelProvider = 'local' | 'openai' | 'openai-compatible'
|
||||
|
||||
export type ModelingUnit = 'bpe' | 'cjkchar' | 'cjkchar+bpe'
|
||||
|
||||
export type SpeechModelDownloadFile = {
|
||||
name: string
|
||||
url: string
|
||||
@@ -30,7 +28,6 @@ export type SpeechModelManifest = {
|
||||
legacyDownloadFiles?: SpeechModelDownloadFile[]
|
||||
sampleRate: number
|
||||
streaming: boolean
|
||||
modelingUnit?: ModelingUnit
|
||||
recommended?: boolean
|
||||
/** Retired: never offered for selection, and a saved selection moves to this model on load. */
|
||||
replacedBy?: string
|
||||
|
||||
Reference in New Issue
Block a user