mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
feat(speech): dictate through any OpenAI-compatible transcription service
Add an OpenAI-compatible model whose base URL, model and optional key are set up in a Voice settings dialog and sealed together in ~/.orca, so a key is never sent to a host it wasn't saved for. It reaches OpenAI, Groq, Mistral and local servers, and replaces the unused userModels setting.
This commit is contained in:
+28
-1
@@ -17,7 +17,15 @@ import {
|
||||
getOpenAiSpeechApiKeyProtection,
|
||||
hasOpenAiSpeechApiKey
|
||||
} from '../speech/openai-api-key-store'
|
||||
import { saveVerifiedOpenAiSpeechApiKey } from '../speech/openai-api-key-verification'
|
||||
import {
|
||||
saveVerifiedOpenAiCompatibleSpeechProvider,
|
||||
saveVerifiedOpenAiSpeechApiKey
|
||||
} from '../speech/openai-api-key-verification'
|
||||
import {
|
||||
clearOpenAiCompatibleSpeechProvider,
|
||||
summarizeOpenAiCompatibleSpeechProvider
|
||||
} from '../speech/openai-compatible-speech-provider-store'
|
||||
import type { OpenAiCompatibleSpeechProvider } from '../../shared/speech-types'
|
||||
import type { Store } from '../persistence/loading-store/store'
|
||||
|
||||
export function registerSpeechHandlers(store: Store): void {
|
||||
@@ -43,6 +51,25 @@ export function registerSpeechHandlers(store: Store): void {
|
||||
return { configured: false, protection: null }
|
||||
})
|
||||
|
||||
ipcMain.handle('speech:getOpenAiCompatibleProvider', () => {
|
||||
try {
|
||||
return summarizeOpenAiCompatibleSpeechProvider()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
})
|
||||
|
||||
ipcMain.handle(
|
||||
'speech:saveOpenAiCompatibleProvider',
|
||||
async (_event, input: OpenAiCompatibleSpeechProvider) => {
|
||||
await saveVerifiedOpenAiCompatibleSpeechProvider(input)
|
||||
}
|
||||
)
|
||||
|
||||
ipcMain.handle('speech:clearOpenAiCompatibleProvider', () => {
|
||||
clearOpenAiCompatibleSpeechProvider()
|
||||
})
|
||||
|
||||
ipcMain.handle('speech:downloadModel', async (event, modelId: string) => {
|
||||
const manager = getSpeechModelManager(store)
|
||||
const window = BrowserWindow.fromWebContents(event.sender)
|
||||
|
||||
@@ -18,6 +18,7 @@ import { API_KEY_ENVELOPE_PREFIX, getZcodePlanApiKeyPath } from '../zcode/zcode-
|
||||
import { COOKIE_ENVELOPE_PREFIX, getMiniMaxCookiePath } from '../minimax/minimax-cookie-store'
|
||||
import { miniMaxApiKeyEnvelopeFile } from '../minimax/minimax-api-key-store'
|
||||
import { openCodeGoApiKeyEnvelopeFile } from '../opencode/opencode-go-api-key-store'
|
||||
import { openAiCompatibleSpeechProviderEnvelopeFile } from '../speech/openai-compatible-speech-provider-store'
|
||||
|
||||
type PlaintextCredentialTarget = {
|
||||
path: string
|
||||
@@ -53,7 +54,8 @@ function listPlaintextCredentialTargets(): PlaintextCredentialTarget[] {
|
||||
{ path: getMiniMaxCookiePath(), envelopePrefix: COOKIE_ENVELOPE_PREFIX, raw: true },
|
||||
{ path: getZcodePlanApiKeyPath(), envelopePrefix: API_KEY_ENVELOPE_PREFIX },
|
||||
miniMaxApiKeyEnvelopeFile(),
|
||||
openCodeGoApiKeyEnvelopeFile()
|
||||
openCodeGoApiKeyEnvelopeFile(),
|
||||
openAiCompatibleSpeechProviderEnvelopeFile()
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { ModelManager } from '../speech/model-manager'
|
||||
import { openNodeSpeechDownloadRequest } from '../speech/node-speech-download-request'
|
||||
import { canReadOpenAiSpeechApiKey } from '../speech/openai-api-key-store'
|
||||
import { canReadCloudSpeechCredentials } from '../speech/cloud-transcription-target'
|
||||
import type { SpeechServiceFactories } from '../speech/speech-runtime-service'
|
||||
import { SttService } from '../speech/stt-service'
|
||||
import { createSttGpuPolicy } from '../speech/stt-gpu-failure-marker'
|
||||
@@ -11,7 +11,7 @@ import { spawnSttForkedProcess } from '../speech/stt-worker-forked-process'
|
||||
export const orcadSpeechServiceFactories: SpeechServiceFactories = {
|
||||
createModelManager: (customModelsDir) =>
|
||||
// Headless key reads cannot open keychain prompts, so readiness may verify decryption.
|
||||
new ModelManager(customModelsDir, openNodeSpeechDownloadRequest, canReadOpenAiSpeechApiKey),
|
||||
new ModelManager(customModelsDir, openNodeSpeechDownloadRequest, canReadCloudSpeechCredentials),
|
||||
createSttService: (models, gpuEnabled) => {
|
||||
const gpu = createSttGpuPolicy(models.getModelsDir(), gpuEnabled)
|
||||
return new SttService(models, () => spawnSttWorkerWithCpuFallback(spawnSttForkedProcess, gpu))
|
||||
|
||||
@@ -43,14 +43,18 @@ export class RuntimeMobileSpeechCatalog {
|
||||
const runnable = SPEECH_MODEL_CATALOG.filter(
|
||||
(manifest) =>
|
||||
isOfferedSpeechModel(manifest) &&
|
||||
(!isLocalSpeechModel(manifest) || hasLocalSpeechRecognizer(manifest))
|
||||
(!isLocalSpeechModel(manifest) || hasLocalSpeechRecognizer(manifest)) &&
|
||||
// Phones offer an unready cloud row as "add an OpenAI key", which can't set this one up.
|
||||
(manifest.provider !== 'openai-compatible' ||
|
||||
stateById.get(manifest.id)?.status === 'ready')
|
||||
)
|
||||
const models: RuntimeSpeechModelSummary[] = runnable.map((manifest) => {
|
||||
const state = stateById.get(manifest.id)
|
||||
return {
|
||||
id: manifest.id,
|
||||
label: manifest.label,
|
||||
provider: manifest.provider === 'openai' ? 'openai' : 'local',
|
||||
// Older phones know only these two; both cloud providers transcribe on this host.
|
||||
provider: isLocalSpeechModel(manifest) ? 'local' : 'openai',
|
||||
sizeBytes: manifest.sizeBytes ?? null,
|
||||
recommended: manifest.recommended === true,
|
||||
status: state?.status ?? 'not-downloaded',
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { SpeechModelProvider } from '../../shared/speech-types'
|
||||
import { getCatalogModel, replaceRetiredSpeechModelId } from './model-catalog'
|
||||
import {
|
||||
canReadOpenAiSpeechApiKey,
|
||||
hasOpenAiSpeechApiKey,
|
||||
readOpenAiSpeechApiKey
|
||||
} from './openai-api-key-store'
|
||||
import {
|
||||
canReadOpenAiCompatibleSpeechProvider,
|
||||
hasOpenAiCompatibleSpeechProvider,
|
||||
openAiCompatibleTranscriptionUrl,
|
||||
readOpenAiCompatibleSpeechProvider
|
||||
} from './openai-compatible-speech-provider-store'
|
||||
|
||||
export const OPENAI_TRANSCRIPTION_URL = 'https://api.openai.com/v1/audio/transcriptions'
|
||||
|
||||
const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record<string, string> = {
|
||||
'openai-gpt-transcribe': 'gpt-transcribe'
|
||||
}
|
||||
|
||||
export type CloudTranscriptionTarget = {
|
||||
url: string
|
||||
model: string
|
||||
/** Empty when the server takes no key. */
|
||||
apiKey: string
|
||||
/** Names the service in errors. */
|
||||
label: string
|
||||
}
|
||||
|
||||
export function hasCloudSpeechCredentials(provider: SpeechModelProvider): boolean {
|
||||
return provider === 'openai-compatible'
|
||||
? hasOpenAiCompatibleSpeechProvider()
|
||||
: hasOpenAiSpeechApiKey()
|
||||
}
|
||||
|
||||
/** Headless hosts can't raise keychain prompts, so their readiness verifies decryption. */
|
||||
export function canReadCloudSpeechCredentials(provider: SpeechModelProvider): boolean {
|
||||
return provider === 'openai-compatible'
|
||||
? canReadOpenAiCompatibleSpeechProvider()
|
||||
: canReadOpenAiSpeechApiKey()
|
||||
}
|
||||
|
||||
/** Reads the key (and the compatible provider's URL) only when audio is ready to upload. */
|
||||
export function resolveCloudTranscriptionTarget(modelId: string): CloudTranscriptionTarget {
|
||||
if (getCatalogModel(modelId)?.provider === 'openai-compatible') {
|
||||
const provider = readOpenAiCompatibleSpeechProvider()
|
||||
if (!provider) {
|
||||
throw new Error('The OpenAI-compatible speech provider is not set up')
|
||||
}
|
||||
const url = openAiCompatibleTranscriptionUrl(provider.baseUrl)
|
||||
return { url, model: provider.model, apiKey: provider.apiKey, label: new URL(url).host }
|
||||
}
|
||||
// Settings load already moves retired ids; this catches a caller still holding one.
|
||||
const model = OPENAI_TRANSCRIPTION_MODEL_BY_ID[replaceRetiredSpeechModelId(modelId)]
|
||||
if (!model) {
|
||||
throw new Error(`Unknown OpenAI transcription model: ${modelId}`)
|
||||
}
|
||||
return { url: OPENAI_TRANSCRIPTION_URL, model, apiKey: readOpenAiSpeechApiKey(), label: 'OpenAI' }
|
||||
}
|
||||
@@ -142,6 +142,18 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
||||
sampleRate: 16000,
|
||||
streaming: false
|
||||
},
|
||||
{
|
||||
id: 'openai-compatible',
|
||||
label: 'OpenAI-compatible',
|
||||
description:
|
||||
'Any service with an OpenAI-style transcription API, such as Groq, Mistral or a local server.',
|
||||
type: 'openai',
|
||||
provider: 'openai-compatible',
|
||||
language: 'multilingual',
|
||||
dictationLanguages: DICTATION_LANGUAGES,
|
||||
sampleRate: 16000,
|
||||
streaming: false
|
||||
},
|
||||
{
|
||||
id: 'openai-gpt-4o-mini-transcribe',
|
||||
label: 'GPT-4o mini Transcribe',
|
||||
|
||||
@@ -7,7 +7,7 @@ import type {
|
||||
SpeechModelStatus
|
||||
} from '../../shared/speech-types'
|
||||
import { SPEECH_MODEL_CATALOG, getCatalogModel, isLocalSpeechModel } from './model-catalog'
|
||||
import { hasOpenAiSpeechApiKey } from './openai-api-key-store'
|
||||
import { hasCloudSpeechCredentials } from './cloud-transcription-target'
|
||||
import {
|
||||
getSpeechModelCacheDirCandidates,
|
||||
migrateSpeechModelCacheIfNeeded,
|
||||
@@ -38,7 +38,7 @@ export class ModelManager extends SpeechModelDownloadTransport {
|
||||
constructor(
|
||||
customModelsDir: string | undefined,
|
||||
openRequest: OpenSpeechDownloadRequest,
|
||||
private readonly hasOpenAiKey: () => boolean = hasOpenAiSpeechApiKey
|
||||
private readonly hasCloudCredentials = hasCloudSpeechCredentials
|
||||
) {
|
||||
super(openRequest)
|
||||
const requestedModelsDir =
|
||||
@@ -103,8 +103,9 @@ export class ModelManager extends SpeechModelDownloadTransport {
|
||||
return { id: modelId, status: 'error', error: 'Unknown model' }
|
||||
}
|
||||
|
||||
if (manifest.provider === 'openai') {
|
||||
return { id: modelId, status: this.hasOpenAiKey() ? 'ready' : 'not-downloaded' }
|
||||
if (!isLocalSpeechModel(manifest)) {
|
||||
const ready = this.hasCloudCredentials(manifest.provider)
|
||||
return { id: modelId, status: ready ? 'ready' : 'not-downloaded' }
|
||||
}
|
||||
|
||||
const modelDir = this.getModelDir(modelId)
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
import { saveOpenAiSpeechApiKey } from './openai-api-key-store'
|
||||
import { OPENAI_TRANSCRIPTION_URL } from './cloud-transcription-target'
|
||||
import {
|
||||
openAiCompatibleTranscriptionUrl,
|
||||
resolveOpenAiCompatibleSpeechProviderInput,
|
||||
saveOpenAiCompatibleSpeechProvider
|
||||
} from './openai-compatible-speech-provider-store'
|
||||
import type { OpenAiCompatibleSpeechProvider } from '../../shared/speech-types'
|
||||
import {
|
||||
postOpenAiTranscription,
|
||||
readOpenAiErrorMessage,
|
||||
@@ -11,16 +18,27 @@ const KEY_CHECK_TIMEOUT_MS = 10_000
|
||||
export async function saveVerifiedOpenAiSpeechApiKey(apiKey: string): Promise<void> {
|
||||
const trimmed = apiKey.trim()
|
||||
if (trimmed) {
|
||||
await rejectUnauthorizedOpenAiKey(trimmed)
|
||||
await rejectUnauthorizedKey(OPENAI_TRANSCRIPTION_URL, trimmed, 'OpenAI')
|
||||
}
|
||||
saveOpenAiSpeechApiKey(apiKey)
|
||||
}
|
||||
|
||||
async function rejectUnauthorizedOpenAiKey(apiKey: string): Promise<void> {
|
||||
/** Same check against the provider's own endpoint, which also catches a wrong base URL. */
|
||||
export async function saveVerifiedOpenAiCompatibleSpeechProvider(
|
||||
input: OpenAiCompatibleSpeechProvider
|
||||
): Promise<void> {
|
||||
const provider = resolveOpenAiCompatibleSpeechProviderInput(input)
|
||||
const url = openAiCompatibleTranscriptionUrl(provider.baseUrl)
|
||||
await rejectUnauthorizedKey(url, provider.apiKey, new URL(url).host)
|
||||
saveOpenAiCompatibleSpeechProvider(provider)
|
||||
}
|
||||
|
||||
async function rejectUnauthorizedKey(url: string, apiKey: string, label: string): Promise<void> {
|
||||
let response: Response
|
||||
try {
|
||||
// An empty form is refused before any transcription, but only after OpenAI checks the key.
|
||||
// An empty form is refused before any transcription, but only after the server checks the key.
|
||||
response = await postOpenAiTranscription(
|
||||
url,
|
||||
apiKey,
|
||||
new FormData(),
|
||||
AbortSignal.timeout(KEY_CHECK_TIMEOUT_MS)
|
||||
@@ -28,10 +46,15 @@ async function rejectUnauthorizedOpenAiKey(apiKey: string): Promise<void> {
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
if (response.status === 404 || response.status === 405) {
|
||||
void response.body?.cancel().catch(() => undefined)
|
||||
throw new Error(`${label} has no transcription endpoint at ${url}. Check the base URL.`)
|
||||
}
|
||||
if (response.status !== 401) {
|
||||
void response.body?.cancel().catch(() => undefined)
|
||||
return
|
||||
}
|
||||
const data: OpenAiTranscriptionResponse = await response.json().catch(() => ({}))
|
||||
throw new Error(readOpenAiErrorMessage(data) ?? 'OpenAI rejected this API key.')
|
||||
const fallback = apiKey ? `${label} rejected this API key.` : `${label} needs an API key.`
|
||||
throw new Error(readOpenAiErrorMessage(data, apiKey) ?? fallback)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { mkdtempSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
async function loadStoreModule() {
|
||||
vi.resetModules()
|
||||
const { setSecretStore } = await import('../../shared/secret-store')
|
||||
setSecretStore({
|
||||
decryptString: (value: Buffer) => value.toString('utf8'),
|
||||
encryptString: (value: string) => Buffer.from(value),
|
||||
isEncryptionAvailable: () => true,
|
||||
describeProtectionGap: () => null
|
||||
})
|
||||
return import('./openai-compatible-speech-provider-store')
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'orca-compatible-speech-'))
|
||||
vi.stubEnv('HOME', home)
|
||||
vi.stubEnv('USERPROFILE', home)
|
||||
})
|
||||
|
||||
describe('openAiCompatibleTranscriptionUrl', () => {
|
||||
it('appends the transcriptions path to a base URL and keeps a full one', async () => {
|
||||
const { openAiCompatibleTranscriptionUrl } = await loadStoreModule()
|
||||
expect(openAiCompatibleTranscriptionUrl(' https://api.groq.com/openai/v1/ ')).toBe(
|
||||
'https://api.groq.com/openai/v1/audio/transcriptions'
|
||||
)
|
||||
expect(openAiCompatibleTranscriptionUrl('https://api.mistral.ai/v1/audio/transcriptions')).toBe(
|
||||
'https://api.mistral.ai/v1/audio/transcriptions'
|
||||
)
|
||||
expect(openAiCompatibleTranscriptionUrl('http://127.0.0.1:8000/v1')).toBe(
|
||||
'http://127.0.0.1:8000/v1/audio/transcriptions'
|
||||
)
|
||||
expect(() => openAiCompatibleTranscriptionUrl('file:///etc/passwd')).toThrow()
|
||||
expect(() => openAiCompatibleTranscriptionUrl('api.groq.com')).toThrow()
|
||||
expect(() => openAiCompatibleTranscriptionUrl('https://user:pw@host.example/v1')).toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveOpenAiCompatibleSpeechProviderInput', () => {
|
||||
it('keeps a saved key for a blank field only while the base URL is unchanged', async () => {
|
||||
const store = await loadStoreModule()
|
||||
store.saveOpenAiCompatibleSpeechProvider({
|
||||
baseUrl: 'https://api.groq.com/openai/v1',
|
||||
model: 'whisper-large-v3-turbo',
|
||||
apiKey: 'gsk_saved'
|
||||
})
|
||||
|
||||
expect(
|
||||
store.resolveOpenAiCompatibleSpeechProviderInput({
|
||||
baseUrl: 'https://api.groq.com/openai/v1/',
|
||||
model: 'whisper-large-v3',
|
||||
apiKey: ''
|
||||
}).apiKey
|
||||
).toBe('gsk_saved')
|
||||
expect(
|
||||
store.resolveOpenAiCompatibleSpeechProviderInput({
|
||||
baseUrl: 'https://evil.example/v1',
|
||||
model: 'whisper-large-v3',
|
||||
apiKey: ''
|
||||
}).apiKey
|
||||
).toBe('')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,98 @@
|
||||
import { z } from 'zod'
|
||||
import { createEncryptedApiKeyFileStore } from '../credentials/encrypted-api-key-file-store'
|
||||
import { savedCredentialUnreadable } from '../credentials/saved-credential-readability'
|
||||
import type {
|
||||
OpenAiCompatibleSpeechProvider,
|
||||
OpenAiCompatibleSpeechProviderSummary
|
||||
} from '../../shared/speech-types'
|
||||
|
||||
// URL, model and key are sealed together so a key can never be sent to a host it wasn't saved for.
|
||||
const store = createEncryptedApiKeyFileStore({
|
||||
fileName: 'openai-compatible-speech-provider.enc',
|
||||
envelopePrefix: 'orca-openai-compatible-speech:v1:',
|
||||
providerLabel: 'OpenAI-compatible speech provider',
|
||||
logScope: 'speech'
|
||||
})
|
||||
|
||||
export const hasOpenAiCompatibleSpeechProvider = store.has
|
||||
export const clearOpenAiCompatibleSpeechProvider = store.clear
|
||||
export const openAiCompatibleSpeechProviderEnvelopeFile = store.envelopeFile
|
||||
|
||||
const TRANSCRIPTION_PATH = '/audio/transcriptions'
|
||||
|
||||
const providerSchema = z.object({
|
||||
baseUrl: z.string().catch(''),
|
||||
model: z.string().catch(''),
|
||||
apiKey: z.string().catch('')
|
||||
}) satisfies z.ZodType<OpenAiCompatibleSpeechProvider>
|
||||
|
||||
/** Accepts a base like `https://api.groq.com/openai/v1` or the full transcriptions URL. */
|
||||
export function openAiCompatibleTranscriptionUrl(baseUrl: string): string {
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(baseUrl.trim())
|
||||
} catch {
|
||||
throw new Error('Enter a full base URL, such as https://api.groq.com/openai/v1')
|
||||
}
|
||||
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
|
||||
throw new Error('The base URL must start with https:// or http://')
|
||||
}
|
||||
// fetch refuses these, and its error would echo the password.
|
||||
if (url.username || url.password) {
|
||||
throw new Error('Put the key in the API Key field, not in the base URL')
|
||||
}
|
||||
const path = url.pathname.replace(/\/+$/, '')
|
||||
url.pathname = path.endsWith(TRANSCRIPTION_PATH) ? path : `${path}${TRANSCRIPTION_PATH}`
|
||||
return url.toString()
|
||||
}
|
||||
|
||||
export function readOpenAiCompatibleSpeechProvider(): OpenAiCompatibleSpeechProvider | null {
|
||||
// The store caches what it read; another process (desktop vs orca serve) may have removed the file since.
|
||||
const raw = store.has() ? store.read() : null
|
||||
if (raw === null) {
|
||||
return null
|
||||
}
|
||||
return providerSchema.parse(JSON.parse(raw))
|
||||
}
|
||||
|
||||
/** Headless hosts may not open a desktop-sealed file; checks the disk quietly, since readiness is polled. */
|
||||
export function canReadOpenAiCompatibleSpeechProvider(): boolean {
|
||||
return store.has() && savedCredentialUnreadable(store.envelopeFile()) === undefined
|
||||
}
|
||||
|
||||
export function summarizeOpenAiCompatibleSpeechProvider(): OpenAiCompatibleSpeechProviderSummary | null {
|
||||
const provider = readOpenAiCompatibleSpeechProvider()
|
||||
return provider
|
||||
? { baseUrl: provider.baseUrl, model: provider.model, hasApiKey: provider.apiKey !== '' }
|
||||
: null
|
||||
}
|
||||
|
||||
/** A blank key keeps the saved one only while the base URL is unchanged. */
|
||||
export function resolveOpenAiCompatibleSpeechProviderInput(
|
||||
input: OpenAiCompatibleSpeechProvider
|
||||
): OpenAiCompatibleSpeechProvider {
|
||||
const baseUrl = input.baseUrl.trim()
|
||||
const model = input.model.trim()
|
||||
openAiCompatibleTranscriptionUrl(baseUrl)
|
||||
if (!model) {
|
||||
throw new Error('Enter the model name the provider expects')
|
||||
}
|
||||
return { baseUrl, model, apiKey: input.apiKey.trim() || savedKeyFor(baseUrl) }
|
||||
}
|
||||
|
||||
function savedKeyFor(baseUrl: string): string {
|
||||
try {
|
||||
const saved = readOpenAiCompatibleSpeechProvider()
|
||||
const sameEndpoint =
|
||||
saved !== null &&
|
||||
openAiCompatibleTranscriptionUrl(saved.baseUrl) === openAiCompatibleTranscriptionUrl(baseUrl)
|
||||
return sameEndpoint ? saved.apiKey : ''
|
||||
} catch {
|
||||
// An unreadable or invalid saved provider has no key worth keeping.
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
export function saveOpenAiCompatibleSpeechProvider(provider: OpenAiCompatibleSpeechProvider): void {
|
||||
store.save(JSON.stringify(provider))
|
||||
}
|
||||
@@ -47,7 +47,15 @@ describe('OpenAiTranscriptionSession', () => {
|
||||
uploads.push(Buffer.from(await file.arrayBuffer()))
|
||||
return new Response(JSON.stringify({ text: ' complete dictation ' }))
|
||||
})
|
||||
const session = new OpenAiTranscriptionSession('openai-gpt-transcribe', () => 'test-key', 'de')
|
||||
const session = new OpenAiTranscriptionSession(
|
||||
() => ({
|
||||
url: 'https://api.openai.com/v1/audio/transcriptions',
|
||||
model: 'gpt-transcribe',
|
||||
apiKey: 'test-key',
|
||||
label: 'OpenAI'
|
||||
}),
|
||||
'de'
|
||||
)
|
||||
const input = new Float32Array([
|
||||
-2,
|
||||
-1,
|
||||
@@ -76,4 +84,31 @@ describe('OpenAiTranscriptionSession', () => {
|
||||
-32768, -32768, -16384, 0, 0, 16384, 32767, 32767, 0, 32767, -32768, 0, 24575
|
||||
])
|
||||
})
|
||||
|
||||
it('sends a keyless server the singular language field and no Authorization header', async () => {
|
||||
let sent: RequestInit | undefined
|
||||
vi.stubGlobal('fetch', async (_url: string, init: RequestInit) => {
|
||||
sent = init
|
||||
return new Response(JSON.stringify({ text: 'hola' }))
|
||||
})
|
||||
const session = new OpenAiTranscriptionSession(
|
||||
() => ({
|
||||
url: 'http://127.0.0.1:8000/v1/audio/transcriptions',
|
||||
model: 'whisper-large-v3-turbo',
|
||||
apiKey: '',
|
||||
label: '127.0.0.1:8000'
|
||||
}),
|
||||
'es'
|
||||
)
|
||||
session.feedAudio(new Float32Array(160), 16000)
|
||||
|
||||
expect(await session.finish()).toBe('hola')
|
||||
const form = sent?.body
|
||||
if (!(form instanceof FormData)) {
|
||||
throw new Error('Expected multipart body')
|
||||
}
|
||||
expect(form.get('language')).toBe('es')
|
||||
expect(form.has('languages[]')).toBe(false)
|
||||
expect(sent?.headers).toEqual({})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
import { getMainHttpClient } from '../network/http-client'
|
||||
import { resampleToRate } from './stt-audio-resample'
|
||||
import { replaceRetiredSpeechModelId } from './model-catalog'
|
||||
import type { CloudTranscriptionTarget } from './cloud-transcription-target'
|
||||
import {
|
||||
CLOUD_TRANSCRIPTION_SAMPLE_RATE,
|
||||
MAX_CLOUD_TRANSCRIPTION_SECONDS,
|
||||
cloudTranscriptionTimeoutMs
|
||||
} from '../../shared/dictation-finish-deadlines'
|
||||
|
||||
export const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record<string, string> = {
|
||||
'openai-gpt-transcribe': 'gpt-transcribe'
|
||||
}
|
||||
|
||||
export const OPENAI_TRANSCRIPTION_URL = 'https://api.openai.com/v1/audio/transcriptions'
|
||||
const MAX_CLOUD_AUDIO_SAMPLES = MAX_CLOUD_TRANSCRIPTION_SECONDS * CLOUD_TRANSCRIPTION_SAMPLE_RATE
|
||||
|
||||
export type OpenAiTranscriptionResponse = {
|
||||
@@ -21,17 +16,18 @@ export type OpenAiTranscriptionResponse = {
|
||||
}
|
||||
}
|
||||
|
||||
export function sanitizeOpenAiTranscriptionErrorMessage(message: string): string {
|
||||
export function sanitizeOpenAiTranscriptionErrorMessage(message: string, apiKey = ''): string {
|
||||
if (/incorrect api key provided:/i.test(message)) {
|
||||
return 'Incorrect OpenAI API key provided.'
|
||||
}
|
||||
|
||||
const sanitized = message
|
||||
// Other providers' keys have other shapes, so also redact the exact key sent (unless it's a placeholder like "x").
|
||||
const sanitized = (apiKey.length >= 8 ? message.split(apiKey).join('[redacted]') : message)
|
||||
.replace(/\bsk-[A-Za-z0-9_-]+/g, '[redacted]')
|
||||
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [redacted]')
|
||||
.trim()
|
||||
|
||||
return sanitized || 'OpenAI transcription request failed'
|
||||
return sanitized || 'Transcription request failed'
|
||||
}
|
||||
|
||||
function writePcm16Samples(view: DataView, samples: Float32Array, offset: number): void {
|
||||
@@ -72,33 +68,34 @@ function encodePcm16Wav(chunks: Float32Array[], sampleRate: number): Buffer<Arra
|
||||
|
||||
/** The one request both transcription and the key check send; the main HTTP client follows the app's proxy. */
|
||||
export function postOpenAiTranscription(
|
||||
url: string,
|
||||
apiKey: string,
|
||||
form: FormData,
|
||||
signal: AbortSignal
|
||||
): Promise<Response> {
|
||||
return getMainHttpClient().fetch(OPENAI_TRANSCRIPTION_URL, {
|
||||
return getMainHttpClient().fetch(url, {
|
||||
method: 'POST',
|
||||
// Bearer auth only: keep OpenAI's cookies out of the shared desktop session.
|
||||
// Bearer auth only: keep the provider's cookies out of the shared desktop session.
|
||||
credentials: 'omit',
|
||||
signal,
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
// Local servers that take no key get no Authorization header.
|
||||
headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {},
|
||||
body: form
|
||||
})
|
||||
}
|
||||
|
||||
export function readOpenAiErrorMessage(data: OpenAiTranscriptionResponse): string | undefined {
|
||||
export function readOpenAiErrorMessage(
|
||||
data: OpenAiTranscriptionResponse,
|
||||
apiKey = ''
|
||||
): string | undefined {
|
||||
return typeof data.error?.message === 'string'
|
||||
? sanitizeOpenAiTranscriptionErrorMessage(data.error.message)
|
||||
? sanitizeOpenAiTranscriptionErrorMessage(data.error.message, apiKey)
|
||||
: undefined
|
||||
}
|
||||
|
||||
function parseOpenAiTranscriptionResponse(data: OpenAiTranscriptionResponse): string {
|
||||
if (typeof data.text === 'string') {
|
||||
return data.text.trim()
|
||||
}
|
||||
throw new Error(
|
||||
readOpenAiErrorMessage(data) ?? 'OpenAI transcription response did not include text'
|
||||
)
|
||||
// gpt-transcribe takes a list of expected languages in place of the singular field others use.
|
||||
function languageField(model: string): string {
|
||||
return /^gpt-(live-)?transcribe/.test(model) ? 'languages[]' : 'language'
|
||||
}
|
||||
|
||||
export class OpenAiTranscriptionSession {
|
||||
@@ -107,8 +104,7 @@ export class OpenAiTranscriptionSession {
|
||||
private readonly abortController = new AbortController()
|
||||
|
||||
constructor(
|
||||
private readonly modelId: string,
|
||||
private readonly readApiKey: () => string,
|
||||
private readonly readTarget: () => CloudTranscriptionTarget,
|
||||
private readonly language?: string
|
||||
) {}
|
||||
|
||||
@@ -136,46 +132,46 @@ export class OpenAiTranscriptionSession {
|
||||
return ''
|
||||
}
|
||||
|
||||
// Settings load already moves retired ids; this catches a caller still holding one.
|
||||
const apiModel = OPENAI_TRANSCRIPTION_MODEL_BY_ID[replaceRetiredSpeechModelId(this.modelId)]
|
||||
if (!apiModel) {
|
||||
throw new Error(`Unknown OpenAI transcription model: ${this.modelId}`)
|
||||
}
|
||||
|
||||
const target = this.readTarget()
|
||||
const chunks = this.chunks
|
||||
this.chunks = []
|
||||
const wav = encodePcm16Wav(chunks, CLOUD_TRANSCRIPTION_SAMPLE_RATE)
|
||||
// No response_format: JSON is the default everywhere, and not every provider accepts the field.
|
||||
const form = new FormData()
|
||||
// No response_format: JSON is the default, and gpt-transcribe documents no other.
|
||||
form.append('model', apiModel)
|
||||
form.append('model', target.model)
|
||||
if (this.language) {
|
||||
// gpt-transcribe takes a list of expected languages in place of the singular `language`.
|
||||
form.append('languages[]', this.language)
|
||||
form.append(languageField(target.model), this.language)
|
||||
}
|
||||
// A named WAV blob satisfies multipart uploads without temporary files.
|
||||
form.append('file', new Blob([wav], { type: 'audio/wav' }), 'dictation.wav')
|
||||
|
||||
const timeout = AbortSignal.timeout(cloudTranscriptionTimeoutMs(this.audioSamples))
|
||||
try {
|
||||
return await this.upload(form, AbortSignal.any([this.abortController.signal, timeout]))
|
||||
return await upload(target, form, AbortSignal.any([this.abortController.signal, timeout]))
|
||||
} catch (error) {
|
||||
if (timeout.aborted && !this.abortController.signal.aborted) {
|
||||
throw new Error('OpenAI transcription timed out. Check your connection and try again.')
|
||||
throw new Error(
|
||||
`${target.label} transcription timed out. Check your connection and try again.`
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async upload(form: FormData, signal: AbortSignal): Promise<string> {
|
||||
const response = await postOpenAiTranscription(this.readApiKey(), form, signal)
|
||||
|
||||
const data = (await response.json().catch(() => ({}))) as OpenAiTranscriptionResponse
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`OpenAI transcription failed: ${readOpenAiErrorMessage(data) ?? response.statusText}`
|
||||
)
|
||||
}
|
||||
|
||||
return parseOpenAiTranscriptionResponse(data)
|
||||
async function upload(
|
||||
target: CloudTranscriptionTarget,
|
||||
form: FormData,
|
||||
signal: AbortSignal
|
||||
): Promise<string> {
|
||||
const response = await postOpenAiTranscription(target.url, target.apiKey, form, signal)
|
||||
const data = (await response.json().catch(() => ({}))) as OpenAiTranscriptionResponse
|
||||
const error = readOpenAiErrorMessage(data, target.apiKey)
|
||||
if (!response.ok) {
|
||||
throw new Error(`${target.label} transcription failed: ${error ?? response.statusText}`)
|
||||
}
|
||||
if (typeof data.text === 'string') {
|
||||
return data.text.trim()
|
||||
}
|
||||
throw new Error(error ?? `${target.label} transcription response did not include text`)
|
||||
}
|
||||
|
||||
@@ -89,10 +89,7 @@ const {
|
||||
static instances: HoistedMockOpenAiTranscriptionSession[] = []
|
||||
feedCalls: { samples: Float32Array; sampleRate: number }[] = []
|
||||
|
||||
constructor(
|
||||
readonly modelId: string,
|
||||
readonly readApiKey: () => string
|
||||
) {
|
||||
constructor(readonly readTarget: () => { model: string; apiKey: string }) {
|
||||
HoistedMockOpenAiTranscriptionSession.instances.push(this)
|
||||
}
|
||||
|
||||
@@ -101,7 +98,8 @@ const {
|
||||
}
|
||||
|
||||
finish(): Promise<string> {
|
||||
return Promise.resolve(`${this.modelId}:${this.readApiKey()}`)
|
||||
const target = this.readTarget()
|
||||
return Promise.resolve(`${target.model}:${target.apiKey}`)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,8 +183,11 @@ vi.mock('./model-catalog', () => ({
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('./openai-api-key-store', () => ({
|
||||
readOpenAiSpeechApiKey: readOpenAiSpeechApiKeyMock
|
||||
vi.mock('./cloud-transcription-target', () => ({
|
||||
resolveCloudTranscriptionTarget: (modelId: string) => ({
|
||||
model: modelId,
|
||||
apiKey: readOpenAiSpeechApiKeyMock()
|
||||
})
|
||||
}))
|
||||
|
||||
vi.mock('./openai-transcription-client', () => ({
|
||||
|
||||
@@ -2,7 +2,7 @@ import { engineDictationLanguage } from '../../shared/dictation-language'
|
||||
import type { SpeechModelManifest } from '../../shared/speech-types'
|
||||
import { getCatalogModel } from './model-catalog'
|
||||
import { OpenAiTranscriptionSession } from './openai-transcription-client'
|
||||
import { readOpenAiSpeechApiKey } from './openai-api-key-store'
|
||||
import { resolveCloudTranscriptionTarget } from './cloud-transcription-target'
|
||||
import type { SttEventSink } from './stt-service'
|
||||
import {
|
||||
MAX_STT_SESSIONS,
|
||||
@@ -56,7 +56,7 @@ export async function startSttDictation(
|
||||
throw new Error(`Unknown model: ${modelId}`)
|
||||
}
|
||||
const config: SttRecognizerConfig | null =
|
||||
manifest.provider === 'openai'
|
||||
manifest.provider !== 'local'
|
||||
? null
|
||||
: { modelId, hotwordsFilePath, language: engineDictationLanguage(manifest, language) }
|
||||
if (state.sessions.size >= MAX_STT_SESSIONS) {
|
||||
@@ -91,8 +91,7 @@ export async function startSttDictation(
|
||||
// A cloud dictation needs no recognizer; free one nobody else uses.
|
||||
await teardownIdleSttEngine(state)
|
||||
session.cloud = new OpenAiTranscriptionSession(
|
||||
modelId,
|
||||
readOpenAiSpeechApiKey,
|
||||
() => resolveCloudTranscriptionTarget(modelId),
|
||||
engineDictationLanguage(manifest, language)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection'
|
||||
import type {
|
||||
OpenAiCompatibleSpeechProvider,
|
||||
OpenAiCompatibleSpeechProviderSummary,
|
||||
SpeechErrorEvent,
|
||||
SpeechLifecycleEvent,
|
||||
SpeechModelManifest,
|
||||
@@ -16,6 +18,11 @@ export type SpeechApi = {
|
||||
}>
|
||||
saveOpenAiApiKey: (apiKey: string) => Promise<{ configured: boolean }>
|
||||
clearOpenAiApiKey: () => Promise<{ configured: boolean }>
|
||||
/** Never returns the saved key. */
|
||||
getOpenAiCompatibleProvider: () => Promise<OpenAiCompatibleSpeechProviderSummary | null>
|
||||
/** A blank key keeps the saved one while the base URL is unchanged. */
|
||||
saveOpenAiCompatibleProvider: (provider: OpenAiCompatibleSpeechProvider) => Promise<void>
|
||||
clearOpenAiCompatibleProvider: () => Promise<void>
|
||||
downloadModel: (modelId: string) => Promise<void>
|
||||
cancelDownload: (modelId: string) => Promise<void>
|
||||
deleteModel: (modelId: string) => Promise<void>
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection'
|
||||
import { ipcRenderer } from 'electron'
|
||||
import type {
|
||||
OpenAiCompatibleSpeechProvider,
|
||||
OpenAiCompatibleSpeechProviderSummary,
|
||||
SpeechAudioPortMessage,
|
||||
SpeechErrorEvent,
|
||||
SpeechLifecycleEvent,
|
||||
@@ -24,6 +26,12 @@ export const speechApi = {
|
||||
ipcRenderer.invoke('speech:saveOpenAiApiKey', apiKey),
|
||||
clearOpenAiApiKey: (): Promise<{ configured: boolean }> =>
|
||||
ipcRenderer.invoke('speech:clearOpenAiApiKey'),
|
||||
getOpenAiCompatibleProvider: (): Promise<OpenAiCompatibleSpeechProviderSummary | null> =>
|
||||
ipcRenderer.invoke('speech:getOpenAiCompatibleProvider'),
|
||||
saveOpenAiCompatibleProvider: (provider: OpenAiCompatibleSpeechProvider): Promise<void> =>
|
||||
ipcRenderer.invoke('speech:saveOpenAiCompatibleProvider', provider),
|
||||
clearOpenAiCompatibleProvider: (): Promise<void> =>
|
||||
ipcRenderer.invoke('speech:clearOpenAiCompatibleProvider'),
|
||||
downloadModel: (modelId: string): Promise<void> =>
|
||||
ipcRenderer.invoke('speech:downloadModel', modelId),
|
||||
cancelDownload: (modelId: string): Promise<void> =>
|
||||
|
||||
@@ -81,7 +81,7 @@ function useIsCloudModel(): boolean {
|
||||
}
|
||||
void catalogRequest.then((catalog) => {
|
||||
if (current) {
|
||||
setCloud(catalog.some((model) => model.id === modelId && model.provider === 'openai'))
|
||||
setCloud(catalog.some((model) => model.id === modelId && model.provider !== 'local'))
|
||||
}
|
||||
})
|
||||
return () => {
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { Loader2, Lock } from 'lucide-react'
|
||||
import { toast } from 'sonner'
|
||||
import { Button } from '../../ui/button'
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle
|
||||
} from '../../ui/dialog'
|
||||
import { Input } from '../../ui/input'
|
||||
import { Label } from '../../ui/label'
|
||||
import { useTranslate } from '@/i18n/use-translate'
|
||||
import { readIpcErrorMessage } from '@/lib/ipc-error'
|
||||
import type { OpenAiCompatibleSpeechProviderSummary } from '../../../../../shared/speech-types'
|
||||
|
||||
type OpenAiCompatibleProviderDialogProps = {
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
onSaved: () => void
|
||||
}
|
||||
|
||||
export function OpenAiCompatibleProviderDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
onSaved
|
||||
}: OpenAiCompatibleProviderDialogProps): React.JSX.Element {
|
||||
const t = useTranslate()
|
||||
const [baseUrl, setBaseUrl] = useState('')
|
||||
const [model, setModel] = useState('')
|
||||
const [apiKey, setApiKey] = useState('')
|
||||
const [saved, setSaved] = useState<OpenAiCompatibleSpeechProviderSummary | null>(null)
|
||||
const [pending, setPending] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
setApiKey('')
|
||||
void window.api.speech
|
||||
.getOpenAiCompatibleProvider()
|
||||
.then((summary) => {
|
||||
if (cancelled) {
|
||||
return
|
||||
}
|
||||
setBaseUrl(summary?.baseUrl ?? '')
|
||||
setModel(summary?.model ?? '')
|
||||
setSaved(summary)
|
||||
})
|
||||
.catch(() => {})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [open])
|
||||
|
||||
const canSave = !pending && baseUrl.trim() !== '' && model.trim() !== ''
|
||||
const keepsSavedKey = saved?.hasApiKey === true && saved.baseUrl === baseUrl.trim()
|
||||
|
||||
const save = async (event: React.FormEvent): Promise<void> => {
|
||||
event.preventDefault()
|
||||
if (!canSave) {
|
||||
return
|
||||
}
|
||||
setPending(true)
|
||||
try {
|
||||
await window.api.speech.saveOpenAiCompatibleProvider({ baseUrl, model, apiKey })
|
||||
onSaved()
|
||||
onOpenChange(false)
|
||||
} catch (err) {
|
||||
toast.error(
|
||||
readIpcErrorMessage(err) ??
|
||||
t(
|
||||
'components.settings.voice.openAiCompatible.updateFailed',
|
||||
'Could not update the provider'
|
||||
)
|
||||
)
|
||||
}
|
||||
setPending(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
{t('components.settings.voice.openAiCompatible.title', 'OpenAI-compatible provider')}
|
||||
</DialogTitle>
|
||||
<DialogDescription>
|
||||
{t(
|
||||
'components.settings.voice.openAiCompatible.description',
|
||||
'Audio goes to this service only while the OpenAI-compatible model is selected. It works with Groq, Mistral, OpenAI and local servers.'
|
||||
)}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<form className="space-y-3" onSubmit={(event) => void save(event)}>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="openai-compatible-base-url">
|
||||
{t('components.settings.voice.openAiCompatible.baseUrl', 'Base URL')}
|
||||
</Label>
|
||||
<Input
|
||||
id="openai-compatible-base-url"
|
||||
value={baseUrl}
|
||||
placeholder={t(
|
||||
'components.settings.voice.openAiCompatible.baseUrlPlaceholder',
|
||||
'https://api.groq.com/openai/v1'
|
||||
)}
|
||||
spellCheck={false}
|
||||
disabled={pending}
|
||||
onChange={(event) => setBaseUrl(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="openai-compatible-model">
|
||||
{t('components.settings.voice.openAiCompatible.model', 'Model')}
|
||||
</Label>
|
||||
<Input
|
||||
id="openai-compatible-model"
|
||||
value={model}
|
||||
placeholder={t(
|
||||
'components.settings.voice.openAiCompatible.modelPlaceholder',
|
||||
'whisper-large-v3-turbo'
|
||||
)}
|
||||
spellCheck={false}
|
||||
disabled={pending}
|
||||
onChange={(event) => setModel(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="openai-compatible-api-key">
|
||||
{t('components.settings.voice.openAiCompatible.apiKey', 'API Key')}
|
||||
</Label>
|
||||
<Input
|
||||
id="openai-compatible-api-key"
|
||||
type="password"
|
||||
value={apiKey}
|
||||
placeholder={
|
||||
keepsSavedKey
|
||||
? t('components.settings.voice.openAiCompatible.keySaved', 'Saved key kept')
|
||||
: t(
|
||||
'components.settings.voice.openAiCompatible.keyOptional',
|
||||
'Optional for local servers'
|
||||
)
|
||||
}
|
||||
disabled={pending}
|
||||
onChange={(event) => setApiKey(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<p className="flex items-center gap-1.5 text-[11px] text-muted-foreground/70">
|
||||
<Lock className="size-3 shrink-0" />
|
||||
{t(
|
||||
'auto.components.settings.OpenAiTranscriptionKeyDialog.d246b2bdb3',
|
||||
'Local runtime keys are stored in ~/.orca using Electron encrypted storage when available.'
|
||||
)}
|
||||
</p>
|
||||
<DialogFooter>
|
||||
<Button type="submit" disabled={!canSave}>
|
||||
{pending ? <Loader2 className="size-4 animate-spin" /> : null}
|
||||
{t('components.settings.voice.openAiCompatible.save', 'Save')}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import { Label } from '../../ui/label'
|
||||
import { useTranslate } from '@/i18n/use-translate'
|
||||
|
||||
type OpenAiTranscriptionSettingsRowProps = {
|
||||
/** 'compatible' is the base URL + model provider, set up in its own dialog. */
|
||||
variant?: 'openai' | 'compatible'
|
||||
configured: boolean
|
||||
disabled: boolean
|
||||
onConfigure: () => void
|
||||
@@ -11,23 +13,68 @@ type OpenAiTranscriptionSettingsRowProps = {
|
||||
}
|
||||
|
||||
export function OpenAiTranscriptionSettingsRow({
|
||||
variant = 'openai',
|
||||
configured,
|
||||
disabled,
|
||||
onConfigure,
|
||||
onClear
|
||||
}: OpenAiTranscriptionSettingsRowProps): React.JSX.Element {
|
||||
const t = useTranslate()
|
||||
const copy =
|
||||
variant === 'compatible'
|
||||
? {
|
||||
title: t(
|
||||
'components.settings.voice.openAiCompatible.title',
|
||||
'OpenAI-compatible provider'
|
||||
),
|
||||
configured: t(
|
||||
'components.settings.voice.openAiCompatible.rowConfigured',
|
||||
'Base URL and model saved for the OpenAI-compatible model.'
|
||||
),
|
||||
missing: t(
|
||||
'components.settings.voice.openAiCompatible.rowMissing',
|
||||
'Add a base URL and model before selecting the OpenAI-compatible model.'
|
||||
),
|
||||
edit: t('components.settings.voice.openAiCompatible.edit', 'Edit'),
|
||||
clear: t(
|
||||
'components.settings.voice.openAiCompatible.removeLabel',
|
||||
'Remove OpenAI-compatible provider'
|
||||
),
|
||||
add: t('components.settings.voice.openAiCompatible.setUp', 'Set up')
|
||||
}
|
||||
: {
|
||||
title: t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.27e0cb656d',
|
||||
'OpenAI Transcription'
|
||||
),
|
||||
configured: t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.b59b9b2b51',
|
||||
'API key configured for cloud speech-to-text models.'
|
||||
),
|
||||
missing: t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.893790e13b',
|
||||
'Add an OpenAI API key before selecting cloud speech-to-text models.'
|
||||
),
|
||||
edit: t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.a622bc3b37',
|
||||
'Replace key'
|
||||
),
|
||||
clear: t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.ae2df8f511',
|
||||
'Disconnect OpenAI API key'
|
||||
),
|
||||
add: t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.85c589cd61',
|
||||
'Add API key'
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-4 py-2">
|
||||
<div className="min-w-0 space-y-0.5">
|
||||
<div className="flex items-center gap-2">
|
||||
<Cloud className="size-4 shrink-0 text-muted-foreground" />
|
||||
<Label>
|
||||
{t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.27e0cb656d',
|
||||
'OpenAI Transcription'
|
||||
)}
|
||||
</Label>
|
||||
<Label>{copy.title}</Label>
|
||||
{configured && (
|
||||
<span className="flex items-center gap-1 text-xs text-muted-foreground">
|
||||
<CheckCircle2 className="size-3.5" />
|
||||
@@ -36,28 +83,17 @@ export function OpenAiTranscriptionSettingsRow({
|
||||
)}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{configured
|
||||
? t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.b59b9b2b51',
|
||||
'API key configured for cloud speech-to-text models.'
|
||||
)
|
||||
: t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.893790e13b',
|
||||
'Add an OpenAI API key before selecting cloud speech-to-text models.'
|
||||
)}
|
||||
{configured ? copy.configured : copy.missing}
|
||||
</p>
|
||||
</div>
|
||||
{configured ? (
|
||||
<div className="flex shrink-0 items-center gap-1.5">
|
||||
<Button variant="outline" size="sm" disabled={disabled} onClick={onConfigure}>
|
||||
{t('auto.components.settings.OpenAiTranscriptionSettingsRow.a622bc3b37', 'Replace key')}
|
||||
{copy.edit}
|
||||
</Button>
|
||||
<button
|
||||
onClick={onClear}
|
||||
aria-label={t(
|
||||
'auto.components.settings.OpenAiTranscriptionSettingsRow.ae2df8f511',
|
||||
'Disconnect OpenAI API key'
|
||||
)}
|
||||
aria-label={copy.clear}
|
||||
disabled={disabled}
|
||||
className="rounded-md p-1 text-muted-foreground/50 transition-colors hover:text-destructive disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
@@ -66,7 +102,7 @@ export function OpenAiTranscriptionSettingsRow({
|
||||
</div>
|
||||
) : (
|
||||
<Button variant="outline" size="sm" disabled={disabled} onClick={onConfigure}>
|
||||
{t('auto.components.settings.OpenAiTranscriptionSettingsRow.85c589cd61', 'Add API key')}
|
||||
{copy.add}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -45,7 +45,7 @@ export function VoiceModelUnloadSetting({
|
||||
const t = useTranslate()
|
||||
const intlLocale = useIntlLocale()
|
||||
const labelId = useId()
|
||||
const isCloudModel = selectedModel?.provider === 'openai'
|
||||
const isCloudModel = selectedModel !== undefined && selectedModel.provider !== 'local'
|
||||
const current = unloadModelAfterMinutes(voiceSettings)
|
||||
|
||||
return (
|
||||
|
||||
@@ -8,6 +8,7 @@ import { Separator } from '../../ui/separator'
|
||||
import { toast } from 'sonner'
|
||||
import { useAppStore } from '@/store/index'
|
||||
import { OpenAiTranscriptionKeyDialog } from './OpenAiTranscriptionKeyDialog'
|
||||
import { OpenAiCompatibleProviderDialog } from './OpenAiCompatibleProviderDialog'
|
||||
import { OpenAiTranscriptionSettingsRow } from './OpenAiTranscriptionSettingsRow'
|
||||
import { handleVoiceDictationToggle } from './voice-dictation-toggle'
|
||||
import { VoiceDictationLanguageSetting } from './VoiceDictationLanguageSetting'
|
||||
@@ -43,6 +44,7 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J
|
||||
null
|
||||
)
|
||||
const [pendingCloudModelId, setPendingCloudModelId] = useState<string | null>(null)
|
||||
const [compatibleDialogOpen, setCompatibleDialogOpen] = useState(false)
|
||||
const mountedRef = useRef(true)
|
||||
// Merge async whole-object writes into committed current settings; render snapshots could undo toggles changed during IPC and disable the microphone picker.
|
||||
const voiceSettingsRef = useRef(voiceSettings)
|
||||
@@ -146,12 +148,58 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J
|
||||
(settingsSearchQuery.trim() !== '' &&
|
||||
matchesSettingsSearch(settingsSearchQuery, getOpenaiTranscriptionSearchEntry()))
|
||||
|
||||
const compatibleModel = catalog.find((m) => m.provider === 'openai-compatible')
|
||||
const compatibleConfigured =
|
||||
compatibleModel !== undefined &&
|
||||
modelStates.some((s) => s.id === compatibleModel.id && s.status === 'ready')
|
||||
const showCompatibleSettingsRow =
|
||||
compatibleConfigured || selectedModel?.provider === 'openai-compatible'
|
||||
|
||||
const openOpenAiDialog = (modelId: string | null = null): void => {
|
||||
setPendingCloudModelId(modelId)
|
||||
setOpenAiApiKeyDraft('')
|
||||
setOpenAiDialogOpen(true)
|
||||
}
|
||||
|
||||
const openCompatibleDialog = (modelId: string | null): void => {
|
||||
setPendingCloudModelId(modelId)
|
||||
setCompatibleDialogOpen(true)
|
||||
}
|
||||
|
||||
const openCloudSetup = (manifest: SpeechModelManifest): void => {
|
||||
if (manifest.provider === 'openai-compatible') {
|
||||
openCompatibleDialog(manifest.id)
|
||||
} else {
|
||||
openOpenAiDialog(manifest.id)
|
||||
}
|
||||
}
|
||||
|
||||
const handleCompatibleSaved = (): void => {
|
||||
if (pendingCloudModelId) {
|
||||
updateVoiceSettings({ sttModel: pendingCloudModelId })
|
||||
}
|
||||
setPendingCloudModelId(null)
|
||||
refreshModelStates()
|
||||
}
|
||||
|
||||
const removeCompatibleProvider = async (): Promise<void> => {
|
||||
try {
|
||||
await window.api.speech.clearOpenAiCompatibleProvider()
|
||||
if (selectedModel?.provider === 'openai-compatible') {
|
||||
updateVoiceSettings({ sttModel: '' })
|
||||
}
|
||||
refreshModelStates()
|
||||
} catch (err) {
|
||||
toast.error(
|
||||
readIpcErrorMessage(err) ??
|
||||
t(
|
||||
'components.settings.voice.openAiCompatible.updateFailed',
|
||||
'Could not update the provider'
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const saveOpenAiApiKey = async (): Promise<void> => {
|
||||
setOpenAiKeyPending(true)
|
||||
try {
|
||||
@@ -217,7 +265,7 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J
|
||||
catalog={catalog}
|
||||
modelStates={modelStates}
|
||||
onUpdateVoiceSettings={updateVoiceSettings}
|
||||
onOpenOpenAiDialog={openOpenAiDialog}
|
||||
onOpenCloudSetup={openCloudSetup}
|
||||
onRefreshModelStates={refreshModelStates}
|
||||
/>
|
||||
|
||||
@@ -270,6 +318,25 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J
|
||||
</>
|
||||
)}
|
||||
|
||||
{showCompatibleSettingsRow && (
|
||||
<>
|
||||
<Separator />
|
||||
<OpenAiTranscriptionSettingsRow
|
||||
variant="compatible"
|
||||
configured={compatibleConfigured}
|
||||
disabled={false}
|
||||
onConfigure={() => openCompatibleDialog(null)}
|
||||
onClear={() => void removeCompatibleProvider()}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
<OpenAiCompatibleProviderDialog
|
||||
open={compatibleDialogOpen}
|
||||
onOpenChange={setCompatibleDialogOpen}
|
||||
onSaved={handleCompatibleSaved}
|
||||
/>
|
||||
|
||||
<OpenAiTranscriptionKeyDialog
|
||||
open={openAiDialogOpen}
|
||||
configured={voiceSettings.openAiApiKeyConfigured}
|
||||
|
||||
@@ -106,7 +106,7 @@ function renderSection(args: {
|
||||
catalog={catalog}
|
||||
modelStates={modelStates}
|
||||
onUpdateVoiceSettings={vi.fn()}
|
||||
onOpenOpenAiDialog={vi.fn()}
|
||||
onOpenCloudSetup={vi.fn()}
|
||||
onRefreshModelStates={args.refreshModelStates ?? vi.fn()}
|
||||
/>
|
||||
)
|
||||
|
||||
@@ -29,7 +29,8 @@ type VoiceSpeechModelSectionProps = {
|
||||
catalog: SpeechModelManifest[]
|
||||
modelStates: SpeechModelState[]
|
||||
onUpdateVoiceSettings: (updates: Partial<VoiceSettings>) => void
|
||||
onOpenOpenAiDialog: (modelId: string) => void
|
||||
/** A cloud model that isn't set up yet: open its provider's setup. */
|
||||
onOpenCloudSetup: (manifest: SpeechModelManifest) => void
|
||||
onRefreshModelStates: () => void
|
||||
}
|
||||
|
||||
@@ -38,7 +39,7 @@ export function VoiceSpeechModelSection({
|
||||
catalog,
|
||||
modelStates,
|
||||
onUpdateVoiceSettings,
|
||||
onOpenOpenAiDialog,
|
||||
onOpenCloudSetup,
|
||||
onRefreshModelStates
|
||||
}: VoiceSpeechModelSectionProps): React.JSX.Element {
|
||||
const t = useTranslate()
|
||||
@@ -83,7 +84,7 @@ export function VoiceSpeechModelSection({
|
||||
{catalog.map((manifest) => {
|
||||
const mState = getModelState(manifest.id)
|
||||
const isReady = mState?.status === 'ready'
|
||||
const isCloud = manifest.provider === 'openai'
|
||||
const isCloud = manifest.provider !== 'local'
|
||||
const isRetired = !isOfferedSpeechModel(manifest)
|
||||
// Retired local models stay listed only while downloaded, so their files can be deleted.
|
||||
if (isRetired && (isCloud || !isReady)) {
|
||||
@@ -105,7 +106,7 @@ export function VoiceSpeechModelSection({
|
||||
} else if (isReady) {
|
||||
onUpdateVoiceSettings({ sttModel: manifest.id })
|
||||
} else if (isCloud) {
|
||||
onOpenOpenAiDialog(manifest.id)
|
||||
onOpenCloudSetup(manifest)
|
||||
} else if (!isDownloading) {
|
||||
// Keep the menu open so download progress remains visible.
|
||||
event.preventDefault()
|
||||
|
||||
@@ -52,6 +52,11 @@ const DESCRIPTIONS: Record<string, () => string> = {
|
||||
'settings.voice.modelDescriptions.senseVoice',
|
||||
'Chinese, English, Japanese, Korean, and Cantonese with automatic language detection.'
|
||||
),
|
||||
'openai-compatible': () =>
|
||||
translate(
|
||||
'settings.voice.modelDescriptions.openAiCompatible',
|
||||
'Any service with an OpenAI-style transcription API, such as Groq, Mistral or a local server.'
|
||||
),
|
||||
'openai-gpt-transcribe': () =>
|
||||
translate(
|
||||
'settings.voice.modelDescriptions.gptTranscribe',
|
||||
|
||||
@@ -89,7 +89,6 @@ export function installBrowserLinkMaterializationProbe() {
|
||||
dictationHoldToRecord: false,
|
||||
removeFillerWords: true,
|
||||
terminalConfirmBeforeInsert: true,
|
||||
userModels: [],
|
||||
openAiApiKeyConfigured: false,
|
||||
microphoneDeviceId: null,
|
||||
microphoneDeviceLabel: null,
|
||||
|
||||
@@ -1156,7 +1156,8 @@
|
||||
"parakeetJa": "Japanese only. Trained on 35k+ hours of natural speech. Punctuation included.",
|
||||
"whisperTiny": "90+ languages. Lower accuracy than Parakeet but broadest language coverage.",
|
||||
"senseVoice": "Chinese, English, Japanese, Korean, and Cantonese with automatic language detection.",
|
||||
"gptTranscribe": "Cloud transcription with high accuracy. Requires an OpenAI API key."
|
||||
"gptTranscribe": "Cloud transcription with high accuracy. Requires an OpenAI API key.",
|
||||
"openAiCompatible": "Any service with an OpenAI-style transcription API, such as Groq, Mistral or a local server."
|
||||
}
|
||||
},
|
||||
"diagnostics": {
|
||||
@@ -21380,6 +21381,24 @@
|
||||
"fillerWords": {
|
||||
"title": "Remove Filler Words",
|
||||
"description": "Drops sounds like “um” and “uh” from dictated text, in the dictation language."
|
||||
},
|
||||
"openAiCompatible": {
|
||||
"updateFailed": "Could not update the provider",
|
||||
"title": "OpenAI-compatible provider",
|
||||
"description": "Audio goes to this service only while the OpenAI-compatible model is selected. It works with Groq, Mistral, OpenAI and local servers.",
|
||||
"baseUrl": "Base URL",
|
||||
"baseUrlPlaceholder": "https://api.groq.com/openai/v1",
|
||||
"model": "Model",
|
||||
"modelPlaceholder": "whisper-large-v3-turbo",
|
||||
"apiKey": "API Key",
|
||||
"keySaved": "Saved key kept",
|
||||
"keyOptional": "Optional for local servers",
|
||||
"save": "Save",
|
||||
"rowConfigured": "Base URL and model saved for the OpenAI-compatible model.",
|
||||
"rowMissing": "Add a base URL and model before selecting the OpenAI-compatible model.",
|
||||
"edit": "Edit",
|
||||
"removeLabel": "Remove OpenAI-compatible provider",
|
||||
"setUp": "Set up"
|
||||
}
|
||||
},
|
||||
"repositoryHooks": {
|
||||
|
||||
@@ -105,7 +105,6 @@ export function getDefaultVoiceSettings(): VoiceSettings {
|
||||
dictationHoldToRecord: true,
|
||||
removeFillerWords: true,
|
||||
terminalConfirmBeforeInsert: false,
|
||||
userModels: [],
|
||||
openAiApiKeyConfigured: false,
|
||||
microphoneDeviceId: null,
|
||||
microphoneDeviceLabel: null,
|
||||
|
||||
@@ -5,7 +5,7 @@ export type SpeechModelType =
|
||||
| 'senseVoice'
|
||||
| 'nemo-ctc'
|
||||
| 'openai'
|
||||
export type SpeechModelProvider = 'local' | 'openai'
|
||||
export type SpeechModelProvider = 'local' | 'openai' | 'openai-compatible'
|
||||
|
||||
export type ModelingUnit = 'bpe' | 'cjkchar' | 'cjkchar+bpe'
|
||||
|
||||
@@ -36,6 +36,20 @@ export type SpeechModelManifest = {
|
||||
dictationLanguages?: readonly string[]
|
||||
}
|
||||
|
||||
export type OpenAiCompatibleSpeechProvider = {
|
||||
baseUrl: string
|
||||
model: string
|
||||
/** Empty for servers that take no key. */
|
||||
apiKey: string
|
||||
}
|
||||
|
||||
export type OpenAiCompatibleSpeechProviderSummary = Omit<
|
||||
OpenAiCompatibleSpeechProvider,
|
||||
'apiKey'
|
||||
> & {
|
||||
hasApiKey: boolean
|
||||
}
|
||||
|
||||
export type SpeechModelStatus = 'not-downloaded' | 'downloading' | 'extracting' | 'ready' | 'error'
|
||||
|
||||
export type SpeechModelState = {
|
||||
@@ -66,13 +80,6 @@ export type SpeechErrorEvent = {
|
||||
|
||||
export type DictationState = 'idle' | 'starting' | 'listening' | 'stopping' | 'error'
|
||||
|
||||
export type UserModelConfig = {
|
||||
id: string
|
||||
type: SpeechModelType
|
||||
dir: string
|
||||
sampleRate?: number
|
||||
}
|
||||
|
||||
// See dictation-shortcut-mode.ts for Auto.
|
||||
export type DictationMode = 'toggle' | 'hold'
|
||||
|
||||
@@ -89,7 +96,6 @@ export type VoiceSettings = {
|
||||
/** Drops "um", "uh" and their equivalents in the dictation language before text is inserted. */
|
||||
removeFillerWords: boolean
|
||||
terminalConfirmBeforeInsert: boolean
|
||||
userModels: UserModelConfig[]
|
||||
openAiApiKeyConfigured: boolean
|
||||
/** null = system default input device */
|
||||
microphoneDeviceId: string | null
|
||||
|
||||
Reference in New Issue
Block a user