diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 42f92f7445e..6bdba861bbe 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -17,7 +17,15 @@ import { getOpenAiSpeechApiKeyProtection, hasOpenAiSpeechApiKey } from '../speech/openai-api-key-store' -import { saveVerifiedOpenAiSpeechApiKey } from '../speech/openai-api-key-verification' +import { + saveVerifiedOpenAiCompatibleSpeechProvider, + saveVerifiedOpenAiSpeechApiKey +} from '../speech/openai-api-key-verification' +import { + clearOpenAiCompatibleSpeechProvider, + summarizeOpenAiCompatibleSpeechProvider +} from '../speech/openai-compatible-speech-provider-store' +import type { OpenAiCompatibleSpeechProvider } from '../../shared/speech-types' import type { Store } from '../persistence/loading-store/store' export function registerSpeechHandlers(store: Store): void { @@ -43,6 +51,25 @@ export function registerSpeechHandlers(store: Store): void { return { configured: false, protection: null } }) + ipcMain.handle('speech:getOpenAiCompatibleProvider', () => { + try { + return summarizeOpenAiCompatibleSpeechProvider() + } catch { + return null + } + }) + + ipcMain.handle( + 'speech:saveOpenAiCompatibleProvider', + async (_event, input: OpenAiCompatibleSpeechProvider) => { + await saveVerifiedOpenAiCompatibleSpeechProvider(input) + } + ) + + ipcMain.handle('speech:clearOpenAiCompatibleProvider', () => { + clearOpenAiCompatibleSpeechProvider() + }) + ipcMain.handle('speech:downloadModel', async (event, modelId: string) => { const manager = getSpeechModelManager(store) const window = BrowserWindow.fromWebContents(event.sender) diff --git a/src/main/orcad/orcad-plaintext-credential-migration.ts b/src/main/orcad/orcad-plaintext-credential-migration.ts index f93b8cc0441..f2bc98d95d7 100644 --- a/src/main/orcad/orcad-plaintext-credential-migration.ts +++ b/src/main/orcad/orcad-plaintext-credential-migration.ts @@ -18,6 +18,7 @@ import { API_KEY_ENVELOPE_PREFIX, getZcodePlanApiKeyPath } from '../zcode/zcode- import { COOKIE_ENVELOPE_PREFIX, getMiniMaxCookiePath } from '../minimax/minimax-cookie-store' import { miniMaxApiKeyEnvelopeFile } from '../minimax/minimax-api-key-store' import { openCodeGoApiKeyEnvelopeFile } from '../opencode/opencode-go-api-key-store' +import { openAiCompatibleSpeechProviderEnvelopeFile } from '../speech/openai-compatible-speech-provider-store' type PlaintextCredentialTarget = { path: string @@ -53,7 +54,8 @@ function listPlaintextCredentialTargets(): PlaintextCredentialTarget[] { { path: getMiniMaxCookiePath(), envelopePrefix: COOKIE_ENVELOPE_PREFIX, raw: true }, { path: getZcodePlanApiKeyPath(), envelopePrefix: API_KEY_ENVELOPE_PREFIX }, miniMaxApiKeyEnvelopeFile(), - openCodeGoApiKeyEnvelopeFile() + openCodeGoApiKeyEnvelopeFile(), + openAiCompatibleSpeechProviderEnvelopeFile() ] } diff --git a/src/main/orcad/orcad-speech-services.ts b/src/main/orcad/orcad-speech-services.ts index c6e9b47ee60..d9c0c661c90 100644 --- a/src/main/orcad/orcad-speech-services.ts +++ b/src/main/orcad/orcad-speech-services.ts @@ -1,6 +1,6 @@ import { ModelManager } from '../speech/model-manager' import { openNodeSpeechDownloadRequest } from '../speech/node-speech-download-request' -import { canReadOpenAiSpeechApiKey } from '../speech/openai-api-key-store' +import { canReadCloudSpeechCredentials } from '../speech/cloud-transcription-target' import type { SpeechServiceFactories } from '../speech/speech-runtime-service' import { SttService } from '../speech/stt-service' import { createSttGpuPolicy } from '../speech/stt-gpu-failure-marker' @@ -11,7 +11,7 @@ import { spawnSttForkedProcess } from '../speech/stt-worker-forked-process' export const orcadSpeechServiceFactories: SpeechServiceFactories = { createModelManager: (customModelsDir) => // Headless key reads cannot open keychain prompts, so readiness may verify decryption. - new ModelManager(customModelsDir, openNodeSpeechDownloadRequest, canReadOpenAiSpeechApiKey), + new ModelManager(customModelsDir, openNodeSpeechDownloadRequest, canReadCloudSpeechCredentials), createSttService: (models, gpuEnabled) => { const gpu = createSttGpuPolicy(models.getModelsDir(), gpuEnabled) return new SttService(models, () => spawnSttWorkerWithCpuFallback(spawnSttForkedProcess, gpu)) diff --git a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts index fd50666cc38..4da3f086637 100644 --- a/src/main/runtime/speech/runtime-mobile-speech-catalog.ts +++ b/src/main/runtime/speech/runtime-mobile-speech-catalog.ts @@ -43,14 +43,18 @@ export class RuntimeMobileSpeechCatalog { const runnable = SPEECH_MODEL_CATALOG.filter( (manifest) => isOfferedSpeechModel(manifest) && - (!isLocalSpeechModel(manifest) || hasLocalSpeechRecognizer(manifest)) + (!isLocalSpeechModel(manifest) || hasLocalSpeechRecognizer(manifest)) && + // Phones offer an unready cloud row as "add an OpenAI key", which can't set this one up. + (manifest.provider !== 'openai-compatible' || + stateById.get(manifest.id)?.status === 'ready') ) const models: RuntimeSpeechModelSummary[] = runnable.map((manifest) => { const state = stateById.get(manifest.id) return { id: manifest.id, label: manifest.label, - provider: manifest.provider === 'openai' ? 'openai' : 'local', + // Older phones know only these two; both cloud providers transcribe on this host. + provider: isLocalSpeechModel(manifest) ? 'local' : 'openai', sizeBytes: manifest.sizeBytes ?? null, recommended: manifest.recommended === true, status: state?.status ?? 'not-downloaded', diff --git a/src/main/speech/cloud-transcription-target.ts b/src/main/speech/cloud-transcription-target.ts new file mode 100644 index 00000000000..929cfe6377e --- /dev/null +++ b/src/main/speech/cloud-transcription-target.ts @@ -0,0 +1,59 @@ +import type { SpeechModelProvider } from '../../shared/speech-types' +import { getCatalogModel, replaceRetiredSpeechModelId } from './model-catalog' +import { + canReadOpenAiSpeechApiKey, + hasOpenAiSpeechApiKey, + readOpenAiSpeechApiKey +} from './openai-api-key-store' +import { + canReadOpenAiCompatibleSpeechProvider, + hasOpenAiCompatibleSpeechProvider, + openAiCompatibleTranscriptionUrl, + readOpenAiCompatibleSpeechProvider +} from './openai-compatible-speech-provider-store' + +export const OPENAI_TRANSCRIPTION_URL = 'https://api.openai.com/v1/audio/transcriptions' + +const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record = { + 'openai-gpt-transcribe': 'gpt-transcribe' +} + +export type CloudTranscriptionTarget = { + url: string + model: string + /** Empty when the server takes no key. */ + apiKey: string + /** Names the service in errors. */ + label: string +} + +export function hasCloudSpeechCredentials(provider: SpeechModelProvider): boolean { + return provider === 'openai-compatible' + ? hasOpenAiCompatibleSpeechProvider() + : hasOpenAiSpeechApiKey() +} + +/** Headless hosts can't raise keychain prompts, so their readiness verifies decryption. */ +export function canReadCloudSpeechCredentials(provider: SpeechModelProvider): boolean { + return provider === 'openai-compatible' + ? canReadOpenAiCompatibleSpeechProvider() + : canReadOpenAiSpeechApiKey() +} + +/** Reads the key (and the compatible provider's URL) only when audio is ready to upload. */ +export function resolveCloudTranscriptionTarget(modelId: string): CloudTranscriptionTarget { + if (getCatalogModel(modelId)?.provider === 'openai-compatible') { + const provider = readOpenAiCompatibleSpeechProvider() + if (!provider) { + throw new Error('The OpenAI-compatible speech provider is not set up') + } + const url = openAiCompatibleTranscriptionUrl(provider.baseUrl) + return { url, model: provider.model, apiKey: provider.apiKey, label: new URL(url).host } + } + // Settings load already moves retired ids; this catches a caller still holding one. + const model = OPENAI_TRANSCRIPTION_MODEL_BY_ID[replaceRetiredSpeechModelId(modelId)] + if (!model) { + throw new Error(`Unknown OpenAI transcription model: ${modelId}`) + } + return { url: OPENAI_TRANSCRIPTION_URL, model, apiKey: readOpenAiSpeechApiKey(), label: 'OpenAI' } +} diff --git a/src/main/speech/model-catalog.ts b/src/main/speech/model-catalog.ts index 0d6f595db05..719230346cb 100644 --- a/src/main/speech/model-catalog.ts +++ b/src/main/speech/model-catalog.ts @@ -142,6 +142,18 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ sampleRate: 16000, streaming: false }, + { + id: 'openai-compatible', + label: 'OpenAI-compatible', + description: + 'Any service with an OpenAI-style transcription API, such as Groq, Mistral or a local server.', + type: 'openai', + provider: 'openai-compatible', + language: 'multilingual', + dictationLanguages: DICTATION_LANGUAGES, + sampleRate: 16000, + streaming: false + }, { id: 'openai-gpt-4o-mini-transcribe', label: 'GPT-4o mini Transcribe', diff --git a/src/main/speech/model-manager.ts b/src/main/speech/model-manager.ts index 00e3b0de910..f3d8d896588 100644 --- a/src/main/speech/model-manager.ts +++ b/src/main/speech/model-manager.ts @@ -7,7 +7,7 @@ import type { SpeechModelStatus } from '../../shared/speech-types' import { SPEECH_MODEL_CATALOG, getCatalogModel, isLocalSpeechModel } from './model-catalog' -import { hasOpenAiSpeechApiKey } from './openai-api-key-store' +import { hasCloudSpeechCredentials } from './cloud-transcription-target' import { getSpeechModelCacheDirCandidates, migrateSpeechModelCacheIfNeeded, @@ -38,7 +38,7 @@ export class ModelManager extends SpeechModelDownloadTransport { constructor( customModelsDir: string | undefined, openRequest: OpenSpeechDownloadRequest, - private readonly hasOpenAiKey: () => boolean = hasOpenAiSpeechApiKey + private readonly hasCloudCredentials = hasCloudSpeechCredentials ) { super(openRequest) const requestedModelsDir = @@ -103,8 +103,9 @@ export class ModelManager extends SpeechModelDownloadTransport { return { id: modelId, status: 'error', error: 'Unknown model' } } - if (manifest.provider === 'openai') { - return { id: modelId, status: this.hasOpenAiKey() ? 'ready' : 'not-downloaded' } + if (!isLocalSpeechModel(manifest)) { + const ready = this.hasCloudCredentials(manifest.provider) + return { id: modelId, status: ready ? 'ready' : 'not-downloaded' } } const modelDir = this.getModelDir(modelId) diff --git a/src/main/speech/openai-api-key-verification.ts b/src/main/speech/openai-api-key-verification.ts index d7fe69a1710..6249026b8ba 100644 --- a/src/main/speech/openai-api-key-verification.ts +++ b/src/main/speech/openai-api-key-verification.ts @@ -1,4 +1,11 @@ import { saveOpenAiSpeechApiKey } from './openai-api-key-store' +import { OPENAI_TRANSCRIPTION_URL } from './cloud-transcription-target' +import { + openAiCompatibleTranscriptionUrl, + resolveOpenAiCompatibleSpeechProviderInput, + saveOpenAiCompatibleSpeechProvider +} from './openai-compatible-speech-provider-store' +import type { OpenAiCompatibleSpeechProvider } from '../../shared/speech-types' import { postOpenAiTranscription, readOpenAiErrorMessage, @@ -11,16 +18,27 @@ const KEY_CHECK_TIMEOUT_MS = 10_000 export async function saveVerifiedOpenAiSpeechApiKey(apiKey: string): Promise { const trimmed = apiKey.trim() if (trimmed) { - await rejectUnauthorizedOpenAiKey(trimmed) + await rejectUnauthorizedKey(OPENAI_TRANSCRIPTION_URL, trimmed, 'OpenAI') } saveOpenAiSpeechApiKey(apiKey) } -async function rejectUnauthorizedOpenAiKey(apiKey: string): Promise { +/** Same check against the provider's own endpoint, which also catches a wrong base URL. */ +export async function saveVerifiedOpenAiCompatibleSpeechProvider( + input: OpenAiCompatibleSpeechProvider +): Promise { + const provider = resolveOpenAiCompatibleSpeechProviderInput(input) + const url = openAiCompatibleTranscriptionUrl(provider.baseUrl) + await rejectUnauthorizedKey(url, provider.apiKey, new URL(url).host) + saveOpenAiCompatibleSpeechProvider(provider) +} + +async function rejectUnauthorizedKey(url: string, apiKey: string, label: string): Promise { let response: Response try { - // An empty form is refused before any transcription, but only after OpenAI checks the key. + // An empty form is refused before any transcription, but only after the server checks the key. response = await postOpenAiTranscription( + url, apiKey, new FormData(), AbortSignal.timeout(KEY_CHECK_TIMEOUT_MS) @@ -28,10 +46,15 @@ async function rejectUnauthorizedOpenAiKey(apiKey: string): Promise { } catch { return } + if (response.status === 404 || response.status === 405) { + void response.body?.cancel().catch(() => undefined) + throw new Error(`${label} has no transcription endpoint at ${url}. Check the base URL.`) + } if (response.status !== 401) { void response.body?.cancel().catch(() => undefined) return } const data: OpenAiTranscriptionResponse = await response.json().catch(() => ({})) - throw new Error(readOpenAiErrorMessage(data) ?? 'OpenAI rejected this API key.') + const fallback = apiKey ? `${label} rejected this API key.` : `${label} needs an API key.` + throw new Error(readOpenAiErrorMessage(data, apiKey) ?? fallback) } diff --git a/src/main/speech/openai-compatible-speech-provider-store.test.ts b/src/main/speech/openai-compatible-speech-provider-store.test.ts new file mode 100644 index 00000000000..8821eb525b3 --- /dev/null +++ b/src/main/speech/openai-compatible-speech-provider-store.test.ts @@ -0,0 +1,66 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +async function loadStoreModule() { + vi.resetModules() + const { setSecretStore } = await import('../../shared/secret-store') + setSecretStore({ + decryptString: (value: Buffer) => value.toString('utf8'), + encryptString: (value: string) => Buffer.from(value), + isEncryptionAvailable: () => true, + describeProtectionGap: () => null + }) + return import('./openai-compatible-speech-provider-store') +} + +beforeEach(() => { + const home = mkdtempSync(join(tmpdir(), 'orca-compatible-speech-')) + vi.stubEnv('HOME', home) + vi.stubEnv('USERPROFILE', home) +}) + +describe('openAiCompatibleTranscriptionUrl', () => { + it('appends the transcriptions path to a base URL and keeps a full one', async () => { + const { openAiCompatibleTranscriptionUrl } = await loadStoreModule() + expect(openAiCompatibleTranscriptionUrl(' https://api.groq.com/openai/v1/ ')).toBe( + 'https://api.groq.com/openai/v1/audio/transcriptions' + ) + expect(openAiCompatibleTranscriptionUrl('https://api.mistral.ai/v1/audio/transcriptions')).toBe( + 'https://api.mistral.ai/v1/audio/transcriptions' + ) + expect(openAiCompatibleTranscriptionUrl('http://127.0.0.1:8000/v1')).toBe( + 'http://127.0.0.1:8000/v1/audio/transcriptions' + ) + expect(() => openAiCompatibleTranscriptionUrl('file:///etc/passwd')).toThrow() + expect(() => openAiCompatibleTranscriptionUrl('api.groq.com')).toThrow() + expect(() => openAiCompatibleTranscriptionUrl('https://user:pw@host.example/v1')).toThrow() + }) +}) + +describe('resolveOpenAiCompatibleSpeechProviderInput', () => { + it('keeps a saved key for a blank field only while the base URL is unchanged', async () => { + const store = await loadStoreModule() + store.saveOpenAiCompatibleSpeechProvider({ + baseUrl: 'https://api.groq.com/openai/v1', + model: 'whisper-large-v3-turbo', + apiKey: 'gsk_saved' + }) + + expect( + store.resolveOpenAiCompatibleSpeechProviderInput({ + baseUrl: 'https://api.groq.com/openai/v1/', + model: 'whisper-large-v3', + apiKey: '' + }).apiKey + ).toBe('gsk_saved') + expect( + store.resolveOpenAiCompatibleSpeechProviderInput({ + baseUrl: 'https://evil.example/v1', + model: 'whisper-large-v3', + apiKey: '' + }).apiKey + ).toBe('') + }) +}) diff --git a/src/main/speech/openai-compatible-speech-provider-store.ts b/src/main/speech/openai-compatible-speech-provider-store.ts new file mode 100644 index 00000000000..b634d9b26ea --- /dev/null +++ b/src/main/speech/openai-compatible-speech-provider-store.ts @@ -0,0 +1,98 @@ +import { z } from 'zod' +import { createEncryptedApiKeyFileStore } from '../credentials/encrypted-api-key-file-store' +import { savedCredentialUnreadable } from '../credentials/saved-credential-readability' +import type { + OpenAiCompatibleSpeechProvider, + OpenAiCompatibleSpeechProviderSummary +} from '../../shared/speech-types' + +// URL, model and key are sealed together so a key can never be sent to a host it wasn't saved for. +const store = createEncryptedApiKeyFileStore({ + fileName: 'openai-compatible-speech-provider.enc', + envelopePrefix: 'orca-openai-compatible-speech:v1:', + providerLabel: 'OpenAI-compatible speech provider', + logScope: 'speech' +}) + +export const hasOpenAiCompatibleSpeechProvider = store.has +export const clearOpenAiCompatibleSpeechProvider = store.clear +export const openAiCompatibleSpeechProviderEnvelopeFile = store.envelopeFile + +const TRANSCRIPTION_PATH = '/audio/transcriptions' + +const providerSchema = z.object({ + baseUrl: z.string().catch(''), + model: z.string().catch(''), + apiKey: z.string().catch('') +}) satisfies z.ZodType + +/** Accepts a base like `https://api.groq.com/openai/v1` or the full transcriptions URL. */ +export function openAiCompatibleTranscriptionUrl(baseUrl: string): string { + let url: URL + try { + url = new URL(baseUrl.trim()) + } catch { + throw new Error('Enter a full base URL, such as https://api.groq.com/openai/v1') + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new Error('The base URL must start with https:// or http://') + } + // fetch refuses these, and its error would echo the password. + if (url.username || url.password) { + throw new Error('Put the key in the API Key field, not in the base URL') + } + const path = url.pathname.replace(/\/+$/, '') + url.pathname = path.endsWith(TRANSCRIPTION_PATH) ? path : `${path}${TRANSCRIPTION_PATH}` + return url.toString() +} + +export function readOpenAiCompatibleSpeechProvider(): OpenAiCompatibleSpeechProvider | null { + // The store caches what it read; another process (desktop vs orca serve) may have removed the file since. + const raw = store.has() ? store.read() : null + if (raw === null) { + return null + } + return providerSchema.parse(JSON.parse(raw)) +} + +/** Headless hosts may not open a desktop-sealed file; checks the disk quietly, since readiness is polled. */ +export function canReadOpenAiCompatibleSpeechProvider(): boolean { + return store.has() && savedCredentialUnreadable(store.envelopeFile()) === undefined +} + +export function summarizeOpenAiCompatibleSpeechProvider(): OpenAiCompatibleSpeechProviderSummary | null { + const provider = readOpenAiCompatibleSpeechProvider() + return provider + ? { baseUrl: provider.baseUrl, model: provider.model, hasApiKey: provider.apiKey !== '' } + : null +} + +/** A blank key keeps the saved one only while the base URL is unchanged. */ +export function resolveOpenAiCompatibleSpeechProviderInput( + input: OpenAiCompatibleSpeechProvider +): OpenAiCompatibleSpeechProvider { + const baseUrl = input.baseUrl.trim() + const model = input.model.trim() + openAiCompatibleTranscriptionUrl(baseUrl) + if (!model) { + throw new Error('Enter the model name the provider expects') + } + return { baseUrl, model, apiKey: input.apiKey.trim() || savedKeyFor(baseUrl) } +} + +function savedKeyFor(baseUrl: string): string { + try { + const saved = readOpenAiCompatibleSpeechProvider() + const sameEndpoint = + saved !== null && + openAiCompatibleTranscriptionUrl(saved.baseUrl) === openAiCompatibleTranscriptionUrl(baseUrl) + return sameEndpoint ? saved.apiKey : '' + } catch { + // An unreadable or invalid saved provider has no key worth keeping. + return '' + } +} + +export function saveOpenAiCompatibleSpeechProvider(provider: OpenAiCompatibleSpeechProvider): void { + store.save(JSON.stringify(provider)) +} diff --git a/src/main/speech/openai-transcription-client.test.ts b/src/main/speech/openai-transcription-client.test.ts index abec63f2854..c74b8afaf91 100644 --- a/src/main/speech/openai-transcription-client.test.ts +++ b/src/main/speech/openai-transcription-client.test.ts @@ -47,7 +47,15 @@ describe('OpenAiTranscriptionSession', () => { uploads.push(Buffer.from(await file.arrayBuffer())) return new Response(JSON.stringify({ text: ' complete dictation ' })) }) - const session = new OpenAiTranscriptionSession('openai-gpt-transcribe', () => 'test-key', 'de') + const session = new OpenAiTranscriptionSession( + () => ({ + url: 'https://api.openai.com/v1/audio/transcriptions', + model: 'gpt-transcribe', + apiKey: 'test-key', + label: 'OpenAI' + }), + 'de' + ) const input = new Float32Array([ -2, -1, @@ -76,4 +84,31 @@ describe('OpenAiTranscriptionSession', () => { -32768, -32768, -16384, 0, 0, 16384, 32767, 32767, 0, 32767, -32768, 0, 24575 ]) }) + + it('sends a keyless server the singular language field and no Authorization header', async () => { + let sent: RequestInit | undefined + vi.stubGlobal('fetch', async (_url: string, init: RequestInit) => { + sent = init + return new Response(JSON.stringify({ text: 'hola' })) + }) + const session = new OpenAiTranscriptionSession( + () => ({ + url: 'http://127.0.0.1:8000/v1/audio/transcriptions', + model: 'whisper-large-v3-turbo', + apiKey: '', + label: '127.0.0.1:8000' + }), + 'es' + ) + session.feedAudio(new Float32Array(160), 16000) + + expect(await session.finish()).toBe('hola') + const form = sent?.body + if (!(form instanceof FormData)) { + throw new Error('Expected multipart body') + } + expect(form.get('language')).toBe('es') + expect(form.has('languages[]')).toBe(false) + expect(sent?.headers).toEqual({}) + }) }) diff --git a/src/main/speech/openai-transcription-client.ts b/src/main/speech/openai-transcription-client.ts index 452685c9b29..3f18309beb6 100644 --- a/src/main/speech/openai-transcription-client.ts +++ b/src/main/speech/openai-transcription-client.ts @@ -1,17 +1,12 @@ import { getMainHttpClient } from '../network/http-client' import { resampleToRate } from './stt-audio-resample' -import { replaceRetiredSpeechModelId } from './model-catalog' +import type { CloudTranscriptionTarget } from './cloud-transcription-target' import { CLOUD_TRANSCRIPTION_SAMPLE_RATE, MAX_CLOUD_TRANSCRIPTION_SECONDS, cloudTranscriptionTimeoutMs } from '../../shared/dictation-finish-deadlines' -export const OPENAI_TRANSCRIPTION_MODEL_BY_ID: Record = { - 'openai-gpt-transcribe': 'gpt-transcribe' -} - -export const OPENAI_TRANSCRIPTION_URL = 'https://api.openai.com/v1/audio/transcriptions' const MAX_CLOUD_AUDIO_SAMPLES = MAX_CLOUD_TRANSCRIPTION_SECONDS * CLOUD_TRANSCRIPTION_SAMPLE_RATE export type OpenAiTranscriptionResponse = { @@ -21,17 +16,18 @@ export type OpenAiTranscriptionResponse = { } } -export function sanitizeOpenAiTranscriptionErrorMessage(message: string): string { +export function sanitizeOpenAiTranscriptionErrorMessage(message: string, apiKey = ''): string { if (/incorrect api key provided:/i.test(message)) { return 'Incorrect OpenAI API key provided.' } - const sanitized = message + // Other providers' keys have other shapes, so also redact the exact key sent (unless it's a placeholder like "x"). + const sanitized = (apiKey.length >= 8 ? message.split(apiKey).join('[redacted]') : message) .replace(/\bsk-[A-Za-z0-9_-]+/g, '[redacted]') .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [redacted]') .trim() - return sanitized || 'OpenAI transcription request failed' + return sanitized || 'Transcription request failed' } function writePcm16Samples(view: DataView, samples: Float32Array, offset: number): void { @@ -72,33 +68,34 @@ function encodePcm16Wav(chunks: Float32Array[], sampleRate: number): Buffer { - return getMainHttpClient().fetch(OPENAI_TRANSCRIPTION_URL, { + return getMainHttpClient().fetch(url, { method: 'POST', - // Bearer auth only: keep OpenAI's cookies out of the shared desktop session. + // Bearer auth only: keep the provider's cookies out of the shared desktop session. credentials: 'omit', signal, - headers: { Authorization: `Bearer ${apiKey}` }, + // Local servers that take no key get no Authorization header. + headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {}, body: form }) } -export function readOpenAiErrorMessage(data: OpenAiTranscriptionResponse): string | undefined { +export function readOpenAiErrorMessage( + data: OpenAiTranscriptionResponse, + apiKey = '' +): string | undefined { return typeof data.error?.message === 'string' - ? sanitizeOpenAiTranscriptionErrorMessage(data.error.message) + ? sanitizeOpenAiTranscriptionErrorMessage(data.error.message, apiKey) : undefined } -function parseOpenAiTranscriptionResponse(data: OpenAiTranscriptionResponse): string { - if (typeof data.text === 'string') { - return data.text.trim() - } - throw new Error( - readOpenAiErrorMessage(data) ?? 'OpenAI transcription response did not include text' - ) +// gpt-transcribe takes a list of expected languages in place of the singular field others use. +function languageField(model: string): string { + return /^gpt-(live-)?transcribe/.test(model) ? 'languages[]' : 'language' } export class OpenAiTranscriptionSession { @@ -107,8 +104,7 @@ export class OpenAiTranscriptionSession { private readonly abortController = new AbortController() constructor( - private readonly modelId: string, - private readonly readApiKey: () => string, + private readonly readTarget: () => CloudTranscriptionTarget, private readonly language?: string ) {} @@ -136,46 +132,46 @@ export class OpenAiTranscriptionSession { return '' } - // Settings load already moves retired ids; this catches a caller still holding one. - const apiModel = OPENAI_TRANSCRIPTION_MODEL_BY_ID[replaceRetiredSpeechModelId(this.modelId)] - if (!apiModel) { - throw new Error(`Unknown OpenAI transcription model: ${this.modelId}`) - } - + const target = this.readTarget() const chunks = this.chunks this.chunks = [] const wav = encodePcm16Wav(chunks, CLOUD_TRANSCRIPTION_SAMPLE_RATE) + // No response_format: JSON is the default everywhere, and not every provider accepts the field. const form = new FormData() - // No response_format: JSON is the default, and gpt-transcribe documents no other. - form.append('model', apiModel) + form.append('model', target.model) if (this.language) { - // gpt-transcribe takes a list of expected languages in place of the singular `language`. - form.append('languages[]', this.language) + form.append(languageField(target.model), this.language) } // A named WAV blob satisfies multipart uploads without temporary files. form.append('file', new Blob([wav], { type: 'audio/wav' }), 'dictation.wav') const timeout = AbortSignal.timeout(cloudTranscriptionTimeoutMs(this.audioSamples)) try { - return await this.upload(form, AbortSignal.any([this.abortController.signal, timeout])) + return await upload(target, form, AbortSignal.any([this.abortController.signal, timeout])) } catch (error) { if (timeout.aborted && !this.abortController.signal.aborted) { - throw new Error('OpenAI transcription timed out. Check your connection and try again.') + throw new Error( + `${target.label} transcription timed out. Check your connection and try again.` + ) } throw error } } +} - private async upload(form: FormData, signal: AbortSignal): Promise { - const response = await postOpenAiTranscription(this.readApiKey(), form, signal) - - const data = (await response.json().catch(() => ({}))) as OpenAiTranscriptionResponse - if (!response.ok) { - throw new Error( - `OpenAI transcription failed: ${readOpenAiErrorMessage(data) ?? response.statusText}` - ) - } - - return parseOpenAiTranscriptionResponse(data) +async function upload( + target: CloudTranscriptionTarget, + form: FormData, + signal: AbortSignal +): Promise { + const response = await postOpenAiTranscription(target.url, target.apiKey, form, signal) + const data = (await response.json().catch(() => ({}))) as OpenAiTranscriptionResponse + const error = readOpenAiErrorMessage(data, target.apiKey) + if (!response.ok) { + throw new Error(`${target.label} transcription failed: ${error ?? response.statusText}`) + } + if (typeof data.text === 'string') { + return data.text.trim() } + throw new Error(error ?? `${target.label} transcription response did not include text`) } diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index 39b4ade9f76..00862d9af3d 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -89,10 +89,7 @@ const { static instances: HoistedMockOpenAiTranscriptionSession[] = [] feedCalls: { samples: Float32Array; sampleRate: number }[] = [] - constructor( - readonly modelId: string, - readonly readApiKey: () => string - ) { + constructor(readonly readTarget: () => { model: string; apiKey: string }) { HoistedMockOpenAiTranscriptionSession.instances.push(this) } @@ -101,7 +98,8 @@ const { } finish(): Promise { - return Promise.resolve(`${this.modelId}:${this.readApiKey()}`) + const target = this.readTarget() + return Promise.resolve(`${target.model}:${target.apiKey}`) } } @@ -185,8 +183,11 @@ vi.mock('./model-catalog', () => ({ } })) -vi.mock('./openai-api-key-store', () => ({ - readOpenAiSpeechApiKey: readOpenAiSpeechApiKeyMock +vi.mock('./cloud-transcription-target', () => ({ + resolveCloudTranscriptionTarget: (modelId: string) => ({ + model: modelId, + apiKey: readOpenAiSpeechApiKeyMock() + }) })) vi.mock('./openai-transcription-client', () => ({ diff --git a/src/main/speech/stt-session-start.ts b/src/main/speech/stt-session-start.ts index b1271a1501f..bf5c1795f07 100644 --- a/src/main/speech/stt-session-start.ts +++ b/src/main/speech/stt-session-start.ts @@ -2,7 +2,7 @@ import { engineDictationLanguage } from '../../shared/dictation-language' import type { SpeechModelManifest } from '../../shared/speech-types' import { getCatalogModel } from './model-catalog' import { OpenAiTranscriptionSession } from './openai-transcription-client' -import { readOpenAiSpeechApiKey } from './openai-api-key-store' +import { resolveCloudTranscriptionTarget } from './cloud-transcription-target' import type { SttEventSink } from './stt-service' import { MAX_STT_SESSIONS, @@ -56,7 +56,7 @@ export async function startSttDictation( throw new Error(`Unknown model: ${modelId}`) } const config: SttRecognizerConfig | null = - manifest.provider === 'openai' + manifest.provider !== 'local' ? null : { modelId, hotwordsFilePath, language: engineDictationLanguage(manifest, language) } if (state.sessions.size >= MAX_STT_SESSIONS) { @@ -91,8 +91,7 @@ export async function startSttDictation( // A cloud dictation needs no recognizer; free one nobody else uses. await teardownIdleSttEngine(state) session.cloud = new OpenAiTranscriptionSession( - modelId, - readOpenAiSpeechApiKey, + () => resolveCloudTranscriptionTarget(modelId), engineDictationLanguage(manifest, language) ) } diff --git a/src/preload/api/speech-api.ts b/src/preload/api/speech-api.ts index 8acba71dcc2..dc5d63bb3ca 100644 --- a/src/preload/api/speech-api.ts +++ b/src/preload/api/speech-api.ts @@ -1,5 +1,7 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' import type { + OpenAiCompatibleSpeechProvider, + OpenAiCompatibleSpeechProviderSummary, SpeechErrorEvent, SpeechLifecycleEvent, SpeechModelManifest, @@ -16,6 +18,11 @@ export type SpeechApi = { }> saveOpenAiApiKey: (apiKey: string) => Promise<{ configured: boolean }> clearOpenAiApiKey: () => Promise<{ configured: boolean }> + /** Never returns the saved key. */ + getOpenAiCompatibleProvider: () => Promise + /** A blank key keeps the saved one while the base URL is unchanged. */ + saveOpenAiCompatibleProvider: (provider: OpenAiCompatibleSpeechProvider) => Promise + clearOpenAiCompatibleProvider: () => Promise downloadModel: (modelId: string) => Promise cancelDownload: (modelId: string) => Promise deleteModel: (modelId: string) => Promise diff --git a/src/preload/api/speech-bridge.ts b/src/preload/api/speech-bridge.ts index a0b38154039..4e751f13d75 100644 --- a/src/preload/api/speech-bridge.ts +++ b/src/preload/api/speech-bridge.ts @@ -1,6 +1,8 @@ import type { SecretAtRestProtection } from '../../shared/secret-at-rest-protection' import { ipcRenderer } from 'electron' import type { + OpenAiCompatibleSpeechProvider, + OpenAiCompatibleSpeechProviderSummary, SpeechAudioPortMessage, SpeechErrorEvent, SpeechLifecycleEvent, @@ -24,6 +26,12 @@ export const speechApi = { ipcRenderer.invoke('speech:saveOpenAiApiKey', apiKey), clearOpenAiApiKey: (): Promise<{ configured: boolean }> => ipcRenderer.invoke('speech:clearOpenAiApiKey'), + getOpenAiCompatibleProvider: (): Promise => + ipcRenderer.invoke('speech:getOpenAiCompatibleProvider'), + saveOpenAiCompatibleProvider: (provider: OpenAiCompatibleSpeechProvider): Promise => + ipcRenderer.invoke('speech:saveOpenAiCompatibleProvider', provider), + clearOpenAiCompatibleProvider: (): Promise => + ipcRenderer.invoke('speech:clearOpenAiCompatibleProvider'), downloadModel: (modelId: string): Promise => ipcRenderer.invoke('speech:downloadModel', modelId), cancelDownload: (modelId: string): Promise => diff --git a/src/renderer/src/components/dictation/DictationIndicator.tsx b/src/renderer/src/components/dictation/DictationIndicator.tsx index e28f3d62f16..a1fc6e9b63e 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.tsx @@ -81,7 +81,7 @@ function useIsCloudModel(): boolean { } void catalogRequest.then((catalog) => { if (current) { - setCloud(catalog.some((model) => model.id === modelId && model.provider === 'openai')) + setCloud(catalog.some((model) => model.id === modelId && model.provider !== 'local')) } }) return () => { diff --git a/src/renderer/src/components/settings/voice/OpenAiCompatibleProviderDialog.tsx b/src/renderer/src/components/settings/voice/OpenAiCompatibleProviderDialog.tsx new file mode 100644 index 00000000000..50a1e9ea31f --- /dev/null +++ b/src/renderer/src/components/settings/voice/OpenAiCompatibleProviderDialog.tsx @@ -0,0 +1,168 @@ +import { useEffect, useState } from 'react' +import { Loader2, Lock } from 'lucide-react' +import { toast } from 'sonner' +import { Button } from '../../ui/button' +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle +} from '../../ui/dialog' +import { Input } from '../../ui/input' +import { Label } from '../../ui/label' +import { useTranslate } from '@/i18n/use-translate' +import { readIpcErrorMessage } from '@/lib/ipc-error' +import type { OpenAiCompatibleSpeechProviderSummary } from '../../../../../shared/speech-types' + +type OpenAiCompatibleProviderDialogProps = { + open: boolean + onOpenChange: (open: boolean) => void + onSaved: () => void +} + +export function OpenAiCompatibleProviderDialog({ + open, + onOpenChange, + onSaved +}: OpenAiCompatibleProviderDialogProps): React.JSX.Element { + const t = useTranslate() + const [baseUrl, setBaseUrl] = useState('') + const [model, setModel] = useState('') + const [apiKey, setApiKey] = useState('') + const [saved, setSaved] = useState(null) + const [pending, setPending] = useState(false) + + useEffect(() => { + if (!open) { + return + } + let cancelled = false + setApiKey('') + void window.api.speech + .getOpenAiCompatibleProvider() + .then((summary) => { + if (cancelled) { + return + } + setBaseUrl(summary?.baseUrl ?? '') + setModel(summary?.model ?? '') + setSaved(summary) + }) + .catch(() => {}) + return () => { + cancelled = true + } + }, [open]) + + const canSave = !pending && baseUrl.trim() !== '' && model.trim() !== '' + const keepsSavedKey = saved?.hasApiKey === true && saved.baseUrl === baseUrl.trim() + + const save = async (event: React.FormEvent): Promise => { + event.preventDefault() + if (!canSave) { + return + } + setPending(true) + try { + await window.api.speech.saveOpenAiCompatibleProvider({ baseUrl, model, apiKey }) + onSaved() + onOpenChange(false) + } catch (err) { + toast.error( + readIpcErrorMessage(err) ?? + t( + 'components.settings.voice.openAiCompatible.updateFailed', + 'Could not update the provider' + ) + ) + } + setPending(false) + } + + return ( + + + + + {t('components.settings.voice.openAiCompatible.title', 'OpenAI-compatible provider')} + + + {t( + 'components.settings.voice.openAiCompatible.description', + 'Audio goes to this service only while the OpenAI-compatible model is selected. It works with Groq, Mistral, OpenAI and local servers.' + )} + + +
void save(event)}> +
+ + setBaseUrl(event.target.value)} + /> +
+
+ + setModel(event.target.value)} + /> +
+
+ + setApiKey(event.target.value)} + /> +
+

+ + {t( + 'auto.components.settings.OpenAiTranscriptionKeyDialog.d246b2bdb3', + 'Local runtime keys are stored in ~/.orca using Electron encrypted storage when available.' + )} +

+ + + +
+
+
+ ) +} diff --git a/src/renderer/src/components/settings/voice/OpenAiTranscriptionSettingsRow.tsx b/src/renderer/src/components/settings/voice/OpenAiTranscriptionSettingsRow.tsx index 4cb63b113db..12808fc06a6 100644 --- a/src/renderer/src/components/settings/voice/OpenAiTranscriptionSettingsRow.tsx +++ b/src/renderer/src/components/settings/voice/OpenAiTranscriptionSettingsRow.tsx @@ -4,6 +4,8 @@ import { Label } from '../../ui/label' import { useTranslate } from '@/i18n/use-translate' type OpenAiTranscriptionSettingsRowProps = { + /** 'compatible' is the base URL + model provider, set up in its own dialog. */ + variant?: 'openai' | 'compatible' configured: boolean disabled: boolean onConfigure: () => void @@ -11,23 +13,68 @@ type OpenAiTranscriptionSettingsRowProps = { } export function OpenAiTranscriptionSettingsRow({ + variant = 'openai', configured, disabled, onConfigure, onClear }: OpenAiTranscriptionSettingsRowProps): React.JSX.Element { const t = useTranslate() + const copy = + variant === 'compatible' + ? { + title: t( + 'components.settings.voice.openAiCompatible.title', + 'OpenAI-compatible provider' + ), + configured: t( + 'components.settings.voice.openAiCompatible.rowConfigured', + 'Base URL and model saved for the OpenAI-compatible model.' + ), + missing: t( + 'components.settings.voice.openAiCompatible.rowMissing', + 'Add a base URL and model before selecting the OpenAI-compatible model.' + ), + edit: t('components.settings.voice.openAiCompatible.edit', 'Edit'), + clear: t( + 'components.settings.voice.openAiCompatible.removeLabel', + 'Remove OpenAI-compatible provider' + ), + add: t('components.settings.voice.openAiCompatible.setUp', 'Set up') + } + : { + title: t( + 'auto.components.settings.OpenAiTranscriptionSettingsRow.27e0cb656d', + 'OpenAI Transcription' + ), + configured: t( + 'auto.components.settings.OpenAiTranscriptionSettingsRow.b59b9b2b51', + 'API key configured for cloud speech-to-text models.' + ), + missing: t( + 'auto.components.settings.OpenAiTranscriptionSettingsRow.893790e13b', + 'Add an OpenAI API key before selecting cloud speech-to-text models.' + ), + edit: t( + 'auto.components.settings.OpenAiTranscriptionSettingsRow.a622bc3b37', + 'Replace key' + ), + clear: t( + 'auto.components.settings.OpenAiTranscriptionSettingsRow.ae2df8f511', + 'Disconnect OpenAI API key' + ), + add: t( + 'auto.components.settings.OpenAiTranscriptionSettingsRow.85c589cd61', + 'Add API key' + ) + } + return (
- + {configured && ( @@ -36,28 +83,17 @@ export function OpenAiTranscriptionSettingsRow({ )}

- {configured - ? t( - 'auto.components.settings.OpenAiTranscriptionSettingsRow.b59b9b2b51', - 'API key configured for cloud speech-to-text models.' - ) - : t( - 'auto.components.settings.OpenAiTranscriptionSettingsRow.893790e13b', - 'Add an OpenAI API key before selecting cloud speech-to-text models.' - )} + {configured ? copy.configured : copy.missing}

{configured ? (
) : ( )}
diff --git a/src/renderer/src/components/settings/voice/VoiceModelUnloadSetting.tsx b/src/renderer/src/components/settings/voice/VoiceModelUnloadSetting.tsx index f0ed5c74840..ac283b525e9 100644 --- a/src/renderer/src/components/settings/voice/VoiceModelUnloadSetting.tsx +++ b/src/renderer/src/components/settings/voice/VoiceModelUnloadSetting.tsx @@ -45,7 +45,7 @@ export function VoiceModelUnloadSetting({ const t = useTranslate() const intlLocale = useIntlLocale() const labelId = useId() - const isCloudModel = selectedModel?.provider === 'openai' + const isCloudModel = selectedModel !== undefined && selectedModel.provider !== 'local' const current = unloadModelAfterMinutes(voiceSettings) return ( diff --git a/src/renderer/src/components/settings/voice/VoicePane.tsx b/src/renderer/src/components/settings/voice/VoicePane.tsx index b85a62c59cc..0044a334037 100644 --- a/src/renderer/src/components/settings/voice/VoicePane.tsx +++ b/src/renderer/src/components/settings/voice/VoicePane.tsx @@ -8,6 +8,7 @@ import { Separator } from '../../ui/separator' import { toast } from 'sonner' import { useAppStore } from '@/store/index' import { OpenAiTranscriptionKeyDialog } from './OpenAiTranscriptionKeyDialog' +import { OpenAiCompatibleProviderDialog } from './OpenAiCompatibleProviderDialog' import { OpenAiTranscriptionSettingsRow } from './OpenAiTranscriptionSettingsRow' import { handleVoiceDictationToggle } from './voice-dictation-toggle' import { VoiceDictationLanguageSetting } from './VoiceDictationLanguageSetting' @@ -43,6 +44,7 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J null ) const [pendingCloudModelId, setPendingCloudModelId] = useState(null) + const [compatibleDialogOpen, setCompatibleDialogOpen] = useState(false) const mountedRef = useRef(true) // Merge async whole-object writes into committed current settings; render snapshots could undo toggles changed during IPC and disable the microphone picker. const voiceSettingsRef = useRef(voiceSettings) @@ -146,12 +148,58 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J (settingsSearchQuery.trim() !== '' && matchesSettingsSearch(settingsSearchQuery, getOpenaiTranscriptionSearchEntry())) + const compatibleModel = catalog.find((m) => m.provider === 'openai-compatible') + const compatibleConfigured = + compatibleModel !== undefined && + modelStates.some((s) => s.id === compatibleModel.id && s.status === 'ready') + const showCompatibleSettingsRow = + compatibleConfigured || selectedModel?.provider === 'openai-compatible' + const openOpenAiDialog = (modelId: string | null = null): void => { setPendingCloudModelId(modelId) setOpenAiApiKeyDraft('') setOpenAiDialogOpen(true) } + const openCompatibleDialog = (modelId: string | null): void => { + setPendingCloudModelId(modelId) + setCompatibleDialogOpen(true) + } + + const openCloudSetup = (manifest: SpeechModelManifest): void => { + if (manifest.provider === 'openai-compatible') { + openCompatibleDialog(manifest.id) + } else { + openOpenAiDialog(manifest.id) + } + } + + const handleCompatibleSaved = (): void => { + if (pendingCloudModelId) { + updateVoiceSettings({ sttModel: pendingCloudModelId }) + } + setPendingCloudModelId(null) + refreshModelStates() + } + + const removeCompatibleProvider = async (): Promise => { + try { + await window.api.speech.clearOpenAiCompatibleProvider() + if (selectedModel?.provider === 'openai-compatible') { + updateVoiceSettings({ sttModel: '' }) + } + refreshModelStates() + } catch (err) { + toast.error( + readIpcErrorMessage(err) ?? + t( + 'components.settings.voice.openAiCompatible.updateFailed', + 'Could not update the provider' + ) + ) + } + } + const saveOpenAiApiKey = async (): Promise => { setOpenAiKeyPending(true) try { @@ -217,7 +265,7 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J catalog={catalog} modelStates={modelStates} onUpdateVoiceSettings={updateVoiceSettings} - onOpenOpenAiDialog={openOpenAiDialog} + onOpenCloudSetup={openCloudSetup} onRefreshModelStates={refreshModelStates} /> @@ -270,6 +318,25 @@ export function VoicePane({ settings, updateSettings }: VoicePaneProps): React.J )} + {showCompatibleSettingsRow && ( + <> + + openCompatibleDialog(null)} + onClear={() => void removeCompatibleProvider()} + /> + + )} + + + ) diff --git a/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.tsx b/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.tsx index ca565de2a9f..30424f99bf1 100644 --- a/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.tsx +++ b/src/renderer/src/components/settings/voice/VoiceSpeechModelSection.tsx @@ -29,7 +29,8 @@ type VoiceSpeechModelSectionProps = { catalog: SpeechModelManifest[] modelStates: SpeechModelState[] onUpdateVoiceSettings: (updates: Partial) => void - onOpenOpenAiDialog: (modelId: string) => void + /** A cloud model that isn't set up yet: open its provider's setup. */ + onOpenCloudSetup: (manifest: SpeechModelManifest) => void onRefreshModelStates: () => void } @@ -38,7 +39,7 @@ export function VoiceSpeechModelSection({ catalog, modelStates, onUpdateVoiceSettings, - onOpenOpenAiDialog, + onOpenCloudSetup, onRefreshModelStates }: VoiceSpeechModelSectionProps): React.JSX.Element { const t = useTranslate() @@ -83,7 +84,7 @@ export function VoiceSpeechModelSection({ {catalog.map((manifest) => { const mState = getModelState(manifest.id) const isReady = mState?.status === 'ready' - const isCloud = manifest.provider === 'openai' + const isCloud = manifest.provider !== 'local' const isRetired = !isOfferedSpeechModel(manifest) // Retired local models stay listed only while downloaded, so their files can be deleted. if (isRetired && (isCloud || !isReady)) { @@ -105,7 +106,7 @@ export function VoiceSpeechModelSection({ } else if (isReady) { onUpdateVoiceSettings({ sttModel: manifest.id }) } else if (isCloud) { - onOpenOpenAiDialog(manifest.id) + onOpenCloudSetup(manifest) } else if (!isDownloading) { // Keep the menu open so download progress remains visible. event.preventDefault() diff --git a/src/renderer/src/components/settings/voice/speech-model-description.ts b/src/renderer/src/components/settings/voice/speech-model-description.ts index 22ade6810d1..22a44517222 100644 --- a/src/renderer/src/components/settings/voice/speech-model-description.ts +++ b/src/renderer/src/components/settings/voice/speech-model-description.ts @@ -52,6 +52,11 @@ const DESCRIPTIONS: Record string> = { 'settings.voice.modelDescriptions.senseVoice', 'Chinese, English, Japanese, Korean, and Cantonese with automatic language detection.' ), + 'openai-compatible': () => + translate( + 'settings.voice.modelDescriptions.openAiCompatible', + 'Any service with an OpenAI-style transcription API, such as Groq, Mistral or a local server.' + ), 'openai-gpt-transcribe': () => translate( 'settings.voice.modelDescriptions.gptTranscribe', diff --git a/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx b/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx index 5690af8cad9..46fe1541ecd 100644 --- a/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx +++ b/src/renderer/src/hooks/ipc-events/browser-link-materialization.test-fixture.tsx @@ -89,7 +89,6 @@ export function installBrowserLinkMaterializationProbe() { dictationHoldToRecord: false, removeFillerWords: true, terminalConfirmBeforeInsert: true, - userModels: [], openAiApiKeyConfigured: false, microphoneDeviceId: null, microphoneDeviceLabel: null, diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 035e7163b48..94e56027635 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -1156,7 +1156,8 @@ "parakeetJa": "Japanese only. Trained on 35k+ hours of natural speech. Punctuation included.", "whisperTiny": "90+ languages. Lower accuracy than Parakeet but broadest language coverage.", "senseVoice": "Chinese, English, Japanese, Korean, and Cantonese with automatic language detection.", - "gptTranscribe": "Cloud transcription with high accuracy. Requires an OpenAI API key." + "gptTranscribe": "Cloud transcription with high accuracy. Requires an OpenAI API key.", + "openAiCompatible": "Any service with an OpenAI-style transcription API, such as Groq, Mistral or a local server." } }, "diagnostics": { @@ -21380,6 +21381,24 @@ "fillerWords": { "title": "Remove Filler Words", "description": "Drops sounds like “um” and “uh” from dictated text, in the dictation language." + }, + "openAiCompatible": { + "updateFailed": "Could not update the provider", + "title": "OpenAI-compatible provider", + "description": "Audio goes to this service only while the OpenAI-compatible model is selected. It works with Groq, Mistral, OpenAI and local servers.", + "baseUrl": "Base URL", + "baseUrlPlaceholder": "https://api.groq.com/openai/v1", + "model": "Model", + "modelPlaceholder": "whisper-large-v3-turbo", + "apiKey": "API Key", + "keySaved": "Saved key kept", + "keyOptional": "Optional for local servers", + "save": "Save", + "rowConfigured": "Base URL and model saved for the OpenAI-compatible model.", + "rowMissing": "Add a base URL and model before selecting the OpenAI-compatible model.", + "edit": "Edit", + "removeLabel": "Remove OpenAI-compatible provider", + "setUp": "Set up" } }, "repositoryHooks": { diff --git a/src/shared/constants.ts b/src/shared/constants.ts index 48de46a8e36..2c2b79c257e 100644 --- a/src/shared/constants.ts +++ b/src/shared/constants.ts @@ -105,7 +105,6 @@ export function getDefaultVoiceSettings(): VoiceSettings { dictationHoldToRecord: true, removeFillerWords: true, terminalConfirmBeforeInsert: false, - userModels: [], openAiApiKeyConfigured: false, microphoneDeviceId: null, microphoneDeviceLabel: null, diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index 4f8e8bd2061..481aeb0ddca 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -5,7 +5,7 @@ export type SpeechModelType = | 'senseVoice' | 'nemo-ctc' | 'openai' -export type SpeechModelProvider = 'local' | 'openai' +export type SpeechModelProvider = 'local' | 'openai' | 'openai-compatible' export type ModelingUnit = 'bpe' | 'cjkchar' | 'cjkchar+bpe' @@ -36,6 +36,20 @@ export type SpeechModelManifest = { dictationLanguages?: readonly string[] } +export type OpenAiCompatibleSpeechProvider = { + baseUrl: string + model: string + /** Empty for servers that take no key. */ + apiKey: string +} + +export type OpenAiCompatibleSpeechProviderSummary = Omit< + OpenAiCompatibleSpeechProvider, + 'apiKey' +> & { + hasApiKey: boolean +} + export type SpeechModelStatus = 'not-downloaded' | 'downloading' | 'extracting' | 'ready' | 'error' export type SpeechModelState = { @@ -66,13 +80,6 @@ export type SpeechErrorEvent = { export type DictationState = 'idle' | 'starting' | 'listening' | 'stopping' | 'error' -export type UserModelConfig = { - id: string - type: SpeechModelType - dir: string - sampleRate?: number -} - // See dictation-shortcut-mode.ts for Auto. export type DictationMode = 'toggle' | 'hold' @@ -89,7 +96,6 @@ export type VoiceSettings = { /** Drops "um", "uh" and their equivalents in the dictation language before text is inserted. */ removeFillerWords: boolean terminalConfirmBeforeInsert: boolean - userModels: UserModelConfig[] openAiApiKeyConfigured: boolean /** null = system default input device */ microphoneDeviceId: string | null