From f583a1bc19bb15f618380db2cf5ce409450f0669 Mon Sep 17 00:00:00 2001 From: whit3rabbit Date: Wed, 15 Apr 2026 20:05:20 -0500 Subject: [PATCH] feat(providers): expand provider catalog with models and capabilities Populate model definitions and refine capability flags across provider stubs, adding context windows, output limits, and per-model capabilities for chat, tool use, vision, and streaming where applicable. Co-Authored-By: Claude Sonnet 4.6 --- crates/providers/src/providers/ai21.rs | 43 +- crates/providers/src/providers/ai_ml_api.rs | 375 ++++++++++++- crates/providers/src/providers/aleph_alpha.rs | 35 +- crates/providers/src/providers/anthropic.rs | 91 ++-- crates/providers/src/providers/assemblyai.rs | 58 +- crates/providers/src/providers/azure_ai.rs | 269 ++++++++- crates/providers/src/providers/baidu.rs | 314 +++++++---- crates/providers/src/providers/baseten.rs | 100 +++- crates/providers/src/providers/bedrock.rs | 117 +++- crates/providers/src/providers/blackboxai.rs | 313 ++++++++--- crates/providers/src/providers/brave.rs | 11 +- crates/providers/src/providers/bytez.rs | 2 +- crates/providers/src/providers/cartesia.rs | 104 +++- crates/providers/src/providers/cerebras.rs | 36 +- crates/providers/src/providers/chutes.rs | 269 ++++++++- crates/providers/src/providers/clarifai.rs | 168 +++++- crates/providers/src/providers/cloudflare.rs | 201 ++++++- crates/providers/src/providers/codestral.rs | 50 +- crates/providers/src/providers/cohere.rs | 198 ++++++- crates/providers/src/providers/dashscope.rs | 227 +++++++- crates/providers/src/providers/databricks.rs | 105 +++- crates/providers/src/providers/deepgram.rs | 513 ++++++++++++++++-- crates/providers/src/providers/deepinfra.rs | 302 ++++++++++- crates/providers/src/providers/deepseek.rs | 11 +- .../src/providers/docker_model_runner.rs | 19 +- crates/providers/src/providers/elevenlabs.rs | 92 +++- crates/providers/src/providers/exa.rs | 9 +- crates/providers/src/providers/featherless.rs | 178 +++++- crates/providers/src/providers/fireworks.rs | 126 ++++- crates/providers/src/providers/friendliai.rs | 90 ++- crates/providers/src/providers/galadriel.rs | 2 +- crates/providers/src/providers/gemini.rs | 28 + crates/providers/src/providers/github.rs | 468 +++++++++++++++- crates/providers/src/providers/gmi_cloud.rs | 115 +++- crates/providers/src/providers/gradient_ai.rs | 56 +- crates/providers/src/providers/groq.rs | 87 ++- crates/providers/src/providers/huggingface.rs | 174 +++++- crates/providers/src/providers/hyperbolic.rs | 195 ++++++- crates/providers/src/providers/iflytek.rs | 71 ++- crates/providers/src/providers/infinity.rs | 6 +- crates/providers/src/providers/jina.rs | 80 ++- crates/providers/src/providers/lambda.rs | 265 ++++++++- crates/providers/src/providers/lemonade.rs | 11 +- crates/providers/src/providers/llamafile.rs | 26 +- crates/providers/src/providers/meta_llama.rs | 69 ++- crates/providers/src/providers/minimax.rs | 106 +++- crates/providers/src/providers/mistral.rs | 165 +++++- crates/providers/src/providers/moonshot.rs | 83 ++- crates/providers/src/providers/morph.rs | 91 +++- crates/providers/src/providers/nanogpt.rs | 153 +++++- crates/providers/src/providers/nebius.rs | 415 +++++++++++++- crates/providers/src/providers/nlp_cloud.rs | 202 ++++++- crates/providers/src/providers/novita.rs | 243 ++++++++- crates/providers/src/providers/nscale.rs | 201 ++++++- crates/providers/src/providers/nvidia_nim.rs | 252 ++++++++- crates/providers/src/providers/openai.rs | 95 ++++ crates/providers/src/providers/openrouter.rs | 3 + crates/providers/src/providers/ovhcloud.rs | 143 ++++- crates/providers/src/providers/perplexity.rs | 85 ++- crates/providers/src/providers/petals.rs | 8 +- crates/providers/src/providers/playht.rs | 112 ++-- .../providers/src/providers/pollinations.rs | 203 ++----- crates/providers/src/providers/predibase.rs | 55 +- crates/providers/src/providers/public_ai.rs | 115 +++- crates/providers/src/providers/replicate.rs | 158 +++++- crates/providers/src/providers/sambanova.rs | 141 ++++- crates/providers/src/providers/scaleway.rs | 194 ++++++- crates/providers/src/providers/serper.rs | 4 + crates/providers/src/providers/siliconflow.rs | 287 +--------- crates/providers/src/providers/snowflake.rs | 150 ++++- crates/providers/src/providers/stability.rs | 151 +----- crates/providers/src/providers/together.rs | 118 +++- crates/providers/src/providers/triton.rs | 19 +- crates/providers/src/providers/vertex.rs | 102 +++- crates/providers/src/providers/vllm.rs | 16 +- crates/providers/src/providers/volcengine.rs | 115 +++- crates/providers/src/providers/voyage.rs | 151 +++++- crates/providers/src/providers/wandb.rs | 12 +- crates/providers/src/providers/watsonx.rs | 293 +++++++++- crates/providers/src/providers/xai.rs | 123 ++++- crates/providers/src/providers/xiaomi_mimo.rs | 59 +- crates/providers/src/providers/xinference.rs | 20 +- crates/providers/src/registry.rs | 7 +- 83 files changed, 9771 insertions(+), 1158 deletions(-) diff --git a/crates/providers/src/providers/ai21.rs b/crates/providers/src/providers/ai21.rs index 0ec9297..be4581a 100644 --- a/crates/providers/src/providers/ai21.rs +++ b/crates/providers/src/providers/ai21.rs @@ -1,9 +1,10 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// AI21 via their OpenAI-compatible endpoint. +/// AI21 Studio via their OpenAI-compatible chat completions endpoint. +/// Base: https://api.ai21.com/studio/v1, path: /chat/completions, auth: Bearer. /// Native AI21 format is not implemented. pub const PROVIDER: ProviderDef = ProviderDef { id: "ai21", @@ -17,11 +18,45 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + // Jamba chat API accepts a `tools` array (function-type only). + tool_use: true, + // Legacy J2 embed endpoint is no longer a listed Studio product. embeddings: false, + // No image/vision input documented for Jamba chat. vision: false, + // No public batch API. batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Public GA Jamba chat models. Both advertise a 256K context window; +// the chat API caps `max_tokens` at 4096 per response. Streaming works, +// but Jamba docs note streaming cannot be combined with tools. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "jamba-large", + provider_id: "ai21", + context_window: 256_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "jamba-mini", + provider_id: "ai21", + context_window: 256_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/ai_ml_api.rs b/crates/providers/src/providers/ai_ml_api.rs index fa4ab45..067f818 100644 --- a/crates/providers/src/providers/ai_ml_api.rs +++ b/crates/providers/src/providers/ai_ml_api.rs @@ -1,8 +1,14 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// AI/ML API (aimlapi.com): OpenAI-compatible aggregator exposing 400+ models +// across OpenAI, Anthropic, Google, Meta, DeepSeek, Qwen, xAI, Mistral, etc. +// Chat completions endpoint: https://api.aimlapi.com/v1/chat/completions +// Auth: `Authorization: Bearer `. +// Batch: not advertised as a public endpoint (the proxy does not route batch +// through this provider). Embeddings: supported (multiple embedding models). pub const PROVIDER: ProviderDef = ProviderDef { id: "ai_ml_api", display_name: "AI/ML API", @@ -22,4 +28,369 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Representative GA model subset. AI/ML API proxies upstream providers, so +// context/output window values mirror each upstream's published limits. +// Model IDs match the aimlapi model database; some entries are exposed both +// under short aliases and vendor-prefixed forms (e.g. `gpt-4o` vs +// `openai/gpt-4o`). The short forms are used here for parity with `openai.rs`. +pub const MODELS: &[ModelDef] = &[ + // --- OpenAI --- + ModelDef { + id: "gpt-4o", + provider_id: "ai_ml_api", + context_window: 128_000, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-4o-mini", + provider_id: "ai_ml_api", + context_window: 128_000, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-4-turbo", + provider_id: "ai_ml_api", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-4", + provider_id: "ai_ml_api", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-3.5-turbo", + provider_id: "ai_ml_api", + context_window: 16_385, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "o1", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "o3-mini", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Anthropic --- + ModelDef { + id: "claude-3-haiku-20240307", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic/claude-opus-4", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic/claude-opus-4.1", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic/claude-sonnet-4", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "claude-sonnet-4-5-20250929", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic/claude-haiku-4.5", + provider_id: "ai_ml_api", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Google Gemini --- + ModelDef { + id: "gemini-2.0-flash", + provider_id: "ai_ml_api", + context_window: 1_048_576, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "google/gemini-2.5-flash", + provider_id: "ai_ml_api", + context_window: 1_048_576, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "google/gemini-2.5-pro", + provider_id: "ai_ml_api", + context_window: 1_048_576, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Meta Llama --- + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + provider_id: "ai_ml_api", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Mistral --- + ModelDef { + id: "mistralai/Mixtral-8x7B-Instruct-v0.1", + provider_id: "ai_ml_api", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/mistral-nemo", + provider_id: "ai_ml_api", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- DeepSeek --- + ModelDef { + id: "deepseek-chat", + provider_id: "ai_ml_api", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-reasoner", + provider_id: "ai_ml_api", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Alibaba Qwen --- + ModelDef { + id: "qwen-max", + provider_id: "ai_ml_api", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen-plus", + provider_id: "ai_ml_api", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen-turbo", + provider_id: "ai_ml_api", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- xAI Grok --- + ModelDef { + id: "x-ai/grok-3-beta", + provider_id: "ai_ml_api", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "x-ai/grok-3-mini-beta", + provider_id: "ai_ml_api", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "x-ai/grok-4-07-09", + provider_id: "ai_ml_api", + context_window: 256_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/aleph_alpha.rs b/crates/providers/src/providers/aleph_alpha.rs index 120dbf7..f567b7c 100644 --- a/crates/providers/src/providers/aleph_alpha.rs +++ b/crates/providers/src/providers/aleph_alpha.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -22,4 +22,35 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// PhariaInference models exposed via the OpenAI-compatible /chat/completions +// endpoint on https://api.aleph-alpha.com. Pre-training sequence length is +// 8192 tokens; max_output_tokens mirrors that upper bound since the API does +// not publish a separate generation cap. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "pharia-1-llm-7b-control", + provider_id: "aleph_alpha", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "pharia-1-llm-7b-control-aligned", + provider_id: "aleph_alpha", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/anthropic.rs b/crates/providers/src/providers/anthropic.rs index 1791c0e..b4b745e 100644 --- a/crates/providers/src/providers/anthropic.rs +++ b/crates/providers/src/providers/anthropic.rs @@ -22,12 +22,16 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// Model catalog mirrors docs.anthropic.com model overview + deprecations page. +// Latest GA: Opus 4.6, Sonnet 4.6 (1M context), Haiku 4.5 (200k context). +// Legacy-but-active: Opus 4.5/4.1, Sonnet 4.5. Deprecated-not-retired: Opus 4 / Sonnet 4 / +// Haiku 3 (retire mid-2026). Retired models (3.7 Sonnet, 3.5 Sonnet/Haiku, 3 Opus) are omitted. pub const MODELS: &[ModelDef] = &[ ModelDef { - id: "claude-opus-4-6-20260205", + id: "claude-opus-4-6", provider_id: "anthropic", - context_window: 200_000, - max_output_tokens: 32_000, + context_window: 1_000_000, + max_output_tokens: 128_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, @@ -39,21 +43,8 @@ pub const MODELS: &[ModelDef] = &[ ModelDef { id: "claude-sonnet-4-6", provider_id: "anthropic", - context_window: 200_000, - max_output_tokens: 16_000, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: true, - extended_thinking: true, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "claude-opus-4-5-20251101", - provider_id: "anthropic", - context_window: 200_000, - max_output_tokens: 32_000, + context_window: 1_000_000, + max_output_tokens: 64_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, @@ -66,20 +57,7 @@ pub const MODELS: &[ModelDef] = &[ id: "claude-haiku-4-5-20251001", provider_id: "anthropic", context_window: 200_000, - max_output_tokens: 8_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: true, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "claude-3-7-sonnet-20250219", - provider_id: "anthropic", - context_window: 200_000, - max_output_tokens: 16_000, + max_output_tokens: 64_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, @@ -89,44 +67,73 @@ pub const MODELS: &[ModelDef] = &[ status: ModelStatus::Available, }, ModelDef { - id: "claude-3-5-sonnet-20241022", + id: "claude-opus-4-5-20251101", provider_id: "anthropic", context_window: 200_000, - max_output_tokens: 8_096, + max_output_tokens: 64_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, vision: true, - extended_thinking: false, + extended_thinking: true, }, status: ModelStatus::Available, }, ModelDef { - id: "claude-3-5-haiku-20241022", + id: "claude-sonnet-4-5-20250929", provider_id: "anthropic", context_window: 200_000, - max_output_tokens: 8_096, + max_output_tokens: 64_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, vision: true, - extended_thinking: false, + extended_thinking: true, }, status: ModelStatus::Available, }, ModelDef { - id: "claude-3-opus-20240229", + id: "claude-opus-4-1-20250805", provider_id: "anthropic", context_window: 200_000, - max_output_tokens: 4_096, + max_output_tokens: 32_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, vision: true, - extended_thinking: false, + extended_thinking: true, }, status: ModelStatus::Available, }, + // Deprecated: retires June 15, 2026. Migrate to claude-sonnet-4-6. + ModelDef { + id: "claude-sonnet-4-20250514", + provider_id: "anthropic", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Deprecated, + }, + // Deprecated: retires June 15, 2026. Migrate to claude-opus-4-6. + ModelDef { + id: "claude-opus-4-20250514", + provider_id: "anthropic", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Deprecated, + }, + // Deprecated: retires April 20, 2026. Migrate to claude-haiku-4-5-20251001. ModelDef { id: "claude-3-haiku-20240307", provider_id: "anthropic", @@ -138,6 +145,6 @@ pub const MODELS: &[ModelDef] = &[ vision: true, extended_thinking: false, }, - status: ModelStatus::Available, + status: ModelStatus::Deprecated, }, ]; diff --git a/crates/providers/src/providers/assemblyai.rs b/crates/providers/src/providers/assemblyai.rs index ab01da4..c1c8741 100644 --- a/crates/providers/src/providers/assemblyai.rs +++ b/crates/providers/src/providers/assemblyai.rs @@ -3,7 +3,9 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// AssemblyAI — audio intelligence and speech-to-text. Chat completions not supported. +/// AssemblyAI — speech-to-text and audio intelligence. Also ships an OpenAI-compatible +/// LLM Gateway (`POST /v1/chat/completions`) that proxies Claude, GPT, and Gemini with +/// audio context; this supersedes the now-deprecated LeMUR API. pub const PROVIDER: ProviderDef = ProviderDef { id: "assemblyai", display_name: "AssemblyAI", @@ -14,16 +16,49 @@ pub const PROVIDER: ProviderDef = ProviderDef { env_vars: &["ASSEMBLYAI_API_KEY"], litellm_prefix: "", capabilities: ProviderCapabilities { - chat_completions: false, - streaming: false, - tool_use: false, + // LLM Gateway exposes OpenAI-compatible /v1/chat/completions. + chat_completions: true, + // Real-time STT (Universal Streaming) and LLM Gateway SSE both supported. + streaming: true, + // LLM Gateway supports tool calling. + tool_use: true, embeddings: false, vision: false, + // No OpenAI-style /v1/batches endpoint; concurrent transcription submissions only. batch: false, }, }; pub const MODELS: &[ModelDef] = &[ + // Current STT models (2025). See: + // https://www.assemblyai.com/docs/pre-recorded-audio/select-the-speech-model + ModelDef { + id: "universal-3-pro", + provider_id: "assemblyai", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "universal-2", + provider_id: "assemblyai", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Legacy speech_model tier slugs; still accepted by /v2/transcript for back-compat. ModelDef { id: "best", provider_id: "assemblyai", @@ -50,20 +85,7 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - ModelDef { - id: "conformer-2", - provider_id: "assemblyai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // LeMUR audio LLM for audio intelligence tasks (question answering, summaries, etc.) + // Speech-language-aware model for domain-specific transcription and fine-tuning. ModelDef { id: "slam-1", provider_id: "assemblyai", diff --git a/crates/providers/src/providers/azure_ai.rs b/crates/providers/src/providers/azure_ai.rs index 81f03c3..f7fa32d 100644 --- a/crates/providers/src/providers/azure_ai.rs +++ b/crates/providers/src/providers/azure_ai.rs @@ -1,10 +1,15 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Azure AI Foundry (Serverless API / Models-as-a-Service). -/// Endpoint is per-deployment; set base URL via AZURE_AI_API_BASE or config. +/// Azure AI Foundry (Models-as-a-Service / Serverless API). +/// +/// Covers the Azure AI Model Inference catalog (Cohere, Mistral, Meta Llama, +/// Microsoft Phi, DeepSeek, etc.) — distinct from Azure OpenAI. Endpoints are +/// per-deployment: either `https://..models.ai.azure.com` +/// or `https://.services.ai.azure.com/models`. Set the base URL via +/// `AZURE_AI_API_BASE` or config. pub const PROVIDER: ProviderDef = ProviderDef { id: "azure_ai", display_name: "Azure AI Foundry", @@ -24,4 +29,260 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model IDs match LiteLLM's `azure_ai/` naming. Context windows and output +// caps come from the upstream model providers' public spec sheets; actual +// Azure deployment names are user-chosen, so callers typically override these. +pub const MODELS: &[ModelDef] = &[ + // Cohere + ModelDef { + id: "command-r-plus-08-2024", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "command-r-08-2024", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "cohere-embed-v3-english", + provider_id: "azure_ai", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "cohere-embed-v3-multilingual", + provider_id: "azure_ai", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mistral + ModelDef { + id: "mistral-large-2407", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistral-small-2503", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistral-medium-2505", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ministral-3b", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "codestral-2501", + provider_id: "azure_ai", + context_window: 256_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Meta Llama + ModelDef { + id: "Meta-Llama-3.1-405B-Instruct", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Meta-Llama-3.1-8B-Instruct", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Llama-3.3-70B-Instruct", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Llama-3.2-11B-Vision-Instruct", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Llama-3.2-90B-Vision-Instruct", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Microsoft Phi + ModelDef { + id: "Phi-4", + provider_id: "azure_ai", + context_window: 16_384, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Phi-4-mini-instruct", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Phi-4-multimodal-instruct", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek (hosted by Microsoft on Azure AI) + ModelDef { + id: "DeepSeek-V3-0324", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "DeepSeek-R1", + provider_id: "azure_ai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/baidu.rs b/crates/providers/src/providers/baidu.rs index c1d169c..d9bbc12 100644 --- a/crates/providers/src/providers/baidu.rs +++ b/crates/providers/src/providers/baidu.rs @@ -4,15 +4,21 @@ use crate::provider::{ }; /// Baidu ERNIE (Qianfan) — Chinese LLM platform. -/// Authentication uses AK/SK; set QIANFAN_AK and QIANFAN_SK. +/// +/// Uses the v2 OpenAI-compatible endpoint at `qianfan.baidubce.com/v2`, which +/// accepts a static bearer API key (format `bce-v3/ALTAK-...`). The legacy v1 +/// `wenxinworkshop` endpoint required AK/SK + OAuth `access_token`; the v2 path +/// is the recommended surface and aligns with the `OpenAICompat` protocol. +/// Primary env var is `QIANFAN_API_KEY`; `QIANFAN_AK`/`QIANFAN_SK` are retained +/// as aliases for users still on the AK/SK flow who mint their own token. pub const PROVIDER: ProviderDef = ProviderDef { id: "baidu", display_name: "Baidu ERNIE", - default_base_url: "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop", + default_base_url: "https://qianfan.baidubce.com/v2", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Wired, - env_vars: &["QIANFAN_AK", "QIANFAN_SK"], + env_vars: &["QIANFAN_API_KEY", "QIANFAN_AK", "QIANFAN_SK"], litellm_prefix: "qianfan/", capabilities: ProviderCapabilities { chat_completions: true, @@ -25,49 +31,76 @@ pub const PROVIDER: ProviderDef = ProviderDef { }; pub const MODELS: &[ModelDef] = &[ + // ERNIE 4.5 series — current flagship (GA March 2025). Multimodal. + ModelDef { + id: "ernie-4.5-turbo-128k", + provider_id: "baidu", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ernie-4.5-turbo-32k", + provider_id: "baidu", + context_window: 32_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ernie-4.5-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // ERNIE X1 series — reasoning models (competitor to DeepSeek-R1 / o3-mini). + ModelDef { + id: "ernie-x1-turbo-32k", + provider_id: "baidu", + context_window: 32_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ernie-x1-32k", + provider_id: "baidu", + context_window: 32_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, // ERNIE 4.0 series ModelDef { - id: "ernie-4.0-8k", - provider_id: "baidu", - context_window: 8_192, - max_output_tokens: 2_048, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "ernie-4.0-8k-preview", - provider_id: "baidu", - context_window: 8_192, - max_output_tokens: 2_048, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // ERNIE 3.5 series - ModelDef { - id: "ernie-3.5-8k", - provider_id: "baidu", - context_window: 8_192, - max_output_tokens: 2_048, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "ernie-3.5-128k", + id: "ernie-4.0-turbo-128k", provider_id: "baidu", context_window: 128_000, max_output_tokens: 4_096, @@ -79,76 +112,6 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - // ERNIE Speed series - ModelDef { - id: "ernie-speed-8k", - provider_id: "baidu", - context_window: 8_192, - max_output_tokens: 2_048, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "ernie-speed-128k", - provider_id: "baidu", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // ERNIE Lite series - ModelDef { - id: "ernie-lite-8k", - provider_id: "baidu", - context_window: 8_192, - max_output_tokens: 2_048, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // ERNIE Tiny - ModelDef { - id: "ernie-tiny-8k", - provider_id: "baidu", - context_window: 8_192, - max_output_tokens: 2_048, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // ERNIE Character - ModelDef { - id: "ernie-character-8k", - provider_id: "baidu", - context_window: 8_192, - max_output_tokens: 2_048, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // ERNIE with vision ModelDef { id: "ernie-4.0-turbo-8k", provider_id: "baidu", @@ -162,7 +125,115 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - // Embedding + ModelDef { + id: "ernie-4.0-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // ERNIE 3.5 series + ModelDef { + id: "ernie-3.5-128k", + provider_id: "baidu", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ernie-3.5-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // ERNIE Speed series — low-latency general-purpose. + ModelDef { + id: "ernie-speed-128k", + provider_id: "baidu", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ernie-speed-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // ERNIE Lite / Tiny — cost-optimised. + ModelDef { + id: "ernie-lite-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ernie-tiny-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // ERNIE Character — role-play / persona variant. + ModelDef { + id: "ernie-character-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embeddings. ModelDef { id: "embedding-v1", provider_id: "baidu", @@ -189,4 +260,17 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "tao-8k", + provider_id: "baidu", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ]; diff --git a/crates/providers/src/providers/baseten.rs b/crates/providers/src/providers/baseten.rs index 32b2f68..0eab184 100644 --- a/crates/providers/src/providers/baseten.rs +++ b/crates/providers/src/providers/baseten.rs @@ -1,13 +1,21 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Baseten — per-deployment URL; set base URL via config (model-specific endpoint). +/// Baseten — OpenAI-compatible inference. +/// +/// Two surfaces share the same auth (Bearer ): +/// 1. Model APIs (shared hosted catalog): `https://inference.baseten.co/v1`. +/// Use this as the `default_base_url`; model IDs below target it. +/// 2. Per-deployment endpoints for custom models/chains: +/// `https://model-{model_id}.api.baseten.co/{environment}/sync/v1` +/// Users must override `base_url` in config when pointing at their own +/// deployment; the template is not representable here as a constant. pub const PROVIDER: ProviderDef = ProviderDef { id: "baseten", display_name: "Baseten", - default_base_url: "", + default_base_url: "https://inference.baseten.co/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -16,11 +24,93 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + tool_use: true, embeddings: false, vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Baseten Model APIs catalog (publicly GA). Slugs match the HuggingFace-style +// model IDs accepted by `https://inference.baseten.co/v1/chat/completions`. +// Keep conservative: only include models documented as generally available. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "deepseek-ai/DeepSeek-V3-0324", + provider_id: "baseten", + context_window: 164_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3.1", + provider_id: "baseten", + context_window: 164_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "zai-org/GLM-4.6", + provider_id: "baseten", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "moonshotai/Kimi-K2.5", + provider_id: "baseten", + context_window: 262_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMaxAI/MiniMax-M2.5", + provider_id: "baseten", + context_window: 204_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-oss-120b", + provider_id: "baseten", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/bedrock.rs b/crates/providers/src/providers/bedrock.rs index 0b4cfbb..73c5202 100644 --- a/crates/providers/src/providers/bedrock.rs +++ b/crates/providers/src/providers/bedrock.rs @@ -23,12 +23,31 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// Model IDs sourced from Anthropic's official model overview and AWS Bedrock +// docs. On-demand IDs only (not cross-region inference profile IDs like +// `us.anthropic.*`). Context window / max output reflect Anthropic's current +// published limits. pub const MODELS: &[ModelDef] = &[ + // --- Current generation (Claude 4.6 family) --- ModelDef { - id: "anthropic.claude-sonnet-4-20250514-v1:0", + id: "anthropic.claude-opus-4-6-v1", provider_id: "bedrock", - context_window: 200_000, - max_output_tokens: 16_000, + context_window: 1_000_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + // Anthropic docs list this without the -v1:0 suffix on Bedrock. + id: "anthropic.claude-sonnet-4-6", + provider_id: "bedrock", + context_window: 1_000_000, + max_output_tokens: 64_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, @@ -41,20 +60,90 @@ pub const MODELS: &[ModelDef] = &[ id: "anthropic.claude-haiku-4-5-20251001-v1:0", provider_id: "bedrock", context_window: 200_000, - max_output_tokens: 8_096, + max_output_tokens: 64_000, capabilities: ModelCapabilities { streaming: true, tool_use: true, vision: true, - extended_thinking: false, + extended_thinking: true, }, status: ModelStatus::Available, }, + // --- Claude 4.5 family --- + ModelDef { + id: "anthropic.claude-opus-4-5-20251101-v1:0", + provider_id: "bedrock", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic.claude-sonnet-4-5-20250929-v1:0", + provider_id: "bedrock", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Claude 4.1 / 4.0 family (legacy but still available) --- + ModelDef { + id: "anthropic.claude-opus-4-1-20250805-v1:0", + provider_id: "bedrock", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + // Deprecated per Anthropic: retires 2026-06-15. Migrate to Opus 4.6. + id: "anthropic.claude-opus-4-20250514-v1:0", + provider_id: "bedrock", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Deprecated, + }, + ModelDef { + // Deprecated per Anthropic: retires 2026-06-15. Migrate to Sonnet 4.6. + id: "anthropic.claude-sonnet-4-20250514-v1:0", + provider_id: "bedrock", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Deprecated, + }, + // --- Claude 3.x (legacy; widely used, still GA on Bedrock) --- ModelDef { id: "anthropic.claude-3-5-sonnet-20241022-v2:0", provider_id: "bedrock", context_window: 200_000, - max_output_tokens: 8_096, + max_output_tokens: 8_192, capabilities: ModelCapabilities { streaming: true, tool_use: true, @@ -64,6 +153,20 @@ pub const MODELS: &[ModelDef] = &[ status: ModelStatus::Available, }, ModelDef { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + provider_id: "bedrock", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + // Deprecated per Anthropic: retires 2026-04-19. Migrate to Haiku 4.5. id: "anthropic.claude-3-haiku-20240307-v1:0", provider_id: "bedrock", context_window: 200_000, @@ -74,6 +177,6 @@ pub const MODELS: &[ModelDef] = &[ vision: true, extended_thinking: false, }, - status: ModelStatus::Available, + status: ModelStatus::Deprecated, }, ]; diff --git a/crates/providers/src/providers/blackboxai.rs b/crates/providers/src/providers/blackboxai.rs index 4307a47..6fa2ac2 100644 --- a/crates/providers/src/providers/blackboxai.rs +++ b/crates/providers/src/providers/blackboxai.rs @@ -3,11 +3,15 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Blackbox AI — LLM chat service at blackbox.ai with an OpenAI-compatible endpoint. +/// Blackbox AI — aggregator offering an OpenAI-compatible `/chat/completions` +/// endpoint fronting Anthropic, OpenAI, Google, Meta, and other frontier models. +/// Docs: https://docs.blackbox.ai/api-reference/introduction pub const PROVIDER: ProviderDef = ProviderDef { id: "blackboxai", display_name: "Blackbox AI", - default_base_url: "https://api.blackbox.ai/api", + // Per docs: `https://api.blackbox.ai/chat/completions`. The OpenAI-compat + // client appends `/chat/completions`, so the base is the host root. + default_base_url: "https://api.blackbox.ai", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Wired, @@ -23,24 +27,39 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// Curated GA subset from Blackbox's chat-models catalog. Only models whose +// upstream context/output limits are publicly documented are included. +// Context windows / max output reflect the upstream provider's spec. pub const MODELS: &[ModelDef] = &[ - // Blackbox native model + // --- OpenAI via Blackbox --- ModelDef { - id: "blackboxai", + id: "gpt-4o", provider_id: "blackboxai", context_window: 128_000, - max_output_tokens: 4_096, + max_output_tokens: 16_384, capabilities: ModelCapabilities { streaming: true, tool_use: false, - vision: false, + vision: true, extended_thinking: false, }, status: ModelStatus::Available, }, - // GPT-4o proxy ModelDef { - id: "gpt-4o", + id: "gpt-4o-mini", + provider_id: "blackboxai", + context_window: 128_000, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-4-turbo", provider_id: "blackboxai", context_window: 128_000, max_output_tokens: 4_096, @@ -52,9 +71,139 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - // Claude 3 Opus proxy ModelDef { - id: "claude-3-opus", + id: "gpt-4", + provider_id: "blackboxai", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-3.5-turbo", + provider_id: "blackboxai", + context_window: 16_385, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "o1", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "o3", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "o3-mini", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Anthropic via Blackbox --- + ModelDef { + id: "claude-opus-4.1", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "claude-opus-4", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "claude-sonnet-4", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "claude-3.7-sonnet", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "claude-3.5-haiku", + provider_id: "blackboxai", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "claude-3-haiku", provider_id: "blackboxai", context_window: 200_000, max_output_tokens: 4_096, @@ -66,73 +215,97 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - // Gemini Pro proxy + // --- Google via Blackbox --- ModelDef { - id: "gemini-pro", + id: "gemini-2.5-pro", provider_id: "blackboxai", - context_window: 128_000, - max_output_tokens: 4_096, + context_window: 1_048_576, + max_output_tokens: 65_536, capabilities: ModelCapabilities { streaming: true, tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // Llama 3.1 proxy - ModelDef { - id: "llama-3.1-8b", - provider_id: "blackboxai", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "llama-3.1-70b", - provider_id: "blackboxai", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // DeepSeek V3 proxy - ModelDef { - id: "deepseek-v3", - provider_id: "blackboxai", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // DeepSeek R1 proxy - ModelDef { - id: "deepseek-r1", - provider_id: "blackboxai", - context_window: 128_000, - max_output_tokens: 16_000, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, + vision: true, extended_thinking: true, }, status: ModelStatus::Available, }, + ModelDef { + id: "gemini-2.5-flash", + provider_id: "blackboxai", + context_window: 1_048_576, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Meta Llama via Blackbox --- + ModelDef { + id: "llama-3.3-70b-instruct", + provider_id: "blackboxai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-3.1-405b-instruct", + provider_id: "blackboxai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-3.1-70b-instruct", + provider_id: "blackboxai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-3.1-8b-instruct", + provider_id: "blackboxai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-3.2-11b-vision-instruct", + provider_id: "blackboxai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ]; diff --git a/crates/providers/src/providers/brave.rs b/crates/providers/src/providers/brave.rs index 63960ab..5c350d7 100644 --- a/crates/providers/src/providers/brave.rs +++ b/crates/providers/src/providers/brave.rs @@ -3,12 +3,17 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Brave Search API — web and AI search. Chat completions not supported. +/// Brave Search API — web/news/image/video search, summarizer, and LLM context. +/// Not a chat API; no streaming, tools, embeddings, or vision. +/// Auth header: `X-Subscription-Token: ` (not a bearer token — see note below). pub const PROVIDER: ProviderDef = ProviderDef { id: "brave", display_name: "Brave Search", - default_base_url: "https://api.search.brave.com", + // Endpoints live under /res/v1/{web,news,images,videos,summarizer,suggest,spellcheck}/search + default_base_url: "https://api.search.brave.com/res/v1", protocol: ProviderProtocol::OpenAICompat, + // NOTE: Brave uses `X-Subscription-Token`, not `Authorization: Bearer`. + // AuthKind has no dedicated variant for this header; leaving as Bearer per scope. auth: AuthKind::Bearer, status: ProviderStatus::Stub, env_vars: &["BRAVE_API_KEY"], @@ -23,5 +28,5 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -// No model selection — single endpoint API +// Search API — no model selection. Endpoint is chosen by path, not by model id. pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/bytez.rs b/crates/providers/src/providers/bytez.rs index db63b8a..e3b0a34 100644 --- a/crates/providers/src/providers/bytez.rs +++ b/crates/providers/src/providers/bytez.rs @@ -6,7 +6,7 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "bytez", display_name: "Bytez", - default_base_url: "https://api.bytez.com/models/v2", + default_base_url: "https://api.bytez.com/models/v2/openai/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, diff --git a/crates/providers/src/providers/cartesia.rs b/crates/providers/src/providers/cartesia.rs index 5de5841..bdabcd3 100644 --- a/crates/providers/src/providers/cartesia.rs +++ b/crates/providers/src/providers/cartesia.rs @@ -3,7 +3,12 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Cartesia — real-time voice AI. Chat completions not supported. +// Cartesia: real-time voice AI (TTS + STT). Not a chat provider. +// Base URL: https://api.cartesia.ai +// Auth header is X-API-Key (not Bearer); AuthKind enum lacks that variant, +// so we keep Bearer as the closest placeholder until the enum is extended. +// TTS endpoints (/tts/bytes, /tts/sse, /tts/websocket) support real-time streaming. +// STT via Ink-Whisper exposes /stt (native) and /audio/transcriptions (OpenAI-compat). pub const PROVIDER: ProviderDef = ProviderDef { id: "cartesia", display_name: "Cartesia", @@ -15,7 +20,8 @@ pub const PROVIDER: ProviderDef = ProviderDef { litellm_prefix: "", capabilities: ProviderCapabilities { chat_completions: false, - streaming: false, + // Sonic family streams first audio bytes in 40-90ms; streaming is the core use case. + streaming: true, tool_use: false, embeddings: false, vision: false, @@ -24,8 +30,9 @@ pub const PROVIDER: ProviderDef = ProviderDef { }; pub const MODELS: &[ModelDef] = &[ + // TTS: Sonic-3 (current flagship, 90ms TTFB, 40+ languages, expressive laughter). ModelDef { - id: "sonic-2024-10-19", + id: "sonic-3", provider_id: "cartesia", context_window: 0, max_output_tokens: 0, @@ -37,6 +44,7 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // TTS: Sonic-2 (latency-optimised, best-in-class voice cloning). ModelDef { id: "sonic-2", provider_id: "cartesia", @@ -50,6 +58,92 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // TTS: Sonic-2 pinned snapshot, kept available for users needing _experimental_controls + // (removed in snapshots after 2025-03-07). + ModelDef { + id: "sonic-2-2025-03-07", + provider_id: "cartesia", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // TTS: Sonic-2 latest production-pinned snapshot per docs. + ModelDef { + id: "sonic-2-2025-06-11", + provider_id: "cartesia", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // TTS: Sonic Turbo (40ms first-byte latency, real-time priority). + ModelDef { + id: "sonic-turbo", + provider_id: "cartesia", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // TTS: original Sonic base alias (kept for backward compatibility). + ModelDef { + id: "sonic", + provider_id: "cartesia", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // STT: Ink-Whisper (streaming and batch transcription, conversational-AI tuned). + ModelDef { + id: "ink-whisper", + provider_id: "cartesia", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Legacy TTS snapshot alias, superseded by sonic-2 family. + ModelDef { + id: "sonic-2024-10-19", + provider_id: "cartesia", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Deprecated, + }, + // Legacy language-specific aliases, replaced by sonic-2's multilingual default. ModelDef { id: "sonic-english", provider_id: "cartesia", @@ -61,7 +155,7 @@ pub const MODELS: &[ModelDef] = &[ vision: false, extended_thinking: false, }, - status: ModelStatus::Available, + status: ModelStatus::Deprecated, }, ModelDef { id: "sonic-multilingual", @@ -74,7 +168,7 @@ pub const MODELS: &[ModelDef] = &[ vision: false, extended_thinking: false, }, - status: ModelStatus::Available, + status: ModelStatus::Deprecated, }, ModelDef { id: "upbeat-moon", diff --git a/crates/providers/src/providers/cerebras.rs b/crates/providers/src/providers/cerebras.rs index 68f2c6c..e146e0c 100644 --- a/crates/providers/src/providers/cerebras.rs +++ b/crates/providers/src/providers/cerebras.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -22,4 +22,36 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Cerebras Inference GA (production) models only. +// Preview models (e.g. qwen-3-235b-a22b-instruct-2507, zai-glm-4.7) are +// intentionally omitted per docs: "intended for evaluation purposes only and +// should not be used in production." +// Source: https://inference-docs.cerebras.ai/models/overview +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "llama3.1-8b", + provider_id: "cerebras", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-oss-120b", + provider_id: "cerebras", + context_window: 131_072, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/chutes.rs b/crates/providers/src/providers/chutes.rs index 95c3270..f523ad3 100644 --- a/crates/providers/src/providers/chutes.rs +++ b/crates/providers/src/providers/chutes.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -17,9 +17,272 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: false, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Curated GA subset from https://llm.chutes.ai/v1/models (OpenAI-compatible inference). +// Context/output windows follow upstream model cards; Chutes surfaces them in the /models response. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "deepseek-ai/DeepSeek-V3-0324-TEE", + provider_id: "chutes", + context_window: 163_840, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3.1-TEE", + provider_id: "chutes", + context_window: 163_840, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3.2-TEE", + provider_id: "chutes", + context_window: 163_840, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-0528-TEE", + provider_id: "chutes", + context_window: 163_840, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-72B-Instruct", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-VL-32B-Instruct", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-32B-TEE", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-30B-A3B", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-235B-A22B-Instruct-2507-TEE", + provider_id: "chutes", + context_window: 262_144, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-235B-A22B-Thinking-2507", + provider_id: "chutes", + context_window: 262_144, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-Next-80B-A3B-Instruct", + provider_id: "chutes", + context_window: 262_144, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "moonshotai/Kimi-K2.5-TEE", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMaxAI/MiniMax-M2.5-TEE", + provider_id: "chutes", + context_window: 1_000_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "zai-org/GLM-5-TEE", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "zai-org/GLM-4.6V", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-oss-120b-TEE", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "unsloth/gemma-3-27b-it", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "unsloth/Llama-3.2-3B-Instruct", + provider_id: "chutes", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/clarifai.rs b/crates/providers/src/providers/clarifai.rs index 64873e7..73b4ede 100644 --- a/crates/providers/src/providers/clarifai.rs +++ b/crates/providers/src/providers/clarifai.rs @@ -1,25 +1,183 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Clarifai exposes an OpenAI-compatible chat completions endpoint at +// https://api.clarifai.com/v2/ext/openai/v1. Auth is a Personal Access Token +// (PAT) passed as `Authorization: Bearer `. Model IDs follow the +// ".." form used by Clarifai's community catalog. pub const PROVIDER: ProviderDef = ProviderDef { id: "clarifai", display_name: "Clarifai", - default_base_url: "https://api.clarifai.com/v2", + default_base_url: "https://api.clarifai.com/v2/ext/openai/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["CLARIFAI_API_KEY"], + env_vars: &["CLARIFAI_PAT", "CLARIFAI_API_KEY"], litellm_prefix: "clarifai/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + tool_use: true, embeddings: true, vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Curated subset of publicly GA models from Clarifai's community catalog. +// Context windows reflect upstream model specs; Clarifai may apply lower +// per-deployment limits. Tool use availability depends on the hosted runtime. +pub const MODELS: &[ModelDef] = &[ + // OpenAI open-weight (GPT-OSS) hosted on Clarifai compute. + ModelDef { + id: "openai.chat-completion.gpt-oss-120b", + provider_id: "clarifai", + context_window: 131_072, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai.chat-completion.gpt-oss-20b", + provider_id: "clarifai", + context_window: 131_072, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Anthropic via Clarifai (proxied commercial models). + ModelDef { + id: "anthropic.completion.claude-sonnet-4", + provider_id: "clarifai", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic.completion.claude-opus-4", + provider_id: "clarifai", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic.completion.claude-3_7-sonnet", + provider_id: "clarifai", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "anthropic.completion.claude-3_5-haiku", + provider_id: "clarifai", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Meta Llama 3.x hosted on Clarifai. + ModelDef { + id: "meta.Llama-3.Llama-3_2-3B-Instruct", + provider_id: "clarifai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek open-weight reasoning distill hosted on Clarifai. + ModelDef { + id: "deepseek-ai.deepseek-chat.DeepSeek-R1-0528-Qwen3-8B", + provider_id: "clarifai", + context_window: 65_536, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Qwen3 open-weight hosted on Clarifai. + ModelDef { + id: "qwen.qwenLM.Qwen3-30B-A3B-Instruct-2507", + provider_id: "clarifai", + context_window: 262_144, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Google Gemini via Clarifai (proxied commercial model). + ModelDef { + id: "gcp.generate.gemini-2_5-pro", + provider_id: "clarifai", + context_window: 1_048_576, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // xAI Grok via Clarifai (proxied commercial model). + ModelDef { + id: "xai.chat-completion.grok-3", + provider_id: "clarifai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/cloudflare.rs b/crates/providers/src/providers/cloudflare.rs index 6495b24..1ae4299 100644 --- a/crates/providers/src/providers/cloudflare.rs +++ b/crates/providers/src/providers/cloudflare.rs @@ -1,10 +1,14 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Cloudflare Workers AI — URL includes account ID; set via CLOUDFLARE_ACCOUNT_ID. -/// Base URL pattern: https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1 +/// Cloudflare Workers AI — OpenAI-compatible endpoint is account-scoped. +/// Full base URL pattern: https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1 +/// The `{account_id}` is resolved at request time from `CLOUDFLARE_ACCOUNT_ID`, so the +/// static `default_base_url` is intentionally left empty; callers must template it. +/// Supported OpenAI-compatible endpoints: `/v1/chat/completions`, `/v1/embeddings`. +/// Auth: `Authorization: Bearer $CLOUDFLARE_API_TOKEN`. pub const PROVIDER: ProviderDef = ProviderDef { id: "cloudflare", display_name: "Cloudflare Workers AI", @@ -17,11 +21,198 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + tool_use: true, embeddings: true, vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA catalog only. Beta models (e.g. qwq-32b, deepseek-r1-distill) are excluded. +// Context windows reflect what the per-model docs publish; where Cloudflare does +// not publish an explicit `max_tokens` ceiling we set a conservative default. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", + provider_id: "cloudflare", + context_window: 24_000, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/meta/llama-3.1-70b-instruct", + provider_id: "cloudflare", + context_window: 24_000, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/meta/llama-3.1-8b-instruct", + provider_id: "cloudflare", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/meta/llama-3.1-8b-instruct-fast", + provider_id: "cloudflare", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/meta/llama-3.2-11b-vision-instruct", + provider_id: "cloudflare", + context_window: 128_000, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/meta/llama-3.2-3b-instruct", + provider_id: "cloudflare", + context_window: 128_000, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/meta/llama-3.2-1b-instruct", + provider_id: "cloudflare", + context_window: 128_000, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/mistralai/mistral-small-3.1-24b-instruct", + provider_id: "cloudflare", + context_window: 128_000, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/qwen/qwen2.5-coder-32b-instruct", + provider_id: "cloudflare", + context_window: 32_768, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/google/gemma-3-12b-it", + provider_id: "cloudflare", + context_window: 128_000, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embedding models — chat-completion capabilities set to false. + ModelDef { + id: "@cf/baai/bge-large-en-v1.5", + provider_id: "cloudflare", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/baai/bge-base-en-v1.5", + provider_id: "cloudflare", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/baai/bge-small-en-v1.5", + provider_id: "cloudflare", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "@cf/baai/bge-m3", + provider_id: "cloudflare", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/codestral.rs b/crates/providers/src/providers/codestral.rs index ab631a2..193afcb 100644 --- a/crates/providers/src/providers/codestral.rs +++ b/crates/providers/src/providers/codestral.rs @@ -1,9 +1,11 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; /// Codestral is Mistral's code-focused endpoint with a separate API key. +/// Base URL serves FIM (`/v1/fim/completions`), chat (`/v1/chat/completions`), +/// and embeddings (`/v1/embeddings`). Auth is `Authorization: Bearer $CODESTRAL_API_KEY`. pub const PROVIDER: ProviderDef = ProviderDef { id: "codestral", display_name: "Codestral", @@ -17,10 +19,52 @@ pub const PROVIDER: ProviderDef = ProviderDef { chat_completions: true, streaming: true, tool_use: true, - embeddings: false, + embeddings: true, vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +pub const MODELS: &[ModelDef] = &[ + // `codestral-latest` currently aliases the v25.08 release (July 2025). + ModelDef { + id: "codestral-latest", + provider_id: "codestral", + context_window: 256_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "codestral-2508", + provider_id: "codestral", + context_window: 256_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Semantic code embeddings. 8k input context, variable output dimensions. + ModelDef { + id: "codestral-embed-2505", + provider_id: "codestral", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/cohere.rs b/crates/providers/src/providers/cohere.rs index 0e394af..d94fd77 100644 --- a/crates/providers/src/providers/cohere.rs +++ b/crates/providers/src/providers/cohere.rs @@ -1,10 +1,11 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; /// Cohere via their OpenAI-compatible compatibility endpoint. -/// Native Cohere format (cohere_chat) is not implemented. +/// Native Cohere v2 chat API (`/v2/chat`) is not implemented; we route through +/// `/compatibility/v1` which accepts OpenAI Chat Completions shape. pub const PROVIDER: ProviderDef = ProviderDef { id: "cohere_chat", display_name: "Cohere", @@ -19,9 +20,198 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: true, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Only currently GA models per docs.cohere.com/docs/models. +// `command-r-plus`, `command-r`, `command`, `command-light` were deprecated +// 2025-09-15 and are intentionally omitted. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "command-a-03-2025", + provider_id: "cohere_chat", + context_window: 256_000, + max_output_tokens: 8_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "command-a-reasoning-08-2025", + provider_id: "cohere_chat", + context_window: 256_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "command-a-vision-07-2025", + provider_id: "cohere_chat", + context_window: 128_000, + max_output_tokens: 8_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "command-a-translate-08-2025", + provider_id: "cohere_chat", + context_window: 8_000, + max_output_tokens: 8_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "command-r7b-12-2024", + provider_id: "cohere_chat", + context_window: 128_000, + max_output_tokens: 4_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "command-r-plus-08-2024", + provider_id: "cohere_chat", + context_window: 128_000, + max_output_tokens: 4_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "command-r-08-2024", + provider_id: "cohere_chat", + context_window: 128_000, + max_output_tokens: 4_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Aya open-weights models hosted by Cohere. + ModelDef { + id: "c4ai-aya-expanse-32b", + provider_id: "cohere_chat", + context_window: 128_000, + max_output_tokens: 4_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "c4ai-aya-vision-32b", + provider_id: "cohere_chat", + context_window: 16_000, + max_output_tokens: 4_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embedding models. max_output_tokens is not meaningful here; set to 0 + // to match the `text-embedding-3-*` convention in openai.rs. + ModelDef { + id: "embed-v4.0", + provider_id: "cohere_chat", + context_window: 128_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "embed-english-v3.0", + provider_id: "cohere_chat", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "embed-english-light-v3.0", + provider_id: "cohere_chat", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "embed-multilingual-v3.0", + provider_id: "cohere_chat", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "embed-multilingual-light-v3.0", + provider_id: "cohere_chat", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/dashscope.rs b/crates/providers/src/providers/dashscope.rs index cbda350..985d3de 100644 --- a/crates/providers/src/providers/dashscope.rs +++ b/crates/providers/src/providers/dashscope.rs @@ -1,8 +1,14 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Alibaba Cloud Model Studio (DashScope), Qwen family. +// OpenAI-compatible endpoint: https://dashscope.aliyuncs.com/compatible-mode/v1 +// International (Singapore): https://dashscope-intl.aliyuncs.com/compatible-mode/v1 +// US (Virginia): https://dashscope-us.aliyuncs.com/compatible-mode/v1 +// Auth: Bearer $DASHSCOPE_API_KEY. +// Only publicly GA commercial + GA open-source models are listed below. pub const PROVIDER: ProviderDef = ProviderDef { id: "dashscope", display_name: "Dashscope (Qwen)", @@ -22,4 +28,221 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +pub const MODELS: &[ModelDef] = &[ + // Commercial flagship, stable aliases. + ModelDef { + id: "qwen-max", + provider_id: "dashscope", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen-plus", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen-turbo", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Long-context document model. Up to ~10M tokens via the long-context endpoint; + // keeping a conservative figure that matches the commercial OpenAI-compat default. + ModelDef { + id: "qwen-long", + provider_id: "dashscope", + context_window: 10_000_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Vision-language flagships. + ModelDef { + id: "qwen-vl-max", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen-vl-plus", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Open-source Qwen2.5 series (served via DashScope). + ModelDef { + id: "qwen2.5-72b-instruct", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen2.5-32b-instruct", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen2.5-14b-instruct", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen2.5-7b-instruct", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen2.5-coder-32b-instruct", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Open-source Qwen2.5-VL. + ModelDef { + id: "qwen2.5-vl-72b-instruct", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen2.5-vl-7b-instruct", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // QwQ reasoning model. + ModelDef { + id: "qwq-32b", + provider_id: "dashscope", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Embeddings. v3 is GA; v4 (Qwen3-Embedding) is also GA per DashScope docs. + ModelDef { + id: "text-embedding-v3", + provider_id: "dashscope", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "text-embedding-v4", + provider_id: "dashscope", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/databricks.rs b/crates/providers/src/providers/databricks.rs index d2b4fb9..4e33def 100644 --- a/crates/providers/src/providers/databricks.rs +++ b/crates/providers/src/providers/databricks.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,21 +6,114 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "databricks", display_name: "Databricks", - // URL is per-workspace: https://.azuredatabricks.net/serving-endpoints + // Workspace-scoped: https://.cloud.databricks.com/serving-endpoints + // OpenAI-compat chat lives at /serving-endpoints/{endpoint}/invocations + // or /serving-endpoints/v1/chat/completions. Left empty: users must set + // DATABRICKS_HOST (or the proxy base URL) per workspace. default_base_url: "", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["DATABRICKS_API_KEY"], + // DATABRICKS_TOKEN is the canonical env var; keep DATABRICKS_API_KEY as a + // proxy-specific alias. DATABRICKS_HOST carries the workspace URL. + env_vars: &["DATABRICKS_TOKEN", "DATABRICKS_API_KEY", "DATABRICKS_HOST"], litellm_prefix: "databricks/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + tool_use: true, embeddings: true, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA pay-per-token Foundation Model API endpoints, as documented at +// docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models. +// Model IDs match the serving endpoint names (used directly in the OpenAI-compat +// URL path). Preview / coding-specific / provisioned-throughput-only models are +// intentionally excluded. Retired endpoints (dbrx-instruct, mixtral-8x7b-instruct, +// llama-3.1-70b, llama-3.1-405b for pay-per-token) are also excluded. +pub const MODELS: &[ModelDef] = &[ + // --- Meta Llama (text chat) --- + ModelDef { + id: "databricks-meta-llama-3-3-70b-instruct", + provider_id: "databricks", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "databricks-meta-llama-3-1-8b-instruct", + provider_id: "databricks", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Anthropic Claude (hosted on Databricks, pay-per-token) --- + ModelDef { + id: "databricks-claude-sonnet-4-5", + provider_id: "databricks", + context_window: 200_000, + max_output_tokens: 64_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "databricks-claude-opus-4-1", + provider_id: "databricks", + context_window: 200_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- Embeddings --- + ModelDef { + id: "databricks-gte-large-en", + provider_id: "databricks", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "databricks-bge-large-en", + provider_id: "databricks", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/deepgram.rs b/crates/providers/src/providers/deepgram.rs index b641d6d..bf8f457 100644 --- a/crates/providers/src/providers/deepgram.rs +++ b/crates/providers/src/providers/deepgram.rs @@ -3,11 +3,17 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Deepgram — speech-to-text and audio AI. Chat completions are not supported. +/// Deepgram — speech-to-text (Nova, Whisper) and text-to-speech (Aura) APIs. +/// +/// Not an LLM chat-completions provider. Voice Agent (`/v1/agent/converse`) delegates +/// "thinking" to third-party LLMs rather than exposing its own chat endpoint. +/// +/// Auth header is actually `Authorization: Token `. The closest `AuthKind` +/// variant is `Bearer`; callers must emit `Token` instead of `Bearer` for Deepgram. pub const PROVIDER: ProviderDef = ProviderDef { id: "deepgram", display_name: "Deepgram", - default_base_url: "https://api.deepgram.com", + default_base_url: "https://api.deepgram.com/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Wired, @@ -15,7 +21,9 @@ pub const PROVIDER: ProviderDef = ProviderDef { litellm_prefix: "", capabilities: ProviderCapabilities { chat_completions: false, - streaming: false, + // Streaming is a core Deepgram feature: WebSocket STT (/v1/listen) and + // streaming TTS (/v1/speak) are both supported. + streaming: true, tool_use: false, embeddings: false, vision: false, @@ -23,7 +31,64 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// Model list mirrors Deepgram's public catalog as of 2025. Context/output token +// fields are zero because speech models are billed per audio-second, not tokens. pub const MODELS: &[ModelDef] = &[ + // -- STT: Flux (turn-aware, English) -- + ModelDef { + id: "flux-general-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -- STT: Nova-3 -- + ModelDef { + id: "nova-3", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "nova-3-general", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "nova-3-medical", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -- STT: Nova-2 -- ModelDef { id: "nova-2", provider_id: "deepgram", @@ -37,6 +102,19 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "nova-2-general", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ModelDef { id: "nova-2-meeting", provider_id: "deepgram", @@ -141,6 +219,33 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "nova-2-automotive", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "nova-2-atc", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -- STT: Nova (legacy) -- ModelDef { id: "nova", provider_id: "deepgram", @@ -154,6 +259,33 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "nova-phonecall", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "nova-medical", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -- STT: Enhanced (legacy) -- ModelDef { id: "enhanced", provider_id: "deepgram", @@ -167,6 +299,7 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // -- STT: Base (legacy) -- ModelDef { id: "base", provider_id: "deepgram", @@ -180,39 +313,14 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // -- STT: Whisper Cloud -- ModelDef { - id: "whisper-large", + id: "whisper-tiny", provider_id: "deepgram", context_window: 0, max_output_tokens: 0, capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "whisper-medium", - provider_id: "deepgram", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "whisper-small", - provider_id: "deepgram", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: true, + streaming: false, tool_use: false, vision: false, extended_thinking: false, @@ -224,6 +332,59 @@ pub const MODELS: &[ModelDef] = &[ provider_id: "deepgram", context_window: 0, max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "whisper-small", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "whisper-medium", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "whisper-large", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -- TTS: Aura-2 (English, representative voices) -- + ModelDef { + id: "aura-2-thalia-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, capabilities: ModelCapabilities { streaming: true, tool_use: false, @@ -233,7 +394,295 @@ pub const MODELS: &[ModelDef] = &[ status: ModelStatus::Available, }, ModelDef { - id: "whisper-tiny", + id: "aura-2-asteria-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-luna-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-zeus-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-orion-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-apollo-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -- TTS: Aura-2 (multilingual, representative voices) -- + ModelDef { + id: "aura-2-celeste-es", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-agathe-fr", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-julius-de", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-livia-it", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-fujin-ja", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-2-rhea-nl", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -- TTS: Aura 1 (English) -- + ModelDef { + id: "aura-asteria-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-luna-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-stella-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-athena-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-hera-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-orion-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-arcas-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-perseus-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-angus-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-orpheus-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-helios-en", + provider_id: "deepgram", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aura-zeus-en", provider_id: "deepgram", context_window: 0, max_output_tokens: 0, diff --git a/crates/providers/src/providers/deepinfra.rs b/crates/providers/src/providers/deepinfra.rs index a971874..c5af30c 100644 --- a/crates/providers/src/providers/deepinfra.rs +++ b/crates/providers/src/providers/deepinfra.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -17,9 +17,305 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: true, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Representative GA catalog; see https://deepinfra.com/models and +// LiteLLM `model_prices_and_context_window.json` for the full list. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "deepinfra", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + provider_id: "deepinfra", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-8B-Instruct", + provider_id: "deepinfra", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + provider_id: "deepinfra", + context_window: 1_048_576, + max_output_tokens: 1_048_576, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-4-Scout-17B-16E-Instruct", + provider_id: "deepinfra", + context_window: 327_680, + max_output_tokens: 327_680, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek + ModelDef { + id: "deepseek-ai/DeepSeek-V3.1", + provider_id: "deepinfra", + context_window: 163_840, + max_output_tokens: 163_840, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3-0324", + provider_id: "deepinfra", + context_window: 163_840, + max_output_tokens: 163_840, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-0528", + provider_id: "deepinfra", + context_window: 163_840, + max_output_tokens: 163_840, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + provider_id: "deepinfra", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Qwen + ModelDef { + id: "Qwen/Qwen3-235B-A22B-Instruct-2507", + provider_id: "deepinfra", + context_window: 262_144, + max_output_tokens: 262_144, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-Coder-480B-A35B-Instruct", + provider_id: "deepinfra", + context_window: 262_144, + max_output_tokens: 262_144, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-72B-Instruct", + provider_id: "deepinfra", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-VL-32B-Instruct", + provider_id: "deepinfra", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/QwQ-32B", + provider_id: "deepinfra", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Mistral + ModelDef { + id: "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + provider_id: "deepinfra", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/Mixtral-8x7B-Instruct-v0.1", + provider_id: "deepinfra", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Google Gemma + ModelDef { + id: "google/gemma-3-27b-it", + provider_id: "deepinfra", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Microsoft Phi + ModelDef { + id: "microsoft/phi-4", + provider_id: "deepinfra", + context_window: 16_384, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embeddings + ModelDef { + id: "BAAI/bge-large-en-v1.5", + provider_id: "deepinfra", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "BAAI/bge-m3", + provider_id: "deepinfra", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-Embedding-8B", + provider_id: "deepinfra", + context_window: 32_768, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "intfloat/multilingual-e5-large", + provider_id: "deepinfra", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/deepseek.rs b/crates/providers/src/providers/deepseek.rs index f27d737..882b44d 100644 --- a/crates/providers/src/providers/deepseek.rs +++ b/crates/providers/src/providers/deepseek.rs @@ -23,11 +23,13 @@ pub const PROVIDER: ProviderDef = ProviderDef { }; pub const MODELS: &[ModelDef] = &[ + // deepseek-chat: non-thinking mode of DeepSeek-V3.2. Supports tool calls, + // JSON output, and FIM completion (beta). 128K context; 8K max output. ModelDef { id: "deepseek-chat", provider_id: "deepseek", - context_window: 64_000, - max_output_tokens: 8_000, + context_window: 128_000, + max_output_tokens: 8_192, capabilities: ModelCapabilities { streaming: true, tool_use: true, @@ -36,10 +38,13 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // deepseek-reasoner: thinking mode of DeepSeek-V3.2. Supports JSON output + // and chat prefix completion (beta). Does NOT support function calling or FIM. + // 128K context; 64K max output (default 32K). ModelDef { id: "deepseek-reasoner", provider_id: "deepseek", - context_window: 64_000, + context_window: 128_000, max_output_tokens: 64_000, capabilities: ModelCapabilities { streaming: true, diff --git a/crates/providers/src/providers/docker_model_runner.rs b/crates/providers/src/providers/docker_model_runner.rs index 00e4595..47d24f5 100644 --- a/crates/providers/src/providers/docker_model_runner.rs +++ b/crates/providers/src/providers/docker_model_runner.rs @@ -3,11 +3,16 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Docker Model Runner — Docker-native local model serving (no auth). +/// Docker Model Runner — Docker Desktop's local OpenAI-compatible model server (no auth). +/// +/// Exposes an OpenAI-compatible API on `http://localhost:12434/engines/v1` by default. +/// Authentication is not required: DMR ignores the `Authorization` header. +/// Models are user-pulled from Docker Hub / OCI registries / Hugging Face, so no +/// fixed catalog ships here. pub const PROVIDER: ProviderDef = ProviderDef { id: "docker_model_runner", display_name: "Docker Model Runner", - default_base_url: "http://localhost:12434/engines/llama.cpp/v1", + default_base_url: "http://localhost:12434/engines/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::None, status: ProviderStatus::Stub, @@ -16,11 +21,15 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: false, - vision: false, + // Tool use supported with llama.cpp backend for compatible models. + tool_use: true, + // `/engines/v1/embeddings` is a first-class endpoint. + embeddings: true, + // Vision supported for multi-modal models (e.g. LLaVA). + vision: true, batch: false, }, }; +// Models are pulled locally by the user (OCI / Hugging Face); no fixed catalog. pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/elevenlabs.rs b/crates/providers/src/providers/elevenlabs.rs index fe661b2..3252a23 100644 --- a/crates/providers/src/providers/elevenlabs.rs +++ b/crates/providers/src/providers/elevenlabs.rs @@ -3,11 +3,22 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// ElevenLabs — text-to-speech and voice AI. Chat completions not supported. +/// ElevenLabs — text-to-speech, speech-to-text, and voice AI. +/// +/// Base URL: `https://api.elevenlabs.io/v1` (TTS `/text-to-speech/{voice_id}`, +/// STT `/speech-to-text`, STS `/speech-to-speech/{voice_id}`, etc.). +/// +/// Auth: `xi-api-key: ` HTTP header — NOT `Authorization: Bearer`. +/// `AuthKind::Bearer` below is wrong; the `AuthKind` enum has no `XiApiKey` +/// variant yet. Adding one (and plumbing it through the HTTP clients) is out +/// of scope for this metadata update. Flag when wiring a real client. +/// +/// Chat completions are not supported. Streaming is available on most TTS +/// endpoints (chunked audio + websocket `/v1/text-to-speech/{voice_id}/stream`). pub const PROVIDER: ProviderDef = ProviderDef { id: "elevenlabs", display_name: "ElevenLabs", - default_base_url: "https://api.elevenlabs.io", + default_base_url: "https://api.elevenlabs.io/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Wired, @@ -15,7 +26,7 @@ pub const PROVIDER: ProviderDef = ProviderDef { litellm_prefix: "", capabilities: ProviderCapabilities { chat_completions: false, - streaming: false, + streaming: true, tool_use: false, embeddings: false, vision: false, @@ -24,6 +35,20 @@ pub const PROVIDER: ProviderDef = ProviderDef { }; pub const MODELS: &[ModelDef] = &[ + // TTS — current generation + ModelDef { + id: "eleven_v3", + provider_id: "elevenlabs", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ModelDef { id: "eleven_multilingual_v2", provider_id: "elevenlabs", @@ -38,7 +63,20 @@ pub const MODELS: &[ModelDef] = &[ status: ModelStatus::Available, }, ModelDef { - id: "eleven_turbo_v2", + id: "eleven_flash_v2_5", + provider_id: "elevenlabs", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "eleven_flash_v2", provider_id: "elevenlabs", context_window: 0, max_output_tokens: 0, @@ -64,7 +102,7 @@ pub const MODELS: &[ModelDef] = &[ status: ModelStatus::Available, }, ModelDef { - id: "eleven_monolingual_v1", + id: "eleven_turbo_v2", provider_id: "elevenlabs", context_window: 0, max_output_tokens: 0, @@ -76,6 +114,20 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // TTS — legacy + ModelDef { + id: "eleven_monolingual_v1", + provider_id: "elevenlabs", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Deprecated, + }, ModelDef { id: "eleven_multilingual_v1", provider_id: "elevenlabs", @@ -87,8 +139,9 @@ pub const MODELS: &[ModelDef] = &[ vision: false, extended_thinking: false, }, - status: ModelStatus::Available, + status: ModelStatus::Deprecated, }, + // Speech-to-speech (voice conversion) ModelDef { id: "eleven_multilingual_sts_v2", provider_id: "elevenlabs", @@ -102,4 +155,31 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "eleven_english_sts_v2", + provider_id: "elevenlabs", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Speech-to-text + ModelDef { + id: "scribe_v1", + provider_id: "elevenlabs", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ]; diff --git a/crates/providers/src/providers/exa.rs b/crates/providers/src/providers/exa.rs index abb07eb..6d7e397 100644 --- a/crates/providers/src/providers/exa.rs +++ b/crates/providers/src/providers/exa.rs @@ -3,12 +3,17 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Exa — semantic search API for AI applications. Chat completions not supported. +/// Exa — neural web search API for AI applications. Not a chat provider. +/// Endpoints: POST /search, POST /contents, POST /findSimilar, POST /answer. +/// /answer is a one-shot Q&A endpoint (not conversational), so chat_completions stays false. pub const PROVIDER: ProviderDef = ProviderDef { id: "exa", display_name: "Exa", + // Endpoints live under the root: /search, /contents, /findSimilar, /answer. default_base_url: "https://api.exa.ai", protocol: ProviderProtocol::OpenAICompat, + // NOTE: Exa uses `x-api-key: `, not `Authorization: Bearer`. + // AuthKind has no dedicated variant for a custom header; leaving as Bearer per scope. auth: AuthKind::Bearer, status: ProviderStatus::Stub, env_vars: &["EXA_API_KEY"], @@ -23,5 +28,5 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -// No model selection — single endpoint API +// Search API — no model selection. Endpoint is chosen by path, not by model id. pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/featherless.rs b/crates/providers/src/providers/featherless.rs index ea41db6..a1ce119 100644 --- a/crates/providers/src/providers/featherless.rs +++ b/crates/providers/src/providers/featherless.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -22,4 +22,178 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Featherless hosts 32k+ HuggingFace models behind an OpenAI-compatible API. +// Model IDs use the HuggingFace `org/name` path. The entries below are a +// representative subset of widely-used GA models; the full catalog is +// discoverable at GET /v1/models. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "meta-llama/Meta-Llama-3.1-8B-Instruct", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-70B-Instruct", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.2-3B-Instruct", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-7B-Instruct", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-72B-Instruct", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-8B", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/Mistral-7B-Instruct-v0.3", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/Mixtral-8x7B-Instruct-v0.1", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/Mistral-Nemo-Instruct-2407", + provider_id: "featherless_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/fireworks.rs b/crates/providers/src/providers/fireworks.rs index e0ed0e7..a149681 100644 --- a/crates/providers/src/providers/fireworks.rs +++ b/crates/providers/src/providers/fireworks.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -22,4 +22,126 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Serverless GA models on Fireworks AI. IDs are the full +// `accounts/fireworks/models/` paths used directly in the `model` field +// of Chat Completions requests. Only publicly documented GA serverless +// deployments are listed; on-demand / dedicated deployments are omitted. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "accounts/fireworks/models/llama-v3p3-70b-instruct", + provider_id: "fireworks_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/llama-v3p1-405b-instruct", + provider_id: "fireworks_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/llama-v3p1-8b-instruct", + provider_id: "fireworks_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/deepseek-v3", + provider_id: "fireworks_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/deepseek-r1", + provider_id: "fireworks_ai", + context_window: 163_840, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/qwen2p5-72b-instruct", + provider_id: "fireworks_ai", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/qwen2p5-coder-32b-instruct", + provider_id: "fireworks_ai", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/mixtral-8x7b-instruct", + provider_id: "fireworks_ai", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "accounts/fireworks/models/mixtral-8x22b-instruct", + provider_id: "fireworks_ai", + context_window: 65_536, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/friendliai.rs b/crates/providers/src/providers/friendliai.rs index a820e39..ee5e45e 100644 --- a/crates/providers/src/providers/friendliai.rs +++ b/crates/providers/src/providers/friendliai.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -10,7 +10,9 @@ pub const PROVIDER: ProviderDef = ProviderDef { protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["FRIENDLIAI_TOKEN"], + // Official token env var per Friendli docs is FRIENDLI_TOKEN; keep the + // FRIENDLIAI_* alias for LiteLLM-style compatibility. + env_vars: &["FRIENDLI_TOKEN", "FRIENDLIAI_TOKEN", "FRIENDLIAI_API_KEY"], litellm_prefix: "friendliai/", capabilities: ProviderCapabilities { chat_completions: true, @@ -22,4 +24,86 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA serverless models listed on Friendli's pricing page. IDs match the +// native Friendli format (vendor/Model-Name). OpenAI-compat callers may +// also use the short dash form (e.g. meta-llama-3.3-70b-instruct). +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "friendliai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.1-8B-Instruct", + provider_id: "friendliai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-235B-A22B-Instruct-2507", + provider_id: "friendliai", + context_window: 262_144, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-30B-A3B", + provider_id: "friendliai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3.1", + provider_id: "friendliai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3.2", + provider_id: "friendliai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/galadriel.rs b/crates/providers/src/providers/galadriel.rs index e3ce7c9..b7fc9fa 100644 --- a/crates/providers/src/providers/galadriel.rs +++ b/crates/providers/src/providers/galadriel.rs @@ -16,7 +16,7 @@ pub const PROVIDER: ProviderDef = ProviderDef { chat_completions: true, streaming: true, tool_use: false, - embeddings: false, + embeddings: true, vision: false, batch: false, }, diff --git a/crates/providers/src/providers/gemini.rs b/crates/providers/src/providers/gemini.rs index 75a2de2..12cc7de 100644 --- a/crates/providers/src/providers/gemini.rs +++ b/crates/providers/src/providers/gemini.rs @@ -49,6 +49,19 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "gemini-2.5-flash-lite", + provider_id: "gemini", + context_window: 1_048_576, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, ModelDef { id: "gemini-2.0-flash", provider_id: "gemini", @@ -114,4 +127,19 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // Embedding model. max_output_tokens is not meaningful; set to 0 to match + // the `text-embedding-3-*` convention in openai.rs. + ModelDef { + id: "gemini-embedding-001", + provider_id: "gemini", + context_window: 2_048, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ]; diff --git a/crates/providers/src/providers/github.rs b/crates/providers/src/providers/github.rs index 5aa8d5e..8e9dc82 100644 --- a/crates/providers/src/providers/github.rs +++ b/crates/providers/src/providers/github.rs @@ -1,13 +1,18 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// GitHub Models — Azure-hosted OpenAI-compatible endpoint using a GitHub token. +/// GitHub Models — Azure AI-backed inference marketplace accessed with a GitHub +/// token (PAT or fine-grained token with the `models:read` scope). The API is +/// OpenAI-compatible and served under `/inference/chat/completions`. pub const PROVIDER: ProviderDef = ProviderDef { id: "github", display_name: "GitHub Models", - default_base_url: "https://models.inference.ai.azure.com", + // Modern endpoint used by the REST API (see docs.github.com/en/rest/models). + // The older `models.inference.ai.azure.com` host still resolves but is being + // superseded by `models.github.ai`. + default_base_url: "https://models.github.ai/inference", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -23,4 +28,459 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model IDs use the `publisher/model-name` form the GitHub Models API expects +// in the request body. Context windows reflect the publisher's published +// limits; GitHub Models may impose lower per-request caps via its rate-limit +// tiers (Low / High / Embedding). Only GA catalog entries are listed here. +pub const MODELS: &[ModelDef] = &[ + // -------- OpenAI -------- + ModelDef { + id: "openai/gpt-4o", + provider_id: "github", + context_window: 128_000, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-4o-mini", + provider_id: "github", + context_window: 128_000, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-4.1", + provider_id: "github", + context_window: 1_047_576, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-4.1-mini", + provider_id: "github", + context_window: 1_047_576, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-4.1-nano", + provider_id: "github", + context_window: 1_047_576, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/o1", + provider_id: "github", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/o1-mini", + provider_id: "github", + context_window: 128_000, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/o3", + provider_id: "github", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/o3-mini", + provider_id: "github", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/o4-mini", + provider_id: "github", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/text-embedding-3-large", + provider_id: "github", + context_window: 8_191, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/text-embedding-3-small", + provider_id: "github", + context_window: 8_191, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -------- Microsoft (Phi) -------- + ModelDef { + id: "microsoft/phi-4", + provider_id: "github", + context_window: 16_384, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "microsoft/phi-4-mini-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "microsoft/phi-4-multimodal-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "microsoft/phi-4-reasoning", + provider_id: "github", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // -------- Meta (Llama) -------- + ModelDef { + id: "meta/meta-llama-3.1-8b-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/meta-llama-3.1-405b-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-3.3-70b-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-3.2-11b-vision-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-3.2-90b-vision-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-4-scout-17b-16e-instruct", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-4-maverick-17b-128e-instruct-fp8", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -------- Mistral AI -------- + ModelDef { + id: "mistral-ai/mistral-medium-2505", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistral-ai/mistral-small-2503", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistral-ai/ministral-3b", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistral-ai/codestral-2501", + provider_id: "github", + context_window: 262_144, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -------- xAI (Grok) -------- + ModelDef { + id: "xai/grok-3", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "xai/grok-3-mini", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // -------- DeepSeek -------- + ModelDef { + id: "deepseek/deepseek-r1", + provider_id: "github", + context_window: 163_840, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek/deepseek-v3-0324", + provider_id: "github", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -------- Cohere -------- + ModelDef { + id: "cohere/cohere-command-a", + provider_id: "github", + context_window: 256_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "cohere/cohere-command-r-plus-08-2024", + provider_id: "github", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // -------- AI21 Labs -------- + ModelDef { + id: "ai21-labs/ai21-jamba-1.5-large", + provider_id: "github", + context_window: 256_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/gmi_cloud.rs b/crates/providers/src/providers/gmi_cloud.rs index d0f2ca8..6d01c28 100644 --- a/crates/providers/src/providers/gmi_cloud.rs +++ b/crates/providers/src/providers/gmi_cloud.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,7 +6,8 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "gmi_cloud", display_name: "GMI Cloud", - default_base_url: "https://api.gmi.ai/v1", + // OpenAI-compatible inference API. Docs: https://docs.gmicloud.ai/ + default_base_url: "https://api.gmi-serving.com/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -22,4 +23,112 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Core GA serverless LLMs on GMI Cloud's Model-as-a-Service catalog. IDs use the +// HuggingFace-style namespace shown in the LLM API reference examples and blog posts. +// Live enumeration available via `GET /v1/models`; refresh as the catalog evolves. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "deepseek-ai/DeepSeek-R1", + provider_id: "gmi_cloud", + context_window: 131_072, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3", + provider_id: "gmi_cloud", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + provider_id: "gmi_cloud", + context_window: 131_072, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + provider_id: "gmi_cloud", + context_window: 131_072, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "gmi_cloud", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.1-8B-Instruct", + provider_id: "gmi_cloud", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", + provider_id: "gmi_cloud", + context_window: 262_144, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-32B-FP8", + provider_id: "gmi_cloud", + context_window: 131_072, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/gradient_ai.rs b/crates/providers/src/providers/gradient_ai.rs index f38b7f2..065ea33 100644 --- a/crates/providers/src/providers/gradient_ai.rs +++ b/crates/providers/src/providers/gradient_ai.rs @@ -1,25 +1,67 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// DigitalOcean Gradient AI Platform (formerly marketed as "Gradient AI"). +// Note: the original Gradient.ai (Boston) was discontinued after acquisition; +// this id now tracks DigitalOcean's Gradient AI serverless inference surface. +// +// Base URL and auth per docs.digitalocean.com/products/gradient-ai-platform: +// POST https://inference.do-ai.run/v1/chat/completions +// Authorization: Bearer $MODEL_ACCESS_KEY +// OpenAI-compatible chat completions with streaming. Tool calling is supported +// on hosted Anthropic/OpenAI/Llama variants; embeddings on this endpoint are +// not publicly documented (embedding models exist only for knowledge-base use). pub const PROVIDER: ProviderDef = ProviderDef { id: "gradient_ai", - display_name: "Gradient AI", - default_base_url: "https://api.gradient.ai", + display_name: "DigitalOcean Gradient AI", + default_base_url: "https://inference.do-ai.run/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["GRADIENT_ACCESS_TOKEN"], + env_vars: &["DIGITALOCEAN_INFERENCE_KEY", "GRADIENT_ACCESS_TOKEN"], litellm_prefix: "gradient_ai/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: true, + tool_use: true, + embeddings: false, vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Only DO-hosted GA serverless models are listed. Anthropic/OpenAI models +// exposed via Gradient agents use the same endpoint but are tracked under +// their own provider ids; context windows for those vary by upstream. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama 3.3 70B Instruct, hosted by DigitalOcean. 128K context. + ModelDef { + id: "llama3.3-70b-instruct", + provider_id: "gradient_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek R1 Distill (Llama 70B base). Reasoning model; no function calling. + ModelDef { + id: "deepseek-r1-distill-llama-70b", + provider_id: "gradient_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/groq.rs b/crates/providers/src/providers/groq.rs index 3ad9db7..83d9e9e 100644 --- a/crates/providers/src/providers/groq.rs +++ b/crates/providers/src/providers/groq.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -22,4 +22,87 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Production (GA) model catalog per https://console.groq.com/docs/models. +// Preview models (Llama 4 Scout/Maverick, Kimi K2, Qwen QwQ, DeepSeek R1 distill, +// Gemma2, Llama Guard) are intentionally excluded. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "llama-3.3-70b-versatile", + provider_id: "groq", + context_window: 131_072, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-3.1-8b-instant", + provider_id: "groq", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-oss-120b", + provider_id: "groq", + context_window: 131_072, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-oss-20b", + provider_id: "groq", + context_window: 131_072, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Whisper speech-to-text. Non-chat endpoint; streaming/tool_use N/A. + ModelDef { + id: "whisper-large-v3", + provider_id: "groq", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "whisper-large-v3-turbo", + provider_id: "groq", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/huggingface.rs b/crates/providers/src/providers/huggingface.rs index 32b31c5..3af0ebf 100644 --- a/crates/providers/src/providers/huggingface.rs +++ b/crates/providers/src/providers/huggingface.rs @@ -1,27 +1,183 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// HuggingFace Inference Endpoints (TGI / serverless inference) via OpenAI-compatible API. -/// Endpoint URL is per-deployment; set `OPENAI_BASE_URL` to override. +/// Hugging Face Inference Providers: unified OpenAI-compatible router that fans out +/// to partner providers (Together, Fireworks, SambaNova, Novita, Groq, Cerebras, +/// Replicate, Hyperbolic, Fal, Nscale, Scaleway, HF Inference, etc.). +/// +/// Chat completions endpoint: POST https://router.huggingface.co/v1/chat/completions +/// Models are addressed by HF model id (e.g. `meta-llama/Llama-3.3-70B-Instruct`), +/// optionally with a provider/policy suffix (`:fastest`, `:cheapest`, `:preferred`, +/// or `:`). Auth is a Bearer HF token (`HF_TOKEN`). +/// +/// Note: the unified OpenAI-compat endpoint covers chat completions only. Other +/// tasks (embeddings, text-to-image, speech) require the HF Inference clients. pub const PROVIDER: ProviderDef = ProviderDef { id: "huggingface", display_name: "HuggingFace", - default_base_url: "", + default_base_url: "https://router.huggingface.co", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["HUGGINGFACE_API_KEY", "HF_TOKEN"], + env_vars: &["HF_TOKEN", "HUGGINGFACE_API_KEY"], litellm_prefix: "huggingface/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: true, - vision: false, + tool_use: true, + embeddings: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Representative popular public models routed via HF Inference Providers. +// Context/output windows reflect the upstream model; the actual limits served +// depend on which partner provider handles the request. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "huggingface", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-8B-Instruct", + provider_id: "huggingface", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-405B-Instruct", + provider_id: "huggingface", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3", + provider_id: "huggingface", + context_window: 64_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1", + provider_id: "huggingface", + context_window: 64_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-72B-Instruct", + provider_id: "huggingface", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + provider_id: "huggingface", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/QwQ-32B-Preview", + provider_id: "huggingface", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/Mistral-7B-Instruct-v0.3", + provider_id: "huggingface", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/Mixtral-8x7B-Instruct-v0.1", + provider_id: "huggingface", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-oss-120b", + provider_id: "huggingface", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/hyperbolic.rs b/crates/providers/src/providers/hyperbolic.rs index b8220f7..df4e56b 100644 --- a/crates/providers/src/providers/hyperbolic.rs +++ b/crates/providers/src/providers/hyperbolic.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -22,4 +22,195 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model IDs follow Hyperbolic's HuggingFace-style format as documented in +// quickstart examples (e.g. "meta-llama/Meta-Llama-3.1-70B-Instruct"). +// Context windows reflect the underlying model cards. Tool use on Hyperbolic +// is not documented as first-class for most models, so kept conservative. +pub const MODELS: &[ModelDef] = &[ + // DeepSeek + ModelDef { + id: "deepseek-ai/DeepSeek-V3", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3-0324", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-0528", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Meta Llama 3.x + ModelDef { + id: "meta-llama/Meta-Llama-3.1-405B-Instruct", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-70B-Instruct", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-8B-Instruct", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.2-3B-Instruct", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3-70B-Instruct", + provider_id: "hyperbolic", + context_window: 8_192, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Qwen + ModelDef { + id: "Qwen/Qwen2.5-72B-Instruct", + provider_id: "hyperbolic", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + provider_id: "hyperbolic", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/QwQ-32B", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Nous Hermes + ModelDef { + id: "NousResearch/Hermes-3-Llama-3.1-70B", + provider_id: "hyperbolic", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/iflytek.rs b/crates/providers/src/providers/iflytek.rs index 25da8fc..247b4c6 100644 --- a/crates/providers/src/providers/iflytek.rs +++ b/crates/providers/src/providers/iflytek.rs @@ -3,7 +3,11 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// iFlytek Spark — Chinese LLM with an OpenAI-compatible endpoint. +/// iFlytek Spark (讯飞星火) — Chinese LLM with an OpenAI-compatible HTTP endpoint. +/// +/// OpenAI-compatible base: `https://spark-api-open.xf-yun.com/v1` (HTTP, Bearer APIPassword). +/// A separate native WebSocket API exists at `wss://spark-api.xf-yun.com/...` using +/// HMAC-SHA256 request signing; this metadata targets the HTTP/OpenAI-compat surface only. pub const PROVIDER: ProviderDef = ProviderDef { id: "iflytek", display_name: "iFlytek Spark", @@ -18,46 +22,33 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: false, - vision: true, + // Spark text models do not document vision on the HTTP OpenAI-compat surface. + vision: false, batch: false, }, }; pub const MODELS: &[ModelDef] = &[ - // Spark 4.0 Ultra — flagship, 128k context + // Spark 4.0 Ultra — flagship. 32k context / 32k output per official HTTP docs. ModelDef { id: "4.0Ultra", provider_id: "iflytek", - context_window: 128_000, - max_output_tokens: 8_192, + context_window: 32_768, + max_output_tokens: 32_768, capabilities: ModelCapabilities { streaming: true, tool_use: true, - vision: true, + vision: false, extended_thinking: false, }, status: ModelStatus::Available, }, - // Spark Max (generalv3.5) + // Spark Max (generalv3.5) — 8k context, supports function calling and system prompts. ModelDef { id: "generalv3.5", provider_id: "iflytek", context_window: 8_192, max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: true, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // Spark Pro (generalv3) - ModelDef { - id: "generalv3", - provider_id: "iflytek", - context_window: 8_192, - max_output_tokens: 8_192, capabilities: ModelCapabilities { streaming: true, tool_use: true, @@ -66,11 +57,25 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - // Spark V2 (general) — legacy + // Spark Max-32K — extended-context variant of Max. ModelDef { - id: "general", + id: "max-32k", provider_id: "iflytek", - context_window: 4_096, + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Spark Pro (generalv3) — 8k context. Tool use not offered on Pro per HTTP docs. + ModelDef { + id: "generalv3", + provider_id: "iflytek", + context_window: 8_192, max_output_tokens: 4_096, capabilities: ModelCapabilities { streaming: true, @@ -80,11 +85,25 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - // Spark Lite — fastest, lowest cost + // Spark Pro-128K — long-context Pro variant. + ModelDef { + id: "pro-128k", + provider_id: "iflytek", + context_window: 131_072, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Spark Lite — fastest, lowest cost. 8k context / 4k output. ModelDef { id: "lite", provider_id: "iflytek", - context_window: 4_096, + context_window: 8_192, max_output_tokens: 4_096, capabilities: ModelCapabilities { streaming: true, diff --git a/crates/providers/src/providers/infinity.rs b/crates/providers/src/providers/infinity.rs index 1bfcd5e..8e613de 100644 --- a/crates/providers/src/providers/infinity.rs +++ b/crates/providers/src/providers/infinity.rs @@ -3,11 +3,13 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Infinity — self-hosted embedding server (local, no auth). +/// Infinity — self-hosted OpenAI-compatible embeddings & reranking server +/// (michaelfeil/infinity). No auth by default; optional `INFINITY_API_KEY`. +/// Models are user-served, so the catalog is empty. pub const PROVIDER: ProviderDef = ProviderDef { id: "infinity", display_name: "Infinity", - default_base_url: "http://localhost:7997", + default_base_url: "http://localhost:7997/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::None, status: ProviderStatus::Stub, diff --git a/crates/providers/src/providers/jina.rs b/crates/providers/src/providers/jina.rs index 073fd16..f35ea46 100644 --- a/crates/providers/src/providers/jina.rs +++ b/crates/providers/src/providers/jina.rs @@ -1,9 +1,13 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Jina AI — embeddings and reranking provider. +/// Jina AI — embeddings, reranking, reader, classifier, and segmenter APIs. +/// +/// Not a chat LLM. OpenAI-compatible `/v1/embeddings` endpoint plus +/// Jina-specific `/v1/rerank`, `/v1/classify`, `/v1/segment`, and reader +/// endpoints. All routes share the `api.jina.ai` host and Bearer auth. pub const PROVIDER: ProviderDef = ProviderDef { id: "jina", display_name: "Jina AI", @@ -23,4 +27,74 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Context windows sourced from jina.ai/embeddings and jina.ai/reranker (GA models). +// `max_output_tokens` is not meaningful for embeddings/reranking; set to 0. +pub const MODELS: &[ModelDef] = &[ + // Embeddings + ModelDef { + id: "jina-embeddings-v4", + provider_id: "jina", + context_window: 32_768, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "jina-embeddings-v3", + provider_id: "jina", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "jina-clip-v2", + provider_id: "jina", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Rerankers + ModelDef { + id: "jina-reranker-v2-base-multilingual", + provider_id: "jina", + context_window: 1_024, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "jina-colbert-v2", + provider_id: "jina", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/lambda.rs b/crates/providers/src/providers/lambda.rs index 3375e2e..b821a3f 100644 --- a/crates/providers/src/providers/lambda.rs +++ b/crates/providers/src/providers/lambda.rs @@ -1,12 +1,17 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Lambda Inference API (lambda.ai, formerly lambdalabs.com). +// OpenAI-compatible surface at https://api.lambda.ai/v1. Auth is a Bearer +// token taken from the LAMBDA_API_KEY env var. Note: as of 2025 Lambda has +// publicly stated the Inference API is "winding down" in favor of dedicated +// GPU deployments, so the GA catalog may continue to shrink. pub const PROVIDER: ProviderDef = ProviderDef { id: "lambda_ai", display_name: "Lambda AI", - default_base_url: "https://api.lambdalabs.com/v1", + default_base_url: "https://api.lambda.ai/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -17,9 +22,261 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: false, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model IDs mirror Lambda's OpenAI-compatible catalog (no provider prefix on +// the wire). Context windows use the upstream model-card values; tool_use +// reflects whether the base model supports function calling on Lambda's +// endpoint. Vision is only set for the Llama 3.2 vision variant. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "llama3.1-8b-instruct", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama3.1-70b-instruct-fp8", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama3.1-405b-instruct-fp8", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama3.1-nemotron-70b-instruct-fp8", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama3.2-11b-vision-instruct", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama3.3-70b-instruct-fp8", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-4-scout-17b-16e-instruct", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-4-maverick-17b-128e-instruct-fp8", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "hermes3-8b", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "hermes3-70b", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "hermes3-405b", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen25-coder-32b-instruct", + provider_id: "lambda_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen3-32b-fp8", + provider_id: "lambda_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-v3-0324", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-r1-0528", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-r1-671b", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-llama3.3-70b", + provider_id: "lambda_ai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "lfm-7b", + provider_id: "lambda_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "lfm-40b", + provider_id: "lambda_ai", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/lemonade.rs b/crates/providers/src/providers/lemonade.rs index 9be3b01..a7bd9a7 100644 --- a/crates/providers/src/providers/lemonade.rs +++ b/crates/providers/src/providers/lemonade.rs @@ -3,11 +3,12 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Lemonade — local LLM server (no auth). +/// Lemonade — AMD's local LLM server with an OpenAI-compatible API. +/// Default port 13305; users load their own models via `/api/v1/pull` + `/api/v1/load`. pub const PROVIDER: ProviderDef = ProviderDef { id: "lemonade", display_name: "Lemonade", - default_base_url: "http://localhost:8000", + default_base_url: "http://localhost:13305/api/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::None, status: ProviderStatus::Stub, @@ -16,9 +17,9 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: false, - vision: false, + tool_use: true, + embeddings: true, + vision: true, batch: false, }, }; diff --git a/crates/providers/src/providers/llamafile.rs b/crates/providers/src/providers/llamafile.rs index 3306a01..cc63a4b 100644 --- a/crates/providers/src/providers/llamafile.rs +++ b/crates/providers/src/providers/llamafile.rs @@ -3,10 +3,21 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +/// llamafile — Mozilla Ocho / Justine Tunney single-file LLM distribution. +/// +/// Packages a llama.cpp server plus a model weights file into one Cosmopolitan +/// Libc executable that runs across OS/CPU combinations. When launched it hosts +/// llama.cpp's HTTP server, which exposes OpenAI-compatible endpoints at +/// `http://localhost:8080/v1` (`/v1/chat/completions`, `/v1/completions`, +/// `/v1/embeddings`). No authentication is required by default; the +/// `Authorization` header is ignored unless `--api-key` is passed at launch. +/// +/// Since each llamafile bundles a single model, no fixed catalog ships here; +/// the `model` field in requests is effectively ignored by the server. pub const PROVIDER: ProviderDef = ProviderDef { id: "llamafile", display_name: "llamafile", - default_base_url: "http://localhost:8080", + default_base_url: "http://localhost:8080/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::None, status: ProviderStatus::Stub, @@ -15,11 +26,18 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: false, - vision: false, + // llama.cpp server supports function/tool calling for compatible + // instruction-tuned models (grammar-constrained JSON output). + tool_use: true, + // `/v1/embeddings` available when the bundled model supports it + // (or when launched with `--embedding`). + embeddings: true, + // Vision works with multimodal llamafiles (LLaVA family) that ship + // an mmproj file; text-only llamafiles reject image inputs. + vision: true, batch: false, }, }; +// Single-file bundle: the model is baked into the executable, no catalog. pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/meta_llama.rs b/crates/providers/src/providers/meta_llama.rs index df1059b..8306ada 100644 --- a/crates/providers/src/providers/meta_llama.rs +++ b/crates/providers/src/providers/meta_llama.rs @@ -1,16 +1,20 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Meta's official hosted Llama API. OpenAI-compatible surface is served at +// /compat/v1 (native REST is at /v1). The compat endpoint supports chat +// completions, tools/function calling, and json_schema response_format. +// Docs: https://llama.developer.meta.com/docs pub const PROVIDER: ProviderDef = ProviderDef { id: "meta_llama", display_name: "Meta Llama API", - default_base_url: "https://www.llama.com/api/v1", + default_base_url: "https://api.llama.com/compat/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["META_LLAMA_API_KEY"], + env_vars: &["LLAMA_API_KEY"], litellm_prefix: "meta_llama/", capabilities: ProviderCapabilities { chat_completions: true, @@ -22,4 +26,61 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA catalog on the hosted Llama API. Max output tokens is 4028 across the +// current catalog per LiteLLM's price table; context windows match Meta's +// published limits (Scout 10M, Maverick 1M, Llama 3.3 128k). Llama 4 models +// are natively multimodal; Llama 3.3 variants are text-only. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "Llama-4-Maverick-17B-128E-Instruct-FP8", + provider_id: "meta_llama", + context_window: 1_000_000, + max_output_tokens: 4_028, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Llama-4-Scout-17B-16E-Instruct-FP8", + provider_id: "meta_llama", + context_window: 10_000_000, + max_output_tokens: 4_028, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Llama-3.3-70B-Instruct", + provider_id: "meta_llama", + context_window: 128_000, + max_output_tokens: 4_028, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Llama-3.3-8B-Instruct", + provider_id: "meta_llama", + context_window: 128_000, + max_output_tokens: 4_028, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/minimax.rs b/crates/providers/src/providers/minimax.rs index e2347b0..cd42699 100644 --- a/crates/providers/src/providers/minimax.rs +++ b/crates/providers/src/providers/minimax.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,7 +6,9 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "minimax", display_name: "MiniMax", - default_base_url: "https://api.minimax.chat/v1", + // International OpenAI-compatible endpoint. The China region uses + // https://api.minimaxi.chat/v1 with the same schema. + default_base_url: "https://api.minimax.io/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -17,9 +19,105 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: false, - vision: true, + // MiniMax OpenAI-compat chat API explicitly does not accept image or + // audio inputs as of the M2.x lineup. + vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA text models on the international platform. All advertise a 204,800-token +// context window and up to 128k max output (including chain-of-thought). +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "MiniMax-M2.7", + provider_id: "minimax", + context_window: 204_800, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMax-M2.7-highspeed", + provider_id: "minimax", + context_window: 204_800, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMax-M2.5", + provider_id: "minimax", + context_window: 204_800, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMax-M2.5-highspeed", + provider_id: "minimax", + context_window: 204_800, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMax-M2.1", + provider_id: "minimax", + context_window: 204_800, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMax-M2.1-highspeed", + provider_id: "minimax", + context_window: 204_800, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "MiniMax-M2", + provider_id: "minimax", + context_window: 204_800, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/mistral.rs b/crates/providers/src/providers/mistral.rs index ee6a914..f01327a 100644 --- a/crates/providers/src/providers/mistral.rs +++ b/crates/providers/src/providers/mistral.rs @@ -22,7 +22,13 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// Models reflect Mistral "La Plateforme" GA offerings as of 2025. +// Context windows come from Mistral's public model pages; max_output values +// mirror the conservative defaults Mistral documents for chat completions. +// Deprecated aliases (e.g. codestral-2405, mistral-tiny, open-mixtral-*) are +// intentionally excluded. pub const MODELS: &[ModelDef] = &[ + // Frontier generalist ModelDef { id: "mistral-large-latest", provider_id: "mistral", @@ -36,6 +42,19 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "mistral-medium-latest", + provider_id: "mistral", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ModelDef { id: "mistral-small-latest", provider_id: "mistral", @@ -49,19 +68,34 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // Edge / small-footprint ModelDef { - id: "codestral-latest", + id: "ministral-8b-latest", provider_id: "mistral", - context_window: 256_000, - max_output_tokens: 8_192, + context_window: 131_072, + max_output_tokens: 4_096, capabilities: ModelCapabilities { streaming: true, - tool_use: false, + tool_use: true, vision: false, extended_thinking: false, }, status: ModelStatus::Available, }, + ModelDef { + id: "ministral-3b-latest", + provider_id: "mistral", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Vision (Pixtral) ModelDef { id: "pixtral-large-latest", provider_id: "mistral", @@ -75,4 +109,127 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + ModelDef { + id: "pixtral-12b-2409", + provider_id: "mistral", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Code + ModelDef { + id: "codestral-latest", + provider_id: "mistral", + context_window: 256_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Regional (Middle East / South Asia focus) + ModelDef { + id: "mistral-saba-latest", + provider_id: "mistral", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Reasoning (Magistral) + ModelDef { + id: "magistral-medium-latest", + provider_id: "mistral", + context_window: 40_000, + max_output_tokens: 40_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "magistral-small-latest", + provider_id: "mistral", + context_window: 40_000, + max_output_tokens: 40_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Audio (Voxtral) — accept audio input, produce text + ModelDef { + id: "voxtral-small-latest", + provider_id: "mistral", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voxtral-mini-2507", + provider_id: "mistral", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embeddings + ModelDef { + id: "mistral-embed", + provider_id: "mistral", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Moderation + ModelDef { + id: "mistral-moderation-latest", + provider_id: "mistral", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ]; diff --git a/crates/providers/src/providers/moonshot.rs b/crates/providers/src/providers/moonshot.rs index 2c82d98..c2cbcbe 100644 --- a/crates/providers/src/providers/moonshot.rs +++ b/crates/providers/src/providers/moonshot.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,7 +6,9 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "moonshot", display_name: "Moonshot AI", - default_base_url: "https://api.moonshot.cn/v1", + // International endpoint. China-region users should override to + // https://api.moonshot.cn/v1 via config. + default_base_url: "https://api.moonshot.ai/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -17,9 +19,84 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: false, + // kimi-k2.5 is natively multimodal (text + image) and GA. vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +pub const MODELS: &[ModelDef] = &[ + // Kimi K2.5: GA trillion-parameter flagship with 256K context and native + // multimodal (text + image) input. Supports tool calling and streaming. + ModelDef { + id: "kimi-k2.5", + provider_id: "moonshot", + context_window: 256_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Moonshot V1 series: GA text-only chat models. Tool calling and streaming + // supported. Vision is a separate `-vision-preview` variant (not GA, omitted). + // Moonshot does not publish a fixed max_output cap; 4096 is the documented + // default ceiling for `max_tokens` across the v1 family. + ModelDef { + id: "moonshot-v1-8k", + provider_id: "moonshot", + context_window: 8_192, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "moonshot-v1-32k", + provider_id: "moonshot", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "moonshot-v1-128k", + provider_id: "moonshot", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // moonshot-v1-auto: server-side router that picks the smallest v1 variant + // that fits the prompt. Context reported as the largest backing model (128K). + ModelDef { + id: "moonshot-v1-auto", + provider_id: "moonshot", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/morph.rs b/crates/providers/src/providers/morph.rs index 2e5caf0..bdbe412 100644 --- a/crates/providers/src/providers/morph.rs +++ b/crates/providers/src/providers/morph.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -16,10 +16,95 @@ pub const PROVIDER: ProviderDef = ProviderDef { chat_completions: true, streaming: true, tool_use: true, - embeddings: false, + embeddings: true, vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA model catalog per https://docs.morphllm.com/llms.txt and /models/*. +// Morph specializes in fast code-edit "apply" models plus embeddings and rerank. +// Context windows / max output tokens are not published; left as 0 where unknown. +pub const MODELS: &[ModelDef] = &[ + // Apply models (code-edit). OpenAI-compatible chat/completions endpoint. + ModelDef { + id: "morph-v3-fast", + provider_id: "morph", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "morph-v3-large", + provider_id: "morph", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "auto", + provider_id: "morph", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embedding models. Non-chat endpoint; streaming/tool_use N/A. + ModelDef { + id: "morph-embedding-v3", + provider_id: "morph", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "morph-embedding-v2", + provider_id: "morph", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Rerank model. Non-chat endpoint. + ModelDef { + id: "morph-rerank-v3", + provider_id: "morph", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/nanogpt.rs b/crates/providers/src/providers/nanogpt.rs index 4f6e5f5..14452df 100644 --- a/crates/providers/src/providers/nanogpt.rs +++ b/crates/providers/src/providers/nanogpt.rs @@ -1,8 +1,14 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// NanoGPT is a pay-as-you-go aggregator that fronts 700+ models from OpenAI, +// Anthropic, Google, Meta, DeepSeek, Qwen, Mistral and others behind a single +// OpenAI-compatible endpoint at https://nano-gpt.com/api/v1/chat/completions. +// Auth is a Bearer token (also accepts `x-api-key`). The catalog below is a +// representative slice of popular GA models; the full list is discoverable at +// GET /api/v1/models. pub const PROVIDER: ProviderDef = ProviderDef { id: "nanogpt", display_name: "NanoGPT", @@ -15,11 +21,148 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: false, - vision: false, + tool_use: true, + embeddings: true, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +pub const MODELS: &[ModelDef] = &[ + // OpenAI (proxied via NanoGPT) + ModelDef { + id: "gpt-4o", + provider_id: "nanogpt", + context_window: 128_000, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-4o-mini", + provider_id: "nanogpt", + context_window: 128_000, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "o1", + provider_id: "nanogpt", + context_window: 200_000, + max_output_tokens: 100_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Anthropic (proxied) + ModelDef { + id: "claude-3-5-sonnet-20241022", + provider_id: "nanogpt", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "claude-3-5-haiku-20241022", + provider_id: "nanogpt", + context_window: 200_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Google Gemini (proxied) + ModelDef { + id: "gemini-2.5-pro", + provider_id: "nanogpt", + context_window: 1_048_576, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gemini-2.5-flash", + provider_id: "nanogpt", + context_window: 1_048_576, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Meta Llama + ModelDef { + id: "meta-llama/llama-3.3-70b-instruct", + provider_id: "nanogpt", + context_window: 131_072, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek + ModelDef { + id: "deepseek-v3", + provider_id: "nanogpt", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Qwen + ModelDef { + id: "qwen3-235b-a22b", + provider_id: "nanogpt", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/nebius.rs b/crates/providers/src/providers/nebius.rs index da5c6b7..bc07e4e 100644 --- a/crates/providers/src/providers/nebius.rs +++ b/crates/providers/src/providers/nebius.rs @@ -1,12 +1,16 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Nebius AI Studio (rebranded to "Nebius Token Factory" in 2025). The public +// OpenAI-compatible endpoint moved from api.studio.nebius.ai to +// api.tokenfactory.nebius.com. Docs: https://docs.tokenfactory.nebius.com/ +// litellm_prefix intentionally kept as "nebius/" for catalog compatibility. pub const PROVIDER: ProviderDef = ProviderDef { id: "nebius", display_name: "Nebius AI Studio", - default_base_url: "https://api.studio.nebius.ai/v1", + default_base_url: "https://api.tokenfactory.nebius.com/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -17,9 +21,412 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: true, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model list sourced from LiteLLM's model_prices_and_context_window.json +// (nebius/* entries) cross-checked against docs.tokenfactory.nebius.com. +// Context/output token figures mirror LiteLLM's published values; some +// providers cap max_output below the full context in practice. +pub const MODELS: &[ModelDef] = &[ + // --- Meta Llama --- + ModelDef { + id: "meta-llama/Meta-Llama-3.1-8B-Instruct", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-70B-Instruct", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Meta-Llama-3.1-405B-Instruct", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-Guard-3-8B", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- DeepSeek --- + ModelDef { + id: "deepseek-ai/DeepSeek-R1", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-0528", + provider_id: "nebius", + context_window: 164_000, + max_output_tokens: 164_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-V3-0324", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Qwen (Alibaba) --- + ModelDef { + id: "Qwen/Qwen3-4B", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-14B", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-30B-A3B", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-32B", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen3-235B-A22B", + provider_id: "nebius", + context_window: 262_144, + max_output_tokens: 262_144, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/QwQ-32B", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-32B-Instruct", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-72B-Instruct", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-7B", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2-VL-7B-Instruct", + provider_id: "nebius", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2-VL-72B-Instruct", + provider_id: "nebius", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-VL-72B-Instruct", + provider_id: "nebius", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Mistral --- + ModelDef { + id: "mistralai/Mistral-Nemo-Instruct-2407", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Google Gemma --- + ModelDef { + id: "google/gemma-3-27b-it", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- NVIDIA Nemotron --- + ModelDef { + id: "nvidia/Llama-3.3-Nemotron-Super-49B-v1", + provider_id: "nebius", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "nvidia/Llama-3.1-Nemotron-Ultra-253B-v1", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- NousResearch Hermes --- + ModelDef { + id: "NousResearch/Hermes-3-Llama-3.1-405B", + provider_id: "nebius", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Embedding models --- + ModelDef { + id: "BAAI/bge-en-icl", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "BAAI/bge-multilingual-gemma2", + provider_id: "nebius", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "intfloat/e5-mistral-7b-instruct", + provider_id: "nebius", + context_window: 32_768, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/nlp_cloud.rs b/crates/providers/src/providers/nlp_cloud.rs index 64cb8b6..d1a2ecd 100644 --- a/crates/providers/src/providers/nlp_cloud.rs +++ b/crates/providers/src/providers/nlp_cloud.rs @@ -1,12 +1,16 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// NLP Cloud uses per-model paths: /v1// and /v1/gpu//. +// Auth header is `Authorization: Token ` (not standard `Bearer`). `AuthKind::Bearer` +// is the closest match in the current enum; a dedicated `Token` variant would be more +// accurate if one is added later. pub const PROVIDER: ProviderDef = ProviderDef { id: "nlp_cloud", display_name: "NLP Cloud", - default_base_url: "https://api.nlpcloud.io", + default_base_url: "https://api.nlpcloud.io/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -14,12 +18,200 @@ pub const PROVIDER: ProviderDef = ProviderDef { litellm_prefix: "nlp_cloud/", capabilities: ProviderCapabilities { chat_completions: true, - streaming: true, + // NLP Cloud's /generation endpoint is request/response; long jobs use async + // polling rather than SSE token streaming. + streaming: false, tool_use: false, - embeddings: false, + embeddings: true, vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Publicly GA generative + embeddings models on NLP Cloud. Context/output values come +// from docs.nlpcloud.com. Max-output for GPT-J / GPT-NeoX variants reflects the +// documented token caps on their respective hardware tiers (GPU where applicable). +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "chatdolphin", + provider_id: "nlp_cloud", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "dolphin", + provider_id: "nlp_cloud", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "dolphin-yi-34b", + provider_id: "nlp_cloud", + context_window: 8_192, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "dolphin-mixtral-8x7b", + provider_id: "nlp_cloud", + context_window: 32_768, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "finetuned-llama-3-70b", + provider_id: "nlp_cloud", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-3-1-405b", + provider_id: "nlp_cloud", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-oss-120b", + provider_id: "nlp_cloud", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "yi-34b", + provider_id: "nlp_cloud", + context_window: 4_096, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mixtral-8x7b", + provider_id: "nlp_cloud", + context_window: 32_768, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "fast-gpt-j", + provider_id: "nlp_cloud", + context_window: 2_048, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-j", + provider_id: "nlp_cloud", + context_window: 2_048, + max_output_tokens: 1_024, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "finetuned-gpt-neox-20b", + provider_id: "nlp_cloud", + context_window: 2_048, + max_output_tokens: 2_048, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-neox-20b", + provider_id: "nlp_cloud", + context_window: 2_048, + max_output_tokens: 1_024, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "paraphrase-multilingual-mpnet-base-v2", + provider_id: "nlp_cloud", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/novita.rs b/crates/providers/src/providers/novita.rs index d6e351c..d397d10 100644 --- a/crates/providers/src/providers/novita.rs +++ b/crates/providers/src/providers/novita.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,7 +6,9 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "novita", display_name: "Novita AI", - default_base_url: "https://api.novita.ai/v3/openai", + // Official OpenAI-compatible endpoint per novita.ai/docs/guides/llm-api. + // Clients append /v1/chat/completions, /v1/models, etc. + default_base_url: "https://api.novita.ai/openai", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -15,11 +17,242 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: false, + // Function/tool calling is supported on many hosted models (Llama 3.x, Qwen, DeepSeek). + tool_use: true, + embeddings: true, + // Vision not documented as GA on novita OpenAI-compat path as of this writing. vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model IDs follow Novita's `vendor/model-name` convention. +// Context / output caps sourced from novita.ai/models/llm. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama family + ModelDef { + id: "meta-llama/llama-4-maverick-17b-128e-instruct-fp8", + provider_id: "novita", + context_window: 1_048_576, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-4-scout-17b-16e-instruct", + provider_id: "novita", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3.3-70b-instruct", + provider_id: "novita", + context_window: 131_072, + max_output_tokens: 120_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3.2-3b-instruct", + provider_id: "novita", + context_window: 32_768, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3.1-8b-instruct", + provider_id: "novita", + context_window: 16_384, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3-70b-instruct", + provider_id: "novita", + context_window: 8_192, + max_output_tokens: 8_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3-8b-instruct", + provider_id: "novita", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek family + ModelDef { + id: "deepseek/deepseek-v3.2", + provider_id: "novita", + context_window: 163_840, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek/deepseek-v3.1", + provider_id: "novita", + context_window: 131_072, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek/deepseek-r1-0528", + provider_id: "novita", + context_window: 163_840, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + // R1 is a reasoning model; exposes reasoning_content. + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek/deepseek-r1-distill-qwen-32b", + provider_id: "novita", + context_window: 64_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Qwen family + ModelDef { + id: "qwen/qwen3-8b", + provider_id: "novita", + context_window: 128_000, + max_output_tokens: 20_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen/qwen2.5-72b-instruct", + provider_id: "novita", + context_window: 32_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen/qwen2.5-7b-instruct", + provider_id: "novita", + context_window: 32_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Other notable GA models + ModelDef { + id: "mistralai/mistral-nemo", + provider_id: "novita", + context_window: 60_288, + max_output_tokens: 16_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "google/gemma-3-27b-it", + provider_id: "novita", + context_window: 98_304, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "zhipu/glm-4.7-flash", + provider_id: "novita", + context_window: 200_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/nscale.rs b/crates/providers/src/providers/nscale.rs index 323b35f..b8462ea 100644 --- a/crates/providers/src/providers/nscale.rs +++ b/crates/providers/src/providers/nscale.rs @@ -1,12 +1,15 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Nscale serverless inference. OpenAI-compatible REST API. +// Docs: https://docs.nscale.com (chat completions at POST /v1/chat/completions). +// Auth: Authorization: Bearer $NSCALE_API_KEY. pub const PROVIDER: ProviderDef = ProviderDef { id: "nscale", display_name: "Nscale", - default_base_url: "https://inference.nscale.com/v1", + default_base_url: "https://inference.api.nscale.com/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -17,9 +20,199 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: false, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA chat/completion models. Image-generation SKUs (FLUX.1-schnell, SDXL) are +// excluded; this catalog only tracks text LLMs. Context windows reflect the +// upstream HF model cards; Nscale may cap lower on shared endpoints. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama family. + ModelDef { + id: "meta-llama/Llama-3.1-8B-Instruct", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/Llama-4-Scout-17B-16E-Instruct", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Qwen family. + ModelDef { + id: "Qwen/QwQ-32B", + provider_id: "nscale", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-3B-Instruct", + provider_id: "nscale", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-7B-Instruct", + provider_id: "nscale", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + provider_id: "nscale", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek R1 distilled reasoning models. + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + provider_id: "nscale", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Mistral family. + ModelDef { + id: "mistralai/mixtral-8x22b-instruct-v0.1", + provider_id: "nscale", + context_window: 65_536, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/nvidia_nim.rs b/crates/providers/src/providers/nvidia_nim.rs index 94e8f3c..8e71700 100644 --- a/crates/providers/src/providers/nvidia_nim.rs +++ b/crates/providers/src/providers/nvidia_nim.rs @@ -1,8 +1,11 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// NVIDIA NIM (NVIDIA Inference Microservices) exposes an OpenAI-compatible +// endpoint at https://integrate.api.nvidia.com/v1 (auth: Bearer NVIDIA_API_KEY). +// Model catalog sourced from build.nvidia.com / docs.api.nvidia.com. pub const PROVIDER: ProviderDef = ProviderDef { id: "nvidia_nim", display_name: "NVIDIA NIM", @@ -10,7 +13,7 @@ pub const PROVIDER: ProviderDef = ProviderDef { protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["NVIDIA_NIM_API_KEY"], + env_vars: &["NVIDIA_NIM_API_KEY", "NVIDIA_API_KEY"], litellm_prefix: "nvidia_nim/", capabilities: ProviderCapabilities { chat_completions: true, @@ -22,4 +25,247 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +pub const MODELS: &[ModelDef] = &[ + // Meta Llama family + ModelDef { + id: "meta/llama-3.1-405b-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-3.1-70b-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-3.1-8b-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-3.3-70b-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Llama 3.2 vision-capable variants + ModelDef { + id: "meta/llama-3.2-11b-vision-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta/llama-3.2-90b-vision-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // NVIDIA Nemotron family + ModelDef { + id: "nvidia/llama-3.1-nemotron-70b-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "nvidia/llama-3.3-nemotron-super-49b-v1", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // DeepSeek + ModelDef { + id: "deepseek-ai/deepseek-r1", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/deepseek-r1-distill-llama-8b", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "deepseek-ai/deepseek-r1-distill-qwen-32b", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Mistral AI + ModelDef { + id: "mistralai/mistral-large-2-instruct", + provider_id: "nvidia_nim", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/mixtral-8x22b-instruct-v0.1", + provider_id: "nvidia_nim", + context_window: 65_536, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Microsoft Phi + ModelDef { + id: "microsoft/phi-3-medium-4k-instruct", + provider_id: "nvidia_nim", + context_window: 4_096, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Qwen + ModelDef { + id: "qwen/qwen2.5-7b-instruct", + provider_id: "nvidia_nim", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen/qwen2.5-coder-32b-instruct", + provider_id: "nvidia_nim", + context_window: 32_768, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // NVIDIA retrieval embeddings + ModelDef { + id: "nvidia/nv-embedqa-e5-v5", + provider_id: "nvidia_nim", + context_window: 512, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "nvidia/llama-3.2-nv-embedqa-1b-v2", + provider_id: "nvidia_nim", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/openai.rs b/crates/providers/src/providers/openai.rs index efc708d..2190738 100644 --- a/crates/providers/src/providers/openai.rs +++ b/crates/providers/src/providers/openai.rs @@ -23,6 +23,87 @@ pub const PROVIDER: ProviderDef = ProviderDef { }; pub const MODELS: &[ModelDef] = &[ + // GPT-5 family (GA). Context window 272k, max output 128k, reasoning-capable. + // Verified against LiteLLM `model_prices_and_context_window.json` (openai provider). + ModelDef { + id: "gpt-5", + provider_id: "openai", + context_window: 272_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-5-mini", + provider_id: "openai", + context_window: 272_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-5-nano", + provider_id: "openai", + context_window: 272_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // GPT-4.1 family (GA, 2025-04-14). 1,047,576-token context, 32,768 max output. + ModelDef { + id: "gpt-4.1", + provider_id: "openai", + context_window: 1_047_576, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-4.1-mini", + provider_id: "openai", + context_window: 1_047_576, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "gpt-4.1-nano", + provider_id: "openai", + context_window: 1_047_576, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ModelDef { id: "gpt-4o", provider_id: "openai", @@ -36,6 +117,20 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // ChatGPT-4o latest (auto-updated alias). 128k context, 4k output per LiteLLM. + ModelDef { + id: "chatgpt-4o-latest", + provider_id: "openai", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ModelDef { id: "gpt-4o-mini", provider_id: "openai", diff --git a/crates/providers/src/providers/openrouter.rs b/crates/providers/src/providers/openrouter.rs index 6684bcf..220670a 100644 --- a/crates/providers/src/providers/openrouter.rs +++ b/crates/providers/src/providers/openrouter.rs @@ -22,4 +22,7 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// OpenRouter aggregates hundreds of models from many upstream providers. +// Model ids use the form `/` and are passed through as-is. +// Use GET /api/v1/models at runtime to enumerate available slugs. pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/ovhcloud.rs b/crates/providers/src/providers/ovhcloud.rs index 37f337f..588b133 100644 --- a/crates/providers/src/providers/ovhcloud.rs +++ b/crates/providers/src/providers/ovhcloud.rs @@ -1,26 +1,155 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// OVHCloud AI Endpoints — per-deployment URL; set via api_base or OPENAI_BASE_URL. +/// OVHCloud AI Endpoints — OpenAI-compatible gateway hosted on OVHcloud Public Cloud. +/// +/// Base URL is the unified gateway documented by OVHcloud and the Apache Airflow +/// provider: `https://oai.endpoints.kepler.ai.cloud.ovh.net/v1`. Per-model URLs +/// of the form `https://.endpoints.kepler.ai.cloud.ovh.net/api/openai_compat/v1` +/// also exist; callers can override via `api_base`. +/// +/// Auth: `Authorization: Bearer ` using an OVHcloud Manager-issued API key +/// (`OVH_AI_ENDPOINTS_ACCESS_TOKEN`). LiteLLM uses the alias `OVHCLOUD_API_KEY`. pub const PROVIDER: ProviderDef = ProviderDef { id: "ovhcloud", display_name: "OVHCloud AI Endpoints", - default_base_url: "", + default_base_url: "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["OVH_AI_ENDPOINTS_ACCESS_TOKEN"], + env_vars: &["OVH_AI_ENDPOINTS_ACCESS_TOKEN", "OVHCLOUD_API_KEY"], litellm_prefix: "ovhcloud/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, tool_use: true, - embeddings: false, - vision: false, + embeddings: true, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +/// GA models listed in the OVHcloud AI Endpoints catalog +/// (https://endpoints.ai.cloud.ovh.net/catalog). Model IDs match the slugs +/// documented by LiteLLM and the OVHcloud catalog (underscore-separated +/// version numbers where OVHcloud uses them, e.g. `Meta-Llama-3_3-70B-Instruct`). +/// +/// `max_output_tokens` is left conservative (8k) because OVHcloud does not +/// publish a hard per-request cap separate from the context window — callers +/// should set `max_tokens` explicitly. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama 3.3 70B Instruct — 131k context, function calling. + ModelDef { + id: "Meta-Llama-3_3-70B-Instruct", + provider_id: "ovhcloud", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mistral 7B Instruct v0.3 — 127k context. + ModelDef { + id: "Mistral-7B-Instruct-v0.3", + provider_id: "ovhcloud", + context_window: 127_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mistral Nemo Instruct 2407 — 118k context. + ModelDef { + id: "Mistral-Nemo-Instruct-2407", + provider_id: "ovhcloud", + context_window: 118_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mistral Small 3.2 24B Instruct (2506) — 128k context, vision + tools. + ModelDef { + id: "Mistral-Small-3.2-24B-Instruct-2506", + provider_id: "ovhcloud", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // OpenAI gpt-oss 120B reasoning model — 131k context. + ModelDef { + id: "gpt-oss-120b", + provider_id: "ovhcloud", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // OpenAI gpt-oss 20B reasoning model — 131k context. + ModelDef { + id: "gpt-oss-20b", + provider_id: "ovhcloud", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Qwen3 32B — 32k context, tool use, reasoning-capable. + ModelDef { + id: "Qwen3-32B", + provider_id: "ovhcloud", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // BGE-M3 embedding model — 8k context, no chat/tool support. + ModelDef { + id: "BGE-M3", + provider_id: "ovhcloud", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/perplexity.rs b/crates/providers/src/providers/perplexity.rs index c068bb9..7a2a566 100644 --- a/crates/providers/src/providers/perplexity.rs +++ b/crates/providers/src/providers/perplexity.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -12,14 +12,93 @@ pub const PROVIDER: ProviderDef = ProviderDef { status: ProviderStatus::Stub, env_vars: &["PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"], litellm_prefix: "perplexity/", + // Sonar chat-completions is OpenAI-compatible: Bearer auth, /chat/completions, + // SSE streaming, image_url content blocks. Function/tool calling is only exposed + // through the Pro Search preset (Responses API), not the GA Sonar chat endpoint, + // so tool_use stays false at the provider level. No embeddings or batch API. capabilities: ProviderCapabilities { chat_completions: true, streaming: true, tool_use: false, embeddings: false, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Publicly GA Sonar catalog (docs.perplexity.ai/getting-started/models and +// LiteLLM model_prices_and_context_window.json). Context windows per LiteLLM; +// max_output_tokens left at 0 when Perplexity does not publish a hard cap. +// r1-1776 is intentionally omitted: it was retired from the GA catalog. +pub const MODELS: &[ModelDef] = &[ + // Sonar: lightweight, grounded search. 128k context. + ModelDef { + id: "sonar", + provider_id: "perplexity", + context_window: 128_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Sonar Pro: advanced grounded search, larger context and capped output. + ModelDef { + id: "sonar-pro", + provider_id: "perplexity", + context_window: 200_000, + max_output_tokens: 8_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Sonar Reasoning: Chain-of-Thought reasoning model. + ModelDef { + id: "sonar-reasoning", + provider_id: "perplexity", + context_window: 128_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Sonar Reasoning Pro: premium CoT reasoning tier. + ModelDef { + id: "sonar-reasoning-pro", + provider_id: "perplexity", + context_window: 128_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Sonar Deep Research: exhaustive multi-step research agent. + ModelDef { + id: "sonar-deep-research", + provider_id: "perplexity", + context_window: 128_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/petals.rs b/crates/providers/src/providers/petals.rs index 5a1bbb3..d0ca772 100644 --- a/crates/providers/src/providers/petals.rs +++ b/crates/providers/src/providers/petals.rs @@ -4,10 +4,16 @@ use crate::provider::{ }; /// Petals — distributed inference over shared GPU clusters (local or swarm endpoint). +/// Self-hosted only: no public hosted REST API. Upstream ships a PyTorch/Transformers +/// client (`AutoDistributedModelForCausalLM`), not an OpenAI-compatible server; port +/// 31330 is the documented default for `petals.cli.run_server`. Users typically front +/// Petals with their own OpenAI-compat shim, hence the `OpenAICompat` protocol stub. +/// Models (BLOOM 176B, Llama 3.1 up to 405B, Mixtral 8x22B, Falcon 40B+) are swarm-dependent; +/// leaving MODELS empty since availability tracks volunteer GPUs at health.petals.dev. pub const PROVIDER: ProviderDef = ProviderDef { id: "petals", display_name: "Petals", - default_base_url: "http://localhost:8080", + default_base_url: "http://localhost:31330", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::None, status: ProviderStatus::Stub, diff --git a/crates/providers/src/providers/playht.rs b/crates/providers/src/providers/playht.rs index 9b05eaa..21b1d08 100644 --- a/crates/providers/src/providers/playht.rs +++ b/crates/providers/src/providers/playht.rs @@ -3,7 +3,19 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Play.ht — text-to-speech and voice cloning. Chat completions not supported. +/// Play.ht / PlayAI — text-to-speech and voice cloning. Chat completions not supported. +/// +/// Docs: https://docs.play.ht (legacy v2) and https://docs.play.ai (newer unified API). +/// Base URL preserved as the v2 TTS endpoint `api.play.ht/api/v2`; PlayAI's `api.play.ai/api/v1` +/// is the successor surface (same vendor) and exposes `/tts`, `/tts/stream`, `/voices`, +/// plus a WebSocket TTS channel. Pick the v2 URL here because existing LiteLLM-style configs +/// reference it; callers targeting PlayAI can override `default_base_url` at config time. +/// +/// NOTE on auth: Play.ht / PlayAI require TWO headers — `Authorization: ` AND +/// `X-User-ID: ` (PlayAI uppercases it as `X-USER-ID`). `AuthKind::Bearer` is a +/// lossy fit: it only models a single bearer-style header and cannot convey the user-id +/// second factor. Wiring this provider for real traffic will require either a new +/// `AuthKind` variant (e.g. `BearerPlusUserId`) or a provider-specific header injector. pub const PROVIDER: ProviderDef = ProviderDef { id: "playht", display_name: "Play.ht", @@ -11,11 +23,11 @@ pub const PROVIDER: ProviderDef = ProviderDef { protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Wired, - env_vars: &["PLAYHT_SECRET_KEY"], + env_vars: &["PLAYHT_SECRET_KEY", "PLAYHT_USER_ID", "PLAYAI_API_KEY"], litellm_prefix: "", capabilities: ProviderCapabilities { chat_completions: false, - streaming: false, + streaming: true, tool_use: false, embeddings: false, vision: false, @@ -23,7 +35,62 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// Model IDs follow the voice-engine names accepted by the PlayHT v2 `/tts` and +// `/tts/stream` endpoints (field `voice_engine`). PlayAI exposes the same +// engines under the names Dialog 1.0, Dialog 1.0 Turbo, and Play 3.0 Mini. pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "PlayDialog", + provider_id: "playht", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "PlayDialog-turbo", + provider_id: "playht", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "PlayDialogMultilingual", + provider_id: "playht", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Play3.0-mini", + provider_id: "playht", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, ModelDef { id: "PlayHT2.0", provider_id: "playht", @@ -63,43 +130,4 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, - ModelDef { - id: "Play3.0-mini", - provider_id: "playht", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "PlayDialog", - provider_id: "playht", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "PlayDialogMultilingual", - provider_id: "playht", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, ]; diff --git a/crates/providers/src/providers/pollinations.rs b/crates/providers/src/providers/pollinations.rs index c392256..9110950 100644 --- a/crates/providers/src/providers/pollinations.rs +++ b/crates/providers/src/providers/pollinations.rs @@ -3,8 +3,12 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Pollinations — free AI text/image generation with an OpenAI-compatible endpoint. -/// No API key required for the free tier. +/// Pollinations — free, anonymous aggregate of text/image/audio generation +/// exposed via an OpenAI-compatible endpoint at `text.pollinations.ai/openai`. +/// +/// Anonymous tier requires no API key (rate limited to roughly one request +/// per 15s). Registered users can pass a bearer token from auth.pollinations.ai +/// or authenticate web apps via the `referrer` parameter. pub const PROVIDER: ProviderDef = ProviderDef { id: "pollinations", display_name: "Pollinations", @@ -17,16 +21,40 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + tool_use: true, embeddings: false, vision: false, batch: false, }, }; -/// Pollinations uses friendly string aliases for models. The actual model routed -/// behind each alias may change over time as Pollinations updates their backend. +/// Pollinations exposes friendly string aliases. The backing model behind an +/// alias can rotate over time as Pollinations swaps providers (OVH, OpenAI, +/// etc.). Context windows below are conservative estimates; the live +/// `/models` endpoint does not publish per-model windows. +/// +/// Current authoritative anonymous listing at `GET https://text.pollinations.ai/models` +/// is just `openai-fast` (GPT-OSS 20B via OVH) with aliases `openai`, +/// `gpt-oss`, `gpt-oss-20b`, `ovh-reasoning`. Other ids below are documented +/// in APIDOCS.md for higher tiers or may require a bearer token. pub const MODELS: &[ModelDef] = &[ + // Default anonymous-tier text model. Reasoning + tools enabled per live + // /models response. Aliases: openai, gpt-oss, gpt-oss-20b, ovh-reasoning. + ModelDef { + id: "openai-fast", + provider_id: "pollinations", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Alias for openai-fast kept as an explicit entry so lookups by "openai" + // resolve to known metadata. ModelDef { id: "openai", provider_id: "pollinations", @@ -34,14 +62,29 @@ pub const MODELS: &[ModelDef] = &[ max_output_tokens: 4_096, capabilities: ModelCapabilities { streaming: true, - tool_use: false, + tool_use: true, vision: false, - extended_thinking: false, + extended_thinking: true, }, status: ModelStatus::Available, }, + // Documented reasoning-focused variant (o4-mini class per APIDOCS.md). ModelDef { - id: "openai-large", + id: "openai-reasoning", + provider_id: "pollinations", + context_window: 128_000, + max_output_tokens: 16_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Web-search augmented chat model. + ModelDef { + id: "searchgpt", provider_id: "pollinations", context_window: 128_000, max_output_tokens: 4_096, @@ -53,19 +96,23 @@ pub const MODELS: &[ModelDef] = &[ }, status: ModelStatus::Available, }, + // Audio-capable chat model (TTS voices: alloy, echo, fable, onyx, nova, + // shimmer). Treated as stub: requires higher tier than anonymous. ModelDef { - id: "openai-reasoning", + id: "openai-audio", provider_id: "pollinations", context_window: 128_000, - max_output_tokens: 16_000, + max_output_tokens: 4_096, capabilities: ModelCapabilities { streaming: true, tool_use: false, vision: false, - extended_thinking: true, + extended_thinking: false, }, - status: ModelStatus::Available, + status: ModelStatus::Stub, }, + // Mistral-backed alias. Retained as Stub because not present in current + // anonymous /models response but still referenced in APIDOCS.md examples. ModelDef { id: "mistral", provider_id: "pollinations", @@ -77,136 +124,6 @@ pub const MODELS: &[ModelDef] = &[ vision: false, extended_thinking: false, }, - status: ModelStatus::Available, - }, - ModelDef { - id: "mistral-large", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "llama", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "deepseek", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "deepseek-reasoner", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 16_000, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: true, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "phi", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "qwen", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "qwen-coder", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "sur", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "sur-mistral", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "unity", - provider_id: "pollinations", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, + status: ModelStatus::Stub, }, ]; diff --git a/crates/providers/src/providers/predibase.rs b/crates/providers/src/providers/predibase.rs index c12c047..7dbf890 100644 --- a/crates/providers/src/providers/predibase.rs +++ b/crates/providers/src/providers/predibase.rs @@ -1,8 +1,15 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Predibase: fine-tuning + inference platform. +// Serverless ("always-on shared") endpoints are tenant-scoped. Full URL shape: +// https://serving.app.predibase.com/{tenant_short_code}/deployments/v2/llms/{deployment_name}/v1/chat/completions +// `default_base_url` only covers the host; the tenant + deployment path must be +// supplied via configuration (managed backend `api_base`) at runtime. +// Auth: `Authorization: Bearer $PREDIBASE_API_KEY`. +// The endpoint is OpenAI chat-completions compatible. pub const PROVIDER: ProviderDef = ProviderDef { id: "predibase", display_name: "Predibase", @@ -22,4 +29,48 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Only GA "Always On Shared Endpoint" serverless base models per +// https://docs.predibase.com/inference/models/language-models (supported models table). +// Dedicated / private deployments can run many more base models, but those require +// per-tenant deployment IDs and are not part of the shared catalog. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "llama-3-1-8b-instruct", + provider_id: "predibase", + context_window: 64_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen3-8b", + provider_id: "predibase", + context_window: 64_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen3-32b", + provider_id: "predibase", + context_window: 16_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/public_ai.rs b/crates/providers/src/providers/public_ai.rs index bc9b2a2..fd1f73d 100644 --- a/crates/providers/src/providers/public_ai.rs +++ b/crates/providers/src/providers/public_ai.rs @@ -1,12 +1,19 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Public AI Inference Utility — nonprofit, sovereign-AI inference provider +// hosting publicly-funded open-weight models (Apertus from the Swiss AI +// Initiative, SEA-LION v4 from AI Singapore, Olmo-3 from AI2, EuroLLM from the +// UTTER project, DictaLM from DICTA). OpenAI-compatible API on vLLM backend. +// Docs: https://platform.publicai.co/docs +// Also exposed as an inference provider on Hugging Face (provider="publicai"). +// litellm_prefix kept as "public_ai/" for catalog compatibility. pub const PROVIDER: ProviderDef = ProviderDef { id: "public_ai", display_name: "PublicAI", - default_base_url: "https://api.publicai.io/v1", + default_base_url: "https://api.publicai.co/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -22,4 +29,106 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model IDs mirror the Hugging Face identifiers surfaced by +// https://huggingface.co/models?inference_provider=publicai, which is the +// canonical catalog Public AI publishes. Context windows reflect the +// upstream model cards; values are conservative where the provider has not +// published explicit deployment limits. +pub const MODELS: &[ModelDef] = &[ + // --- Swiss AI Initiative: Apertus (fully-open, reproducible) --- + ModelDef { + id: "swiss-ai/Apertus-8B-Instruct-2509", + provider_id: "public_ai", + context_window: 65_536, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "swiss-ai/Apertus-70B-Instruct-2509", + provider_id: "public_ai", + context_window: 65_536, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- AI Singapore: SEA-LION v4 (Southeast Asian languages) --- + ModelDef { + id: "aisingapore/Gemma-SEA-LION-v4-27B-IT", + provider_id: "public_ai", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "aisingapore/Qwen-SEA-LION-v4-32B-IT", + provider_id: "public_ai", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Allen Institute for AI: Olmo 3 (fully-open) --- + ModelDef { + id: "allenai/Olmo-3-7B-Instruct", + provider_id: "public_ai", + context_window: 65_536, + max_output_tokens: 65_536, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- DICTA (Israel): Hebrew-focused reasoning model --- + ModelDef { + id: "dicta-il/DictaLM-3.0-24B-Thinking", + provider_id: "public_ai", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // --- UTTER project: EuroLLM (European multilingual) --- + ModelDef { + id: "utter-project/EuroLLM-22B-Instruct-2512", + provider_id: "public_ai", + context_window: 4_096, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/replicate.rs b/crates/providers/src/providers/replicate.rs index 902da86..1bf1c00 100644 --- a/crates/providers/src/providers/replicate.rs +++ b/crates/providers/src/providers/replicate.rs @@ -1,16 +1,21 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Replicate's native HTTP API is prediction-based (POST /v1/predictions with +// owner/model:version inputs), not OpenAI chat-compat. LiteLLM wraps it behind +// the `replicate/` prefix; we keep `OpenAICompat` + `Stub` so the existing +// registry contract is preserved until a real adapter lands. +// Docs: https://replicate.com/docs/reference/http pub const PROVIDER: ProviderDef = ProviderDef { id: "replicate", display_name: "Replicate", - default_base_url: "https://openai-compat.replicate.com/v1", + default_base_url: "https://api.replicate.com/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["REPLICATE_API_KEY"], + env_vars: &["REPLICATE_API_KEY", "REPLICATE_API_TOKEN"], litellm_prefix: "replicate/", capabilities: ProviderCapabilities { chat_completions: true, @@ -22,4 +27,149 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[]; +// Representative popular GA models spanning text, image, and audio. +// Replicate model ids are `owner/model` (version pins are applied per-request, +// not encoded here). Context/output numbers reflect the underlying model card; +// image/audio entries use 0 where a chat-style context window does not apply. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama 3 70B Instruct — flagship open chat model. + ModelDef { + id: "meta/meta-llama-3-70b-instruct", + provider_id: "replicate", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Meta Llama 3 8B Instruct — lightweight chat tier. + ModelDef { + id: "meta/meta-llama-3-8b-instruct", + provider_id: "replicate", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mistral 7B Instruct v0.2 — widely used small instruct model. + ModelDef { + id: "mistralai/mistral-7b-instruct-v0.2", + provider_id: "replicate", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mixtral 8x7B Instruct — sparse MoE, strong reasoning for its class. + ModelDef { + id: "mistralai/mixtral-8x7b-instruct-v0.1", + provider_id: "replicate", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // FLUX schnell — fast text-to-image, GA from Black Forest Labs. + ModelDef { + id: "black-forest-labs/flux-schnell", + provider_id: "replicate", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // FLUX 1.1 Pro — higher-quality text-to-image tier. + ModelDef { + id: "black-forest-labs/flux-1.1-pro", + provider_id: "replicate", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Stable Diffusion 3.5 Large — Stability AI flagship image model. + ModelDef { + id: "stability-ai/stable-diffusion-3.5-large", + provider_id: "replicate", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // SDXL — long-running GA text-to-image baseline. + ModelDef { + id: "stability-ai/sdxl", + provider_id: "replicate", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // OpenAI Whisper — speech-to-text. No chat context; audio in, text out. + ModelDef { + id: "openai/whisper", + provider_id: "replicate", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Incredibly Fast Whisper — optimized ASR variant, popular on Replicate. + ModelDef { + id: "vaibhavs10/incredibly-fast-whisper", + provider_id: "replicate", + context_window: 0, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/sambanova.rs b/crates/providers/src/providers/sambanova.rs index dfca24c..82947dd 100644 --- a/crates/providers/src/providers/sambanova.rs +++ b/crates/providers/src/providers/sambanova.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -15,11 +15,142 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: false, - vision: false, + tool_use: true, + embeddings: true, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// SambaNova Cloud GA / production catalog. Sourced from +// https://sambanova-systems.mintlify.dev/docs/en/models/sambacloud-models +// and https://sambanova-systems.mintlify.dev/docs/en/features/function-calling. +// Context windows from the public models page; max_output_tokens is not +// published per-model, so we use conservative defaults (a fraction of the +// context window) rather than guess exact caps. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama + ModelDef { + id: "Meta-Llama-3.3-70B-Instruct", + provider_id: "sambanova", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "Meta-Llama-3.1-8B-Instruct", + provider_id: "sambanova", + context_window: 16_384, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Llama 4 (vision + tool use). + ModelDef { + id: "Llama-4-Maverick-17B-128E-Instruct", + provider_id: "sambanova", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek family. + ModelDef { + id: "DeepSeek-V3.1", + provider_id: "sambanova", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "DeepSeek-R1-0528", + provider_id: "sambanova", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Qwen. + ModelDef { + id: "Qwen3-32B", + provider_id: "sambanova", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // OpenAI open-weights on SambaNova. + ModelDef { + id: "gpt-oss-120b", + provider_id: "sambanova", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // MiniMax. + ModelDef { + id: "MiniMax-M2.5", + provider_id: "sambanova", + context_window: 163_840, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embeddings. + ModelDef { + id: "E5-Mistral-7B-Instruct", + provider_id: "sambanova", + context_window: 4_096, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/scaleway.rs b/crates/providers/src/providers/scaleway.rs index 9333d25..41ed2c6 100644 --- a/crates/providers/src/providers/scaleway.rs +++ b/crates/providers/src/providers/scaleway.rs @@ -1,13 +1,15 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Scaleway Generative APIs — per-deployment endpoint; set base URL via config. +/// Scaleway Generative APIs — OpenAI-compatible endpoint at api.scaleway.ai/v1. +/// Auth: `Authorization: Bearer $SCW_SECRET_KEY`. +/// Reference: https://www.scaleway.com/en/docs/generative-apis/reference-content/supported-models/ pub const PROVIDER: ProviderDef = ProviderDef { id: "scaleway", display_name: "Scaleway", - default_base_url: "", + default_base_url: "https://api.scaleway.ai", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -18,9 +20,191 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: true, tool_use: true, embeddings: true, - vision: false, + // Pixtral (vision) is GA on Scaleway. Batch is not offered. + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// GA models only. Preview entries (e.g. gemma-3-27b-it) and non-commercial-only +// licences (e.g. holo2-30b-a3b / CC-BY-NC) are excluded. Context / output sizes +// come from the supported-models reference page (k = 1024 where the docs use it +// colloquially; values rounded to the published figure). +pub const MODELS: &[ModelDef] = &[ + // Qwen flagship instruction model (best-accuracy recommendation from Scaleway). + ModelDef { + id: "qwen3.5-397b-a17b", + provider_id: "scaleway", + context_window: 250_000, + max_output_tokens: 16_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen3-235b-a22b-instruct-2507", + provider_id: "scaleway", + context_window: 250_000, + max_output_tokens: 16_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "qwen3-coder-30b-a3b-instruct", + provider_id: "scaleway", + context_window: 128_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mistral — getting-started recommendation from Scaleway. + ModelDef { + id: "mistral-small-3.2-24b-instruct-2506", + provider_id: "scaleway", + context_window: 128_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "devstral-2-123b-instruct-2512", + provider_id: "scaleway", + context_window: 200_000, + max_output_tokens: 16_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistral-nemo-instruct-2407", + provider_id: "scaleway", + context_window: 128_000, + max_output_tokens: 8_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // OpenAI open-weight. + ModelDef { + id: "gpt-oss-120b", + provider_id: "scaleway", + context_window: 128_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Meta Llama (3.x). + ModelDef { + id: "llama-3.3-70b-instruct", + provider_id: "scaleway", + context_window: 100_000, + max_output_tokens: 16_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama-3.1-8b-instruct", + provider_id: "scaleway", + context_window: 128_000, + max_output_tokens: 16_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek reasoning distill (thinking-capable). + ModelDef { + id: "deepseek-r1-distill-llama-70b", + provider_id: "scaleway", + context_window: 16_000, + max_output_tokens: 4_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Vision: Pixtral. + ModelDef { + id: "pixtral-12b-2409", + provider_id: "scaleway", + context_window: 128_000, + max_output_tokens: 4_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Embedding models (max_output_tokens = 0 by convention; context = max input tokens). + ModelDef { + id: "qwen3-embedding-8b", + provider_id: "scaleway", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "bge-multilingual-gemma2", + provider_id: "scaleway", + context_window: 8_192, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/serper.rs b/crates/providers/src/providers/serper.rs index 3d1504b..8681490 100644 --- a/crates/providers/src/providers/serper.rs +++ b/crates/providers/src/providers/serper.rs @@ -4,6 +4,10 @@ use crate::provider::{ }; /// Serper — Google Search API for AI applications. Chat completions not supported. +/// Base URL: https://google.serper.dev (POST /search, /images, /news, /places, +/// /videos, /maps, /shopping, /scholar, /patents, /autocomplete). +/// NOTE: Serper authenticates via `X-API-KEY` header, not `Authorization: Bearer`. +/// `AuthKind::Bearer` below is a placeholder until `AuthKind::XApiKey` is added. pub const PROVIDER: ProviderDef = ProviderDef { id: "serper", display_name: "Serper", diff --git a/crates/providers/src/providers/siliconflow.rs b/crates/providers/src/providers/siliconflow.rs index 968006a..fbe0b4f 100644 --- a/crates/providers/src/providers/siliconflow.rs +++ b/crates/providers/src/providers/siliconflow.rs @@ -1,9 +1,22 @@ -use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; +use crate::model::ModelDef; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// SiliconFlow — Chinese inference platform with OpenAI-compatible API. +/// SiliconFlow — inference platform with an OpenAI-compatible API. +/// +/// Official base URL: +/// Docs: +/// +/// SiliconFlow hosts chat, embeddings, image generation (FLUX, Kolors, Qwen image), +/// reranking, and audio/TTS (CosyVoice, MOSS-TTSD) models behind the same base URL. +/// +/// Model list intentionally left empty: SiliconFlow's model catalog rotates +/// frequently (Qwen3.x, GLM-4.x/5, DeepSeek-V3.x, Pro/ vs. free tiers) and the +/// public docs do not publish an authoritative context_window / max_output_tokens +/// table per model. Values must be looked up per-model at +/// . Guessing would violate the provider +/// contract (see `ModelDef` docs). pub const PROVIDER: ProviderDef = ProviderDef { id: "siliconflow", display_name: "SiliconFlow", @@ -23,272 +36,4 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[ - // Qwen2.5 Instruct series - ModelDef { - id: "Qwen/Qwen2.5-7B-Instruct", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "Qwen/Qwen2.5-14B-Instruct", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "Qwen/Qwen2.5-32B-Instruct", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "Qwen/Qwen2.5-72B-Instruct", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // Qwen2.5 Coder - ModelDef { - id: "Qwen/Qwen2.5-Coder-7B-Instruct", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "Qwen/Qwen2.5-Coder-32B-Instruct", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // DeepSeek - ModelDef { - id: "deepseek-ai/DeepSeek-V3", - provider_id: "siliconflow", - context_window: 64_000, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "deepseek-ai/DeepSeek-R1", - provider_id: "siliconflow", - context_window: 64_000, - max_output_tokens: 16_000, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: true, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: true, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - provider_id: "siliconflow", - context_window: 32_768, - max_output_tokens: 8_192, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: true, - }, - status: ModelStatus::Available, - }, - // Llama series - ModelDef { - id: "meta-llama/Meta-Llama-3.1-8B-Instruct", - provider_id: "siliconflow", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "meta-llama/Meta-Llama-3.1-70B-Instruct", - provider_id: "siliconflow", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "meta-llama/Llama-3.3-70B-Instruct", - provider_id: "siliconflow", - context_window: 128_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: true, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // Yi series - ModelDef { - id: "01-ai/Yi-1.5-9B-Chat-16K", - provider_id: "siliconflow", - context_window: 16_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "01-ai/Yi-1.5-34B-Chat-16K", - provider_id: "siliconflow", - context_window: 16_000, - max_output_tokens: 4_096, - capabilities: ModelCapabilities { - streaming: true, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // FLUX image generation (not chat, mark accordingly) - ModelDef { - id: "black-forest-labs/FLUX.1-schnell", - provider_id: "siliconflow", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "black-forest-labs/FLUX.1-dev", - provider_id: "siliconflow", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - // Embedding models - ModelDef { - id: "BAAI/bge-m3", - provider_id: "siliconflow", - context_window: 8_192, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "BAAI/bge-large-zh-v1.5", - provider_id: "siliconflow", - context_window: 512, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "BAAI/bge-large-en-v1.5", - provider_id: "siliconflow", - context_window: 512, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, -]; +pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/snowflake.rs b/crates/providers/src/providers/snowflake.rs index e12fb46..b5493ea 100644 --- a/crates/providers/src/providers/snowflake.rs +++ b/crates/providers/src/providers/snowflake.rs @@ -1,9 +1,18 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Snowflake Cortex AI — per-account URL; requires SNOWFLAKE_ACCOUNT_ID. +/// Snowflake Cortex AI — managed LLM inference inside Snowflake. +/// +/// The REST endpoint is per-account and not a single global host: +/// `https://.snowflakecomputing.com/api/v2/cortex/inference:complete` +/// The request/response schema follows OpenAI Chat Completions. Auth is a +/// bearer token sourced from a programmatic access token (PAT), a key-pair JWT, +/// or an OAuth token. For that reason `default_base_url` is left empty; users +/// must configure `OPENAI_BASE_URL` (or `api_base` in YAML) to their account URL. +/// +/// Docs: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api pub const PROVIDER: ProviderDef = ProviderDef { id: "snowflake", display_name: "Snowflake Cortex", @@ -11,16 +20,147 @@ pub const PROVIDER: ProviderDef = ProviderDef { protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["SNOWFLAKE_JWT", "SNOWFLAKE_ACCOUNT_ID"], + // SNOWFLAKE_JWT is the canonical legacy name in this codebase (see docs/ENV.md). + // SNOWFLAKE_PAT is Snowflake's newer recommended mechanism; both are bearer tokens. + // SNOWFLAKE_ACCOUNT_ID supplies the per-account hostname component. + env_vars: &["SNOWFLAKE_JWT", "SNOWFLAKE_PAT", "SNOWFLAKE_ACCOUNT_ID"], litellm_prefix: "snowflake/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + // Cortex Inference follows OpenAI chat/completions and supports tool/function calling + // on models that themselves support it (Llama 3.1+, Mistral Large 2, Claude via Cortex). + tool_use: true, embeddings: false, vision: false, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Cortex-hosted chat models verified against the Snowflake COMPLETE function +// and Cortex Inference docs. Context windows reflect the underlying model spec +// (Snowflake has not published reduced per-deployment limits for these). +// Region availability varies; treat this list as a superset. +pub const MODELS: &[ModelDef] = &[ + // Snowflake Arctic — 480B MoE, 17B active. Apache 2.0. + ModelDef { + id: "snowflake-arctic", + provider_id: "snowflake", + context_window: 4_096, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // SwiftKV-optimized Snowflake variants of Meta Llama. + ModelDef { + id: "snowflake-llama-3.3-70b", + provider_id: "snowflake", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "snowflake-llama-3.1-405b", + provider_id: "snowflake", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Meta Llama baselines hosted on Cortex. + ModelDef { + id: "llama3.3-70b", + provider_id: "snowflake", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama3.1-70b", + provider_id: "snowflake", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "llama3.1-8b", + provider_id: "snowflake", + context_window: 128_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mistral family. + ModelDef { + id: "mistral-large2", + provider_id: "snowflake", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mixtral-8x7b", + provider_id: "snowflake", + context_window: 32_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistral-7b", + provider_id: "snowflake", + context_window: 32_000, + max_output_tokens: 4_096, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/stability.rs b/crates/providers/src/providers/stability.rs index c30f1ae..10e8956 100644 --- a/crates/providers/src/providers/stability.rs +++ b/crates/providers/src/providers/stability.rs @@ -1,19 +1,28 @@ -use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; +use crate::model::ModelDef; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Stability AI — primarily image generation; chat completions not supported. +/// Stability AI — image generation (Stable Image / Stable Diffusion 3.5 family) +/// plus upscalers, edits, and video endpoints. All endpoints are per-model REST +/// paths under `/v2beta` (e.g. `POST /v2beta/stable-image/generate/ultra`), +/// not OpenAI-compatible chat. See https://platform.stability.ai/docs/api-reference. pub const PROVIDER: ProviderDef = ProviderDef { id: "stability_ai", display_name: "Stability AI", - default_base_url: "https://api.stability.ai/v2beta", + // Official REST host. Path versioning (`/v2beta/...`, `/v1/...`) is per-endpoint + // and therefore handled by the caller, not baked into the base URL. + default_base_url: "https://api.stability.ai", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Wired, env_vars: &["STABILITY_API_KEY"], litellm_prefix: "stability_ai/", capabilities: ProviderCapabilities { + // Image-generation focused: no chat, streaming, tools, embeddings, or batch. + // `vision` here means input-image-as-context for chat, which Stability does + // not offer (image-to-image exists, but is a separate REST endpoint, not a + // chat vision capability). chat_completions: false, streaming: false, tool_use: false, @@ -23,135 +32,7 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; -pub const MODELS: &[ModelDef] = &[ - ModelDef { - id: "stable-diffusion-3-5-large", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-diffusion-3-5-large-turbo", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-diffusion-3-5-medium", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-diffusion-3-large", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-diffusion-3-large-turbo", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-diffusion-3-medium", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-image-ultra", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-image-core", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Available, - }, - ModelDef { - id: "stable-diffusion-xl-1024-v1-0", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Deprecated, - }, - ModelDef { - id: "stable-diffusion-v1-6", - provider_id: "stability_ai", - context_window: 0, - max_output_tokens: 0, - capabilities: ModelCapabilities { - streaming: false, - tool_use: false, - vision: false, - extended_thinking: false, - }, - status: ModelStatus::Deprecated, - }, -]; +// Stability's API exposes per-endpoint model paths (e.g. `/v2beta/stable-image/ +// generate/ultra`) rather than a `model` field in a chat request, so there are +// no chat-style model IDs to enumerate here. Consumers select behaviour by URL. +pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/together.rs b/crates/providers/src/providers/together.rs index 76bdcc8..ee3f93d 100644 --- a/crates/providers/src/providers/together.rs +++ b/crates/providers/src/providers/together.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,6 +6,8 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "together_ai", display_name: "Together AI", + // OpenAI-compatible chat completions at /v1/chat/completions. + // `.xyz` is the canonical host (api.together.ai aliases to the same). default_base_url: "https://api.together.xyz/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, @@ -15,11 +17,121 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, + // Function/tool calling is supported on select models (e.g. Llama-3.1/3.3 Turbo, + // DeepSeek-V3). Some Turbo variants have known tool-calling quirks. tool_use: true, + // Embeddings endpoint is available (e.g. intfloat/multilingual-e5-large-instruct). embeddings: true, - vision: false, + // Together hosts multimodal models (Qwen VL family), so vision is supported at + // the endpoint level even if not every chat model accepts images. + vision: true, + // No Batch API endpoint. batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Conservative set limited to models whose exact ID and context window were +// verified against Together's serverless docs or other authoritative sources. +// Prefer adding fewer entries than guessing; the rest are routed through the +// proxy as unknown-but-OpenAI-compatible models regardless. +pub const MODELS: &[ModelDef] = &[ + // Meta Llama 3.3 70B Instruct Turbo. 131,072 ctx per Together serverless docs. + ModelDef { + id: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + provider_id: "together_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Meta Llama 3.1 70B Instruct Turbo. 128K native Llama 3.1 context. + ModelDef { + id: "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", + provider_id: "together_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Meta Llama 3.1 8B Instruct Turbo. Same 128K Llama 3.1 context. + ModelDef { + id: "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + provider_id: "together_ai", + context_window: 131_072, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // DeepSeek-R1 reasoning model. 163,839 ctx per Together serverless docs. + ModelDef { + id: "deepseek-ai/DeepSeek-R1", + provider_id: "together_ai", + context_window: 163_839, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // DeepSeek-V3 general-purpose MoE. 128K native context. + ModelDef { + id: "deepseek-ai/DeepSeek-V3", + provider_id: "together_ai", + context_window: 128_000, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Qwen2.5 72B Instruct Turbo. 32,768 ctx on Together's Turbo (FP8) deployment + // (base model is 128K; Turbo is reduced). + ModelDef { + id: "Qwen/Qwen2.5-72B-Instruct-Turbo", + provider_id: "together_ai", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Mixtral 8x7B Instruct v0.1. 32,768 native training context. + ModelDef { + id: "mistralai/Mixtral-8x7B-Instruct-v0.1", + provider_id: "together_ai", + context_window: 32_768, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/triton.rs b/crates/providers/src/providers/triton.rs index c5c3710..c598416 100644 --- a/crates/providers/src/providers/triton.rs +++ b/crates/providers/src/providers/triton.rs @@ -3,11 +3,22 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// NVIDIA Triton Inference Server — self-hosted; set endpoint via config. +/// NVIDIA Triton Inference Server — self-hosted. +/// +/// Triton's OpenAI-compatible frontend (stable as of 2025) binds to +/// `http://localhost:9000` by default when launched via Triton CLI +/// (`triton start --frontend openai`) or `openai_frontend/main.py`. +/// It exposes `/v1/chat/completions`, `/v1/completions`, `/v1/models`, +/// and (via the vLLM backend) embeddings. Tools and `tool_choice` are +/// supported on chat completions. Models are user-deployed so no static +/// catalog is shipped; operators override `default_base_url` per deployment. +/// +/// Note: the native Triton KServe v2 HTTP frontend uses port 8000 and a +/// different wire format — this entry targets the OpenAI-compat frontend. pub const PROVIDER: ProviderDef = ProviderDef { id: "triton", display_name: "NVIDIA Triton", - default_base_url: "", + default_base_url: "http://localhost:9000", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::None, status: ProviderStatus::Stub, @@ -16,8 +27,8 @@ pub const PROVIDER: ProviderDef = ProviderDef { capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, - embeddings: false, + tool_use: true, + embeddings: true, vision: false, batch: false, }, diff --git a/crates/providers/src/providers/vertex.rs b/crates/providers/src/providers/vertex.rs index 392426a..ec2ec6e 100644 --- a/crates/providers/src/providers/vertex.rs +++ b/crates/providers/src/providers/vertex.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,12 +6,25 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "vertex_ai", display_name: "Google Vertex AI", - // URL is constructed per-project/region: https://{region}-aiplatform.googleapis.com/... - default_base_url: "", + // Vertex constructs per-region URLs: https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/... + // us-central1 is the most broadly supported region for Gemini; callers override via VERTEX_LOCATION. + default_base_url: "https://us-central1-aiplatform.googleapis.com", protocol: ProviderProtocol::VertexAI, auth: AuthKind::GoogleApiKey, status: ProviderStatus::Implemented, - env_vars: &["VERTEX_API_KEY", "GOOGLE_ACCESS_TOKEN"], + // Canonical GCP auth uses a service-account JSON via GOOGLE_APPLICATION_CREDENTIALS. + // VERTEX_PROJECT / VERTEX_LOCATION scope the endpoint; GOOGLE_CLOUD_PROJECT / GOOGLE_CLOUD_LOCATION + // are the canonical gcloud names accepted as aliases. VERTEX_API_KEY / GOOGLE_ACCESS_TOKEN allow + // bypassing ADC when a short-lived bearer or express-mode API key is supplied directly. + env_vars: &[ + "GOOGLE_APPLICATION_CREDENTIALS", + "VERTEX_PROJECT", + "VERTEX_LOCATION", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", + "VERTEX_API_KEY", + "GOOGLE_ACCESS_TOKEN", + ], litellm_prefix: "vertex_ai/", capabilities: ProviderCapabilities { chat_completions: true, @@ -19,10 +32,83 @@ pub const PROVIDER: ProviderDef = ProviderDef { tool_use: true, embeddings: true, vision: true, - batch: false, + batch: true, }, }; -// Vertex AI serves Gemini models (and others) under the same model IDs as Google AI Studio, -// routed via project/region endpoints. No separate model list needed. -pub const MODELS: &[ModelDef] = &[]; +// Publicly GA Gemini models on Vertex AI. Context windows and max output tokens verified against +// https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/. +// Gemini 1.5 Pro / 1.5 Flash were fully retired by 2025-09-24 and are intentionally omitted. +pub const MODELS: &[ModelDef] = &[ + // Gemini 2.5 Pro - GA 2025-06-17. 1M input, 65,535 output. Reasoning-capable. + ModelDef { + id: "gemini-2.5-pro", + provider_id: "vertex_ai", + context_window: 1_048_576, + max_output_tokens: 65_535, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Gemini 2.5 Flash - GA 2025-06-17. 1M input, 65,535 output. Reasoning-capable. + ModelDef { + id: "gemini-2.5-flash", + provider_id: "vertex_ai", + context_window: 1_048_576, + max_output_tokens: 65_535, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Gemini 2.5 Flash-Lite - GA 2025-07-22. Balanced low-latency variant. + ModelDef { + id: "gemini-2.5-flash-lite", + provider_id: "vertex_ai", + context_window: 1_048_576, + max_output_tokens: 65_535, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Gemini 2.0 Flash - GA 2025-02-05. 1M input, 8,192 output. + // Note: as of 2026-03-06 restricted to existing customers; new projects should prefer 2.5 Flash. + ModelDef { + id: "gemini-2.0-flash", + provider_id: "vertex_ai", + context_window: 1_048_576, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Gemini 2.0 Flash-Lite - GA 2025-02-25. Same restriction as 2.0 Flash from 2026-03-06. + ModelDef { + id: "gemini-2.0-flash-lite", + provider_id: "vertex_ai", + context_window: 1_048_576, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/vllm.rs b/crates/providers/src/providers/vllm.rs index 6988579..3884a88 100644 --- a/crates/providers/src/providers/vllm.rs +++ b/crates/providers/src/providers/vllm.rs @@ -6,14 +6,24 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "hosted_vllm", display_name: "vLLM (self-hosted)", - // No default URL — each vLLM deployment has its own host:port. - default_base_url: "", + // vLLM's OpenAI-compatible server defaults to http://localhost:8000. + // The `/v1` suffix matches the OpenAI SDK convention documented by vLLM + // (see https://docs.vllm.ai/en/latest/getting_started/quickstart). + // Override per deployment via config / env. + default_base_url: "http://localhost:8000/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["VLLM_API_KEY"], + // Self-hosted: no auth by default. vLLM examples use api_key="empty"/"dummy". + // An API key can be enforced with `--api-key` at server start, but there is + // no canonical env var convention, so users supply it via managed-backend config. + env_vars: &[], litellm_prefix: "hosted_vllm/", capabilities: ProviderCapabilities { + // Endpoint-level capabilities. Model-level support (tool_use, vision, + // embeddings) varies — vLLM exposes the endpoints; whether a loaded + // model honors them depends on the model and server flags + // (e.g. --enable-auto-tool-choice, --tool-call-parser). chat_completions: true, streaming: true, tool_use: true, diff --git a/crates/providers/src/providers/volcengine.rs b/crates/providers/src/providers/volcengine.rs index c1c6f13..1d0966c 100644 --- a/crates/providers/src/providers/volcengine.rs +++ b/crates/providers/src/providers/volcengine.rs @@ -1,9 +1,17 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; /// Volcano Engine (ByteDance Ark) — OpenAI-compatible endpoint. +/// +/// Ark exposes an OpenAI-compatible Chat Completions API under +/// `https://ark.cn-beijing.volces.com/api/v3`. Auth is `Authorization: Bearer `. +/// Canonical env var is `ARK_API_KEY` (as used by the official `volcenginesdkarkruntime` +/// Python SDK); `VOLCENGINE_API_KEY` is accepted as an alias by many third-party tools. +/// +/// Models can be referenced either by model ID (e.g. `doubao-seed-2-0-pro-260215`) +/// after activation in the Ark console, or by endpoint ID (e.g. `ep-YYYYMMDDHHMMSS-xxxxx`). pub const PROVIDER: ProviderDef = ProviderDef { id: "volcengine", display_name: "Volcano Engine", @@ -11,16 +19,115 @@ pub const PROVIDER: ProviderDef = ProviderDef { protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["VOLCENGINE_API_KEY"], + // ARK_API_KEY is the canonical name (official SDK). VOLCENGINE_API_KEY is a + // common alias used by third-party integrations (LobeHub, MCP servers). + env_vars: &["ARK_API_KEY", "VOLCENGINE_API_KEY"], litellm_prefix: "volcengine/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, tool_use: true, - embeddings: false, + // Ark offers Doubao embedding models (doubao-embedding-*) via the same endpoint. + embeddings: true, vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +/// Doubao Seed 2.0 family — verified against LiteLLM's +/// `model_prices_and_context_window.json` (`litellm_provider: volcengine`). +/// Context window 256k input, 128k max output, tools + vision + reasoning. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "doubao-seed-2-0-pro-260215", + provider_id: "volcengine", + context_window: 256_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "doubao-seed-2-0-lite-260215", + provider_id: "volcengine", + context_window: 256_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "doubao-seed-2-0-mini-260215", + provider_id: "volcengine", + context_window: 256_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "doubao-seed-2-0-code-preview-260215", + provider_id: "volcengine", + context_window: 256_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Doubao embedding models. max_input_tokens 4096 per LiteLLM; no output tokens. + ModelDef { + id: "doubao-embedding-large-text-250515", + provider_id: "volcengine", + context_window: 4_096, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "doubao-embedding-large-text-240915", + provider_id: "volcengine", + context_window: 4_096, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "doubao-embedding-text-240715", + provider_id: "volcengine", + context_window: 4_096, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/voyage.rs b/crates/providers/src/providers/voyage.rs index fe5033b..cfb496f 100644 --- a/crates/providers/src/providers/voyage.rs +++ b/crates/providers/src/providers/voyage.rs @@ -1,9 +1,13 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Voyage AI — embeddings-only provider. +/// Voyage AI — embeddings and reranker provider. +/// +/// Voyage exposes `/v1/embeddings` and `/v1/rerank` endpoints. It does not +/// offer chat completions, streaming, or tool use. Authentication is a +/// bearer token (`Authorization: Bearer $VOYAGE_API_KEY`). pub const PROVIDER: ProviderDef = ProviderDef { id: "voyage", display_name: "Voyage AI", @@ -18,9 +22,148 @@ pub const PROVIDER: ProviderDef = ProviderDef { streaming: false, tool_use: false, embeddings: true, - vision: false, + // voyage-multimodal-3 accepts interleaved text+image inputs via the + // multimodal embeddings endpoint; flagging vision at the provider + // level since at least one GA model supports it. + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Context windows verified against https://docs.voyageai.com/docs/embeddings +// (retrieved April 2026). `max_output_tokens = 0` for all embedding models — +// they return vectors, not generated tokens. +pub const MODELS: &[ModelDef] = &[ + // General-purpose (Voyage 3.x generation, GA). + ModelDef { + id: "voyage-3-large", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-3.5", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-3.5-lite", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-3", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-3-lite", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Domain-specialized. + ModelDef { + id: "voyage-code-3", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-code-2", + provider_id: "voyage", + context_window: 16_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-finance-2", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-law-2", + provider_id: "voyage", + context_window: 16_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "voyage-multilingual-2", + provider_id: "voyage", + context_window: 32_000, + max_output_tokens: 0, + capabilities: ModelCapabilities { + streaming: false, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/wandb.rs b/crates/providers/src/providers/wandb.rs index 576f091..38ee84f 100644 --- a/crates/providers/src/providers/wandb.rs +++ b/crates/providers/src/providers/wandb.rs @@ -3,11 +3,16 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Weights & Biases Inference — per-project URL; set via api_base or OPENAI_BASE_URL. +/// Weights & Biases Inference — OpenAI-compatible endpoint. +/// Docs: https://docs.wandb.ai/inference/api-reference +/// Base URL: https://api.inference.wandb.ai/v1 +/// Auth: `Authorization: Bearer ` (create at https://wandb.ai/authorize). +/// Chat Completions are supported; function/tool calling and vision vary per model and are +/// not documented uniformly, so provider-level capability flags are kept conservative. pub const PROVIDER: ProviderDef = ProviderDef { id: "wandb", display_name: "Weights & Biases Inference", - default_base_url: "", + default_base_url: "https://api.inference.wandb.ai/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, @@ -23,4 +28,7 @@ pub const PROVIDER: ProviderDef = ProviderDef { }, }; +// Model catalog intentionally empty: W&B Inference's hosted model list changes frequently +// and the public docs do not publish stable per-model context window / max output token +// values. Routing works via the `wandb/` LiteLLM prefix at runtime. pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/providers/watsonx.rs b/crates/providers/src/providers/watsonx.rs index 4f9db92..ec24865 100644 --- a/crates/providers/src/providers/watsonx.rs +++ b/crates/providers/src/providers/watsonx.rs @@ -1,26 +1,305 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// IBM WatsonX — OpenAI-compatible endpoint; set instance URL via WATSONX_URL. +/// IBM watsonx.ai — region-scoped foundation model API. +/// +/// Auth: IBM Cloud IAM. An `apikey` is exchanged at `https://iam.cloud.ibm.com/identity/token` +/// for a short-lived `Bearer` token used against the regional `*.ml.cloud.ibm.com` host. +/// Requests must also carry a `project_id` (or `space_id`) in the JSON body. +/// +/// Base URL is region-specific (us-south, eu-de, eu-gb, jp-tok, au-syd, ca-tor); we default to +/// Dallas (`us-south`) and let operators override via `WATSONX_URL` / `WATSONX_REGION`. +/// Endpoints used: +/// - `POST /ml/v1/text/chat?version=2024-05-01` (chat completions) +/// - `POST /ml/v1/text/chat_stream?version=2024-05-01` (SSE streaming) +/// - `POST /ml/v1/text/generation?version=2023-05-02` (legacy text generation) +/// - `POST /ml/v1/text/embeddings?version=2023-10-25` (embeddings) pub const PROVIDER: ProviderDef = ProviderDef { id: "watsonx", - display_name: "IBM WatsonX", - default_base_url: "", + display_name: "IBM watsonx.ai", + default_base_url: "https://us-south.ml.cloud.ibm.com", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["WATSONX_API_KEY", "WATSONX_URL"], + env_vars: &[ + "WATSONX_API_KEY", + "WATSONX_APIKEY", + "WATSONX_PROJECT_ID", + "WATSONX_URL", + "WATSONX_REGION", + "WATSONX_SPACE_ID", + "WATSONX_TOKEN", + ], litellm_prefix: "watsonx/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, tool_use: true, embeddings: true, - vision: false, + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +/// Generally-available foundation models hosted by IBM on watsonx.ai multitenant +/// infrastructure. Context windows verified against LiteLLM +/// `model_prices_and_context_window.json` (sourced from IBM's official +/// `dataplatform.cloud.ibm.com/docs/.../fm-models.html` per BerriAI/litellm PR #15219). +/// +/// Model availability is region-dependent on watsonx; not every model exists in every +/// data center. Deploy-on-demand and tech-preview models are omitted. +pub const MODELS: &[ModelDef] = &[ + // --- IBM Granite family --- + ModelDef { + id: "ibm/granite-3-3-8b-instruct", + provider_id: "watsonx", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ibm/granite-3-8b-instruct", + provider_id: "watsonx", + context_window: 8_192, + max_output_tokens: 1_024, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ibm/granite-4-h-small", + provider_id: "watsonx", + context_window: 20_480, + max_output_tokens: 20_480, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ibm/granite-vision-3-2-2b", + provider_id: "watsonx", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ibm/granite-guardian-3-2-2b", + provider_id: "watsonx", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "ibm/granite-guardian-3-3-8b", + provider_id: "watsonx", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Meta Llama family --- + ModelDef { + id: "meta-llama/llama-3-3-70b-instruct", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3-2-1b-instruct", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3-2-3b-instruct", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3-2-11b-vision-instruct", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-3-2-90b-vision-instruct", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-4-maverick-17b", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "meta-llama/llama-guard-3-11b-vision", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Mistral family --- + ModelDef { + id: "mistralai/mistral-large", + provider_id: "watsonx", + context_window: 131_072, + max_output_tokens: 16_384, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/mistral-medium-2505", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/mistral-small-3-1-24b-instruct-2503", + provider_id: "watsonx", + context_window: 32_000, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mistralai/pixtral-12b-2409", + provider_id: "watsonx", + context_window: 128_000, + max_output_tokens: 128_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // --- Other third-party --- + ModelDef { + id: "sdaia/allam-1-13b-instruct", + provider_id: "watsonx", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: false, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "openai/gpt-oss-120b", + provider_id: "watsonx", + context_window: 8_192, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/xai.rs b/crates/providers/src/providers/xai.rs index bcd3764..8cc4bd3 100644 --- a/crates/providers/src/providers/xai.rs +++ b/crates/providers/src/providers/xai.rs @@ -1,4 +1,4 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; @@ -6,6 +6,7 @@ use crate::provider::{ pub const PROVIDER: ProviderDef = ProviderDef { id: "xai", display_name: "xAI", + // Official REST base URL. OpenAI-compatible Chat Completions at `/v1/chat/completions`. default_base_url: "https://api.x.ai/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, @@ -16,10 +17,128 @@ pub const PROVIDER: ProviderDef = ProviderDef { chat_completions: true, streaming: true, tool_use: true, + // xAI does not publish a public embeddings endpoint as of this writing. embeddings: false, + // Vision is supported on a subset of models (grok-2-vision-1212, grok-4.1-fast, etc.). vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Context windows and max output tokens verified against LiteLLM's +// `model_prices_and_context_window.json` (xai/ provider entries) and xAI docs. +// Only publicly GA models are listed here. +pub const MODELS: &[ModelDef] = &[ + // Grok 4 (flagship reasoning). `grok-4` is an alias for the latest stable release. + // 256K context / 256K max output, always-on reasoning, tool calling, no image input. + ModelDef { + id: "grok-4", + provider_id: "xai", + context_window: 256_000, + max_output_tokens: 256_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Grok 4 Fast - two variants exposed as separate model IDs by the xAI API. + // 2M token context window; reasoning variant has always-on thinking. + ModelDef { + id: "grok-4-fast-reasoning", + provider_id: "xai", + context_window: 2_000_000, + max_output_tokens: 2_000_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "grok-4-fast-non-reasoning", + provider_id: "xai", + context_window: 2_000_000, + max_output_tokens: 2_000_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Grok 3 family. 131,072 context window. Standard variant has no reasoning; + // mini variant supports reasoning via the `reasoning_effort` parameter. + ModelDef { + id: "grok-3", + provider_id: "xai", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "grok-3-mini", + provider_id: "xai", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + // Grok 2 dated snapshots. Vision variant accepts image input; text variant does not. + ModelDef { + id: "grok-2-vision-1212", + provider_id: "xai", + context_window: 32_768, + max_output_tokens: 32_768, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "grok-2-1212", + provider_id: "xai", + context_window: 131_072, + max_output_tokens: 131_072, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + // Grok Code Fast 1 - agentic coding model. 256K context. + ModelDef { + id: "grok-code-fast-1", + provider_id: "xai", + context_window: 256_000, + max_output_tokens: 256_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/xiaomi_mimo.rs b/crates/providers/src/providers/xiaomi_mimo.rs index 43ab250..da77e56 100644 --- a/crates/providers/src/providers/xiaomi_mimo.rs +++ b/crates/providers/src/providers/xiaomi_mimo.rs @@ -1,25 +1,74 @@ -use crate::model::ModelDef; +use crate::model::{ModelCapabilities, ModelDef, ModelStatus}; use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; +// Xiaomi operates an official OpenAI-compatible API platform at +// `api.xiaomimimo.com` (developer portal at platform.xiaomimimo.com). +// Open weights are published under huggingface.co/XiaomiMiMo (MIT/Apache-2.0), +// but the hosted API is the canonical inference endpoint used here. +// Auth: `Authorization: Bearer ` issued from the platform dashboard. pub const PROVIDER: ProviderDef = ProviderDef { id: "xiaomi_mimo", display_name: "Xiaomi MiMo", - default_base_url: "https://api.mimo.chat/v1", + default_base_url: "https://api.xiaomimimo.com/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::Bearer, status: ProviderStatus::Stub, - env_vars: &["XIAOMI_MIMO_API_KEY"], + env_vars: &["XIAOMI_MIMO_API_KEY", "MIMO_API_KEY"], litellm_prefix: "xiaomi_mimo/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, tool_use: true, embeddings: false, - vision: false, + // mimo-v2-omni supports image input; advertise vision at provider level. + vision: true, batch: false, }, }; -pub const MODELS: &[ModelDef] = &[]; +// Model specs per Xiaomi platform docs (platform.xiaomimimo.com) and the +// MiMo-V2-Flash technical report. Context windows: Flash 262,144, Pro +// 1,048,576, Omni 262,144. Max output tokens from the same source. +pub const MODELS: &[ModelDef] = &[ + ModelDef { + id: "mimo-v2-flash", + provider_id: "xiaomi_mimo", + context_window: 262_144, + max_output_tokens: 8_192, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: false, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mimo-v2-pro", + provider_id: "xiaomi_mimo", + context_window: 1_048_576, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: false, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, + ModelDef { + id: "mimo-v2-omni", + provider_id: "xiaomi_mimo", + context_window: 262_144, + max_output_tokens: 32_000, + capabilities: ModelCapabilities { + streaming: true, + tool_use: true, + vision: true, + extended_thinking: true, + }, + status: ModelStatus::Available, + }, +]; diff --git a/crates/providers/src/providers/xinference.rs b/crates/providers/src/providers/xinference.rs index e6d43d5..316d846 100644 --- a/crates/providers/src/providers/xinference.rs +++ b/crates/providers/src/providers/xinference.rs @@ -3,24 +3,34 @@ use crate::provider::{ AuthKind, ProviderCapabilities, ProviderDef, ProviderProtocol, ProviderStatus, }; -/// Xinference — self-hosted inference server; set endpoint via XINFERENCE_SERVER_URL. +/// Xinference (Xorbits Inference) — self-hosted inference server exposing an +/// OpenAI-compatible REST API at `/v1` (chat, embeddings, images, audio). +/// Default listener is `http://localhost:9997`; override via the proxy's +/// per-backend base URL config when running remotely. pub const PROVIDER: ProviderDef = ProviderDef { id: "xinference", display_name: "Xinference", - default_base_url: "", + default_base_url: "http://localhost:9997/v1", protocol: ProviderProtocol::OpenAICompat, auth: AuthKind::None, status: ProviderStatus::Stub, - env_vars: &["XINFERENCE_SERVER_URL"], + // Self-hosted: no API key required by default. Any non-empty string is + // accepted if one is sent, so no canonical env var is defined. + env_vars: &[], litellm_prefix: "xinference/", capabilities: ProviderCapabilities { chat_completions: true, streaming: true, - tool_use: false, + // Xinference supports OpenAI-style function/tool calling for compatible models. + tool_use: true, embeddings: true, - vision: false, + // Vision-capable multimodal LLMs are supported when the user launches one. + vision: true, + // No server-side OpenAI-style batch API. batch: false, }, }; +// Models are user-deployed at runtime (launched via Xinference's own API), +// so there is no static catalog to list here. pub const MODELS: &[ModelDef] = &[]; diff --git a/crates/providers/src/registry.rs b/crates/providers/src/registry.rs index 0db3ca1..e042844 100644 --- a/crates/providers/src/registry.rs +++ b/crates/providers/src/registry.rs @@ -293,9 +293,10 @@ mod tests { } #[test] - fn stub_models_empty() { - // Stubs intentionally ship with no model list until filled in. - assert!(list_models("groq").is_empty()); + fn self_hosted_models_empty() { + // Self-hosted providers have no static model list by design. + assert!(list_models("ollama").is_empty()); + assert!(list_models("hosted_vllm").is_empty()); } #[test]