diff --git a/crates/proxy/src/backend/gemini_client.rs b/crates/proxy/src/backend/gemini_client.rs new file mode 100644 index 0000000..1752b32 --- /dev/null +++ b/crates/proxy/src/backend/gemini_client.rs @@ -0,0 +1,258 @@ +// Gemini native HTTP client for generateContent / streamGenerateContent endpoints. +// No OpenAI translation: sends and receives Gemini-native JSON directly. + +use super::build_http_client; +use crate::config::TlsConfig; +use anyllm_translate::gemini::{GenerateContentRequest, GenerateContentResponse}; +use reqwest::Client; + +/// HTTP client for Google Gemini's native generateContent API. +#[derive(Clone)] +pub struct GeminiNativeClient { + client: Client, + base_url: String, + api_key: String, + big_model: String, + small_model: String, +} + +/// Error type for the Gemini native client. +#[derive(Debug)] +pub enum GeminiClientError { + /// Transport-level error (connection, timeout, DNS). + Transport(String), + /// Upstream returned a non-success status. + ApiError { status: u16, body: String }, + /// Response body could not be deserialized. + Deserialize(String), +} + +impl std::fmt::Display for GeminiClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Transport(e) => write!(f, "Gemini transport error: {e}"), + Self::ApiError { status, body } => { + write!(f, "Gemini API error (status {status}): {body}") + } + Self::Deserialize(e) => write!(f, "Gemini deserialization error: {e}"), + } + } +} + +impl std::error::Error for GeminiClientError {} + +impl GeminiNativeClient { + /// Create a new Gemini native client. + /// + /// `base_url` should be the Gemini API root, e.g. + /// `https://generativelanguage.googleapis.com/v1beta`. + pub fn new( + base_url: String, + api_key: String, + big_model: String, + small_model: String, + tls: &TlsConfig, + ) -> Self { + let client = build_http_client(tls); + Self { + client, + base_url, + api_key, + big_model, + small_model, + } + } + + pub fn big_model(&self) -> &str { + &self.big_model + } + + pub fn small_model(&self) -> &str { + &self.small_model + } + + /// Map an Anthropic model name to the configured Gemini model. + pub fn map_model(&self, anthropic_model: &str) -> String { + let lower = anthropic_model.to_lowercase(); + if lower.contains("haiku") { + self.small_model.clone() + } else { + self.big_model.clone() + } + } + + /// Build the generateContent URL for a given model. + fn generate_url(&self, model: &str) -> String { + format!( + "{}/models/{}:generateContent", + self.base_url.trim_end_matches('/'), + model + ) + } + + /// Build the streamGenerateContent URL for a given model. + fn stream_url(&self, model: &str) -> String { + format!( + "{}/models/{}:streamGenerateContent?alt=sse", + self.base_url.trim_end_matches('/'), + model + ) + } + + /// Non-streaming: POST generateContent, parse response. + pub async fn generate_content( + &self, + body: &GenerateContentRequest, + model: &str, + ) -> Result { + let url = self.generate_url(model); + let resp = self + .client + .post(&url) + .header("x-goog-api-key", &self.api_key) + .header("Content-Type", "application/json") + .json(body) + .send() + .await + .map_err(|e| GeminiClientError::Transport(e.to_string()))?; + + let status = resp.status(); + if !status.is_success() { + let body_text = resp.text().await.unwrap_or_default(); + return Err(GeminiClientError::ApiError { + status: status.as_u16(), + body: body_text, + }); + } + + resp.json::() + .await + .map_err(|e| GeminiClientError::Deserialize(e.to_string())) + } + + /// Streaming: POST streamGenerateContent, return raw Response for SSE reading. + pub async fn generate_content_stream( + &self, + body: &GenerateContentRequest, + model: &str, + ) -> Result { + let url = self.stream_url(model); + let resp = self + .client + .post(&url) + .header("x-goog-api-key", &self.api_key) + .header("Content-Type", "application/json") + .json(body) + .send() + .await + .map_err(|e| GeminiClientError::Transport(e.to_string()))?; + + let status = resp.status(); + if !status.is_success() { + let body_text = resp.text().await.unwrap_or_default(); + return Err(GeminiClientError::ApiError { + status: status.as_u16(), + body: body_text, + }); + } + + Ok(resp) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_client(base_url: &str) -> GeminiNativeClient { + GeminiNativeClient::new( + base_url.to_string(), + "test-key".to_string(), + "gemini-2.5-pro".to_string(), + "gemini-2.5-flash".to_string(), + &TlsConfig::default(), + ) + } + + #[test] + fn generate_url_construction() { + let c = test_client("https://generativelanguage.googleapis.com/v1beta"); + assert_eq!( + c.generate_url("gemini-2.5-pro"), + "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent" + ); + } + + #[test] + fn stream_url_construction() { + let c = test_client("https://generativelanguage.googleapis.com/v1beta"); + assert_eq!( + c.stream_url("gemini-2.5-pro"), + "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse" + ); + } + + #[test] + fn map_model_haiku_to_small() { + let c = test_client("https://example.com"); + assert_eq!(c.map_model("claude-3-haiku-20240307"), "gemini-2.5-flash"); + assert_eq!(c.map_model("claude-sonnet-4-6"), "gemini-2.5-pro"); + } + + #[test] + fn map_model_case_insensitive() { + let c = test_client("https://example.com"); + assert_eq!(c.map_model("Claude-3-HAIKU-20240307"), "gemini-2.5-flash"); + } + + #[test] + fn base_url_trailing_slash_stripped() { + let c = test_client("https://example.com/v1beta/"); + let url = c.generate_url("pro"); + assert!( + url.contains("/v1beta/models/pro:generateContent"), + "got: {url}" + ); + assert!(!url.contains("//models"), "double slash in: {url}"); + } + + #[test] + fn stream_url_trailing_slash_stripped() { + let c = test_client("https://example.com/v1beta/"); + let url = c.stream_url("pro"); + assert!(!url.contains("//models"), "double slash in: {url}"); + } + + #[test] + fn error_display_transport() { + let e = GeminiClientError::Transport("connection refused".to_string()); + let s = e.to_string(); + assert!(s.contains("transport"), "got: {s}"); + assert!(s.contains("connection refused"), "got: {s}"); + } + + #[test] + fn error_display_api() { + let e = GeminiClientError::ApiError { + status: 429, + body: "rate limited".to_string(), + }; + let s = e.to_string(); + assert!(s.contains("429"), "got: {s}"); + assert!(s.contains("rate limited"), "got: {s}"); + } + + #[test] + fn error_display_deserialize() { + let e = GeminiClientError::Deserialize("unexpected token".to_string()); + let s = e.to_string(); + assert!(s.contains("deserialization"), "got: {s}"); + } + + #[test] + fn model_accessors() { + let c = test_client("https://example.com"); + assert_eq!(c.big_model(), "gemini-2.5-pro"); + assert_eq!(c.small_model(), "gemini-2.5-flash"); + } +} diff --git a/crates/proxy/src/backend/mod.rs b/crates/proxy/src/backend/mod.rs index d1e688e..749ea47 100644 --- a/crates/proxy/src/backend/mod.rs +++ b/crates/proxy/src/backend/mod.rs @@ -2,12 +2,15 @@ pub mod anthropic_client; /// AWS Bedrock client with SigV4 request signing. pub mod bedrock_client; +/// Gemini native generateContent client (no OpenAI translation layer). +pub mod gemini_client; /// reqwest client for OpenAI-compatible Chat Completions and Responses APIs with retry/backoff. pub mod openai_client; use crate::config::{BackendAuth, BackendConfig, BackendKind, Config, OpenAIApiFormat, TlsConfig}; use anthropic_client::{AnthropicClient, AnthropicClientError}; use bedrock_client::{BedrockClient, BedrockClientError}; +use gemini_client::{GeminiClientError, GeminiNativeClient}; use openai_client::{OpenAIClient, OpenAIClientError}; // Re-export from the client crate so existing code paths (streaming, routes, etc.) keep working. @@ -71,6 +74,8 @@ pub enum BackendClient { Anthropic(AnthropicClient), /// AWS Bedrock: sends Anthropic-format requests with SigV4 signing. Bedrock(BedrockClient), + /// Gemini native: sends generateContent requests directly (no OpenAI translation). + GeminiNative(GeminiNativeClient), } /// Unified error type for all backend clients. @@ -79,6 +84,7 @@ pub enum BackendError { OpenAI(OpenAIClientError), Anthropic(AnthropicClientError), Bedrock(BedrockClientError), + Gemini(GeminiClientError), } impl BackendError { @@ -87,6 +93,7 @@ impl BackendError { match self { Self::OpenAI(OpenAIClientError::ApiError { status, .. }) => Some(*status), Self::Bedrock(BedrockClientError::ApiError { status, .. }) => Some(*status), + Self::Gemini(GeminiClientError::ApiError { status, .. }) => Some(*status), _ => None, } } @@ -102,6 +109,7 @@ impl BackendError { Self::OpenAI(e) => e.to_string(), Self::Anthropic(e) => e.to_string(), Self::Bedrock(e) => e.to_string(), + Self::Gemini(e) => e.to_string(), } } @@ -123,6 +131,7 @@ impl std::fmt::Display for BackendError { Self::OpenAI(e) => write!(f, "{e}"), Self::Anthropic(e) => write!(f, "{e}"), Self::Bedrock(e) => write!(f, "{e}"), + Self::Gemini(e) => write!(f, "{e}"), } } } @@ -145,6 +154,12 @@ impl From for BackendError { } } +impl From for BackendError { + fn from(e: GeminiClientError) -> Self { + Self::Gemini(e) + } +} + impl BackendClient { /// Forward a raw request to a passthrough endpoint (audio, images, etc.). /// Returns `501 Not Implemented` for Anthropic/Bedrock backends. @@ -165,7 +180,7 @@ impl BackendClient { .await .map_err(BackendError::OpenAI) } - Self::Anthropic(_) | Self::Bedrock(_) => { + Self::Anthropic(_) | Self::Bedrock(_) | Self::GeminiNative(_) => { let err = anyllm_translate::mapping::errors_map::create_anthropic_error( anyllm_translate::anthropic::ErrorType::InvalidRequestError, format!( @@ -200,8 +215,8 @@ impl BackendClient { .embeddings_passthrough(body, content_type) .await .map_err(BackendError::OpenAI), - Self::Anthropic(_) | Self::Bedrock(_) => { - // Anthropic and Bedrock have no embeddings API. + Self::Anthropic(_) | Self::Bedrock(_) | Self::GeminiNative(_) => { + // Anthropic, Bedrock, and Gemini native have no embeddings passthrough. let err = anyllm_translate::mapping::errors_map::create_anthropic_error( anyllm_translate::anthropic::ErrorType::InvalidRequestError, "Embeddings are not supported by this backend.".to_string(), diff --git a/crates/proxy/src/server/chat_completions.rs b/crates/proxy/src/server/chat_completions.rs index 4fe95f1..44a8c65 100644 --- a/crates/proxy/src/server/chat_completions.rs +++ b/crates/proxy/src/server/chat_completions.rs @@ -382,7 +382,7 @@ pub(crate) async fn chat_completions( } } } - BackendClient::Anthropic(_) | BackendClient::Bedrock(_) => openai_error_response( + BackendClient::Anthropic(_) | BackendClient::Bedrock(_) | BackendClient::GeminiNative(_) => openai_error_response( "This backend does not support /v1/chat/completions. Use /v1/messages instead.", "invalid_request_error", StatusCode::BAD_REQUEST, @@ -449,7 +449,7 @@ async fn chat_completions_stream( | BackendClient::Vertex(c) | BackendClient::GeminiOpenAI(c) | BackendClient::OpenAIResponses(c) => c.clone(), - BackendClient::Anthropic(_) | BackendClient::Bedrock(_) => { + BackendClient::Anthropic(_) | BackendClient::Bedrock(_) | BackendClient::GeminiNative(_) => { return openai_error_response( "This backend does not support /v1/chat/completions. Use /v1/messages instead.", "invalid_request_error", diff --git a/crates/proxy/src/server/routes.rs b/crates/proxy/src/server/routes.rs index d53b4a9..6c90fad 100644 --- a/crates/proxy/src/server/routes.rs +++ b/crates/proxy/src/server/routes.rs @@ -969,8 +969,8 @@ async fn messages( } } } - BackendClient::Anthropic(_) | BackendClient::Bedrock(_) => { - // These backends are handled by separate handlers (passthrough / Bedrock). + BackendClient::Anthropic(_) | BackendClient::Bedrock(_) | BackendClient::GeminiNative(_) => { + // These backends are handled by separate handlers (passthrough / Bedrock / Gemini native). // If we reach here, something is misconfigured. let err = mapping::errors_map::create_anthropic_error( anthropic::ErrorType::ApiError, diff --git a/crates/proxy/src/server/streaming.rs b/crates/proxy/src/server/streaming.rs index 6122447..5668e48 100644 --- a/crates/proxy/src/server/streaming.rs +++ b/crates/proxy/src/server/streaming.rs @@ -384,7 +384,7 @@ pub(crate) async fn messages_stream( } }); } - BackendClient::Anthropic(_) | BackendClient::Bedrock(_) => { + BackendClient::Anthropic(_) | BackendClient::Bedrock(_) | BackendClient::GeminiNative(_) => { drop(rl_tx); let _ = tx .send(Ok(Event::default().data(